-
Notifications
You must be signed in to change notification settings - Fork 2.2k
fix(file source): avoid warning for empty files too small to fingerprint #25897
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Avoid emitting a warning for empty files that are too small to fingerprint. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -202,9 +202,12 @@ impl Fingerprinter { | |
| known_small_files: &mut HashMap<PathBuf, time::Instant>, | ||
| emitter: &impl FileSourceInternalEvents, | ||
| ) -> Option<FileFingerprint> { | ||
| let mut file_size = None; | ||
|
|
||
| let metadata = match fs::metadata(path).await { | ||
| Ok(metadata) => { | ||
| if !metadata.is_dir() { | ||
| file_size = Some(metadata.len()); | ||
| self.fingerprint(path).await.map(Some) | ||
| } else { | ||
| Ok(None) | ||
|
|
@@ -222,7 +225,9 @@ impl Fingerprinter { | |
| match error.kind() { | ||
| ErrorKind::UnexpectedEof => { | ||
| if !known_small_files.contains_key(path) { | ||
| emitter.emit_file_checksum_failed(path); | ||
| if file_size != Some(0) { | ||
| emitter.emit_file_checksum_failed(path); | ||
|
Comment on lines
+228
to
+229
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a matched file is first scanned while empty, this branch suppresses the warning but still inserts its path into Useful? React with 👍 / 👎. |
||
| } | ||
| known_small_files.insert(path.to_path_buf(), time::Instant::now()); | ||
| } | ||
| return; | ||
|
|
@@ -277,7 +282,17 @@ async fn fingerprinter_read_until( | |
|
|
||
| #[cfg(test)] | ||
| mod test { | ||
| use std::{collections::HashMap, fs, io::Error, path::Path, time::Duration}; | ||
| use std::{ | ||
| collections::HashMap, | ||
| fs, | ||
| io::Error, | ||
| path::Path, | ||
| sync::{ | ||
| Arc, | ||
| atomic::{AtomicUsize, Ordering}, | ||
| }, | ||
| time::Duration, | ||
| }; | ||
|
|
||
| use async_compression::tokio::bufread::GzipEncoder; | ||
| use bytes::BytesMut; | ||
|
|
@@ -638,6 +653,46 @@ mod test { | |
| ); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn fingerprint_or_emit_only_warns_on_non_empty_small_files() { | ||
| let target_dir = tempdir().unwrap(); | ||
| let empty_path = target_dir.path().join("empty.log"); | ||
| let too_small_path = target_dir.path().join("too_small.log"); | ||
|
|
||
| fs::write(&empty_path, []).unwrap(); | ||
| fs::write(&too_small_path, b"missing newline").unwrap(); | ||
|
|
||
| let mut fingerprinter = Fingerprinter::new( | ||
| FingerprintStrategy::FirstLinesChecksum { | ||
| ignored_header_bytes: 0, | ||
| lines: 1, | ||
| }, | ||
| 1024, | ||
| false, | ||
| ); | ||
|
|
||
| let emitter = RecordingEmitter::default(); | ||
| let mut small_files = HashMap::new(); | ||
|
|
||
| assert!( | ||
| fingerprinter | ||
| .fingerprint_or_emit(&empty_path, &mut small_files, &emitter) | ||
| .await | ||
| .is_none() | ||
| ); | ||
| assert_eq!(emitter.checksum_failed_count(), 0); | ||
| assert!(small_files.contains_key(&empty_path)); | ||
|
|
||
| assert!( | ||
| fingerprinter | ||
| .fingerprint_or_emit(&too_small_path, &mut small_files, &emitter) | ||
| .await | ||
| .is_none() | ||
| ); | ||
| assert_eq!(emitter.checksum_failed_count(), 1); | ||
| assert!(small_files.contains_key(&too_small_path)); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_monotonic_compression_algorithms() { | ||
| // This test is necessary to handle an edge case where when assessing the magic header | ||
|
|
@@ -657,6 +712,47 @@ mod test { | |
| #[derive(Clone)] | ||
| struct NoErrors; | ||
|
|
||
| #[derive(Clone, Default)] | ||
| struct RecordingEmitter { | ||
| checksum_failed: Arc<AtomicUsize>, | ||
| } | ||
|
|
||
| impl RecordingEmitter { | ||
| fn checksum_failed_count(&self) -> usize { | ||
| self.checksum_failed.load(Ordering::SeqCst) | ||
| } | ||
| } | ||
|
|
||
| impl FileSourceInternalEvents for RecordingEmitter { | ||
| fn emit_file_added(&self, _: &Path) {} | ||
|
|
||
| fn emit_file_resumed(&self, _: &Path, _: u64) {} | ||
|
|
||
| fn emit_file_watch_error(&self, _: &Path, _: Error) {} | ||
|
|
||
| fn emit_file_unwatched(&self, _: &Path, _: bool) {} | ||
|
|
||
| fn emit_file_deleted(&self, _: &Path) {} | ||
|
|
||
| fn emit_file_delete_error(&self, _: &Path, _: Error) {} | ||
|
|
||
| fn emit_file_fingerprint_read_error(&self, _: &Path, _: Error) {} | ||
|
|
||
| fn emit_file_checkpointed(&self, _: usize, _: Duration) {} | ||
|
|
||
| fn emit_file_checksum_failed(&self, _: &Path) { | ||
| self.checksum_failed.fetch_add(1, Ordering::SeqCst); | ||
| } | ||
|
|
||
| fn emit_file_checkpoint_write_error(&self, _: Error) {} | ||
|
|
||
| fn emit_files_open(&self, _: usize) {} | ||
|
|
||
| fn emit_path_globbing_failed(&self, _: &Path, _: &Error) {} | ||
|
|
||
| fn emit_file_line_too_long(&self, _: &BytesMut, _: usize, _: usize) {} | ||
| } | ||
|
|
||
| impl FileSourceInternalEvents for NoErrors { | ||
| fn emit_file_added(&self, _: &Path) {} | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For a valid gzip file whose decompressed stream is empty,
metadata.len()is nonzero because it measures the compressed container, whilefingerprintreads the decompressed stream and returnsUnexpectedEof. The new check therefore still emitsemit_file_checksum_failedfor this supported form of empty input, contrary to the changelog's empty-file behavior. Determine emptiness from the stream being fingerprinted rather than the on-disk compressed size.Useful? React with 👍 / 👎.