Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/1065-empty-file-fingerprint-warning.fix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Avoid emitting a warning for empty files that are too small to fingerprint.
100 changes: 98 additions & 2 deletions lib/file-source-common/src/fingerprinter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -202,9 +202,12 @@ impl Fingerprinter {
known_small_files: &mut HashMap<PathBuf, time::Instant>,
emitter: &impl FileSourceInternalEvents,
) -> Option<FileFingerprint> {
let mut file_size = None;

let metadata = match fs::metadata(path).await {
Ok(metadata) => {
if !metadata.is_dir() {
file_size = Some(metadata.len());

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Treat empty compressed streams as empty

For a valid gzip file whose decompressed stream is empty, metadata.len() is nonzero because it measures the compressed container, while fingerprint reads the decompressed stream and returns UnexpectedEof. The new check therefore still emits emit_file_checksum_failed for this supported form of empty input, contrary to the changelog's empty-file behavior. Determine emptiness from the stream being fingerprinted rather than the on-disk compressed size.

Useful? React with 👍 / 👎.

self.fingerprint(path).await.map(Some)
} else {
Ok(None)
Expand All @@ -222,7 +225,9 @@ impl Fingerprinter {
match error.kind() {
ErrorKind::UnexpectedEof => {
if !known_small_files.contains_key(path) {
emitter.emit_file_checksum_failed(path);
if file_size != Some(0) {
emitter.emit_file_checksum_failed(path);
Comment on lines +228 to +229

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Warn when an initially empty file gains partial content

When a matched file is first scanned while empty, this branch suppresses the warning but still inserts its path into known_small_files. If the file later gains non-empty content without enough newlines to fingerprint, subsequent discovery scans skip this entire block because the path is already present, so the warning that this change intends to preserve for non-empty undersized files is never emitted. Track whether a warning was emitted separately from the removal-tracking map, or allow the first non-empty failure to emit it.

Useful? React with 👍 / 👎.

}
known_small_files.insert(path.to_path_buf(), time::Instant::now());
}
return;
Expand Down Expand Up @@ -277,7 +282,17 @@ async fn fingerprinter_read_until(

#[cfg(test)]
mod test {
use std::{collections::HashMap, fs, io::Error, path::Path, time::Duration};
use std::{
collections::HashMap,
fs,
io::Error,
path::Path,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};

use async_compression::tokio::bufread::GzipEncoder;
use bytes::BytesMut;
Expand Down Expand Up @@ -638,6 +653,46 @@ mod test {
);
}

#[tokio::test]
async fn fingerprint_or_emit_only_warns_on_non_empty_small_files() {
let target_dir = tempdir().unwrap();
let empty_path = target_dir.path().join("empty.log");
let too_small_path = target_dir.path().join("too_small.log");

fs::write(&empty_path, []).unwrap();
fs::write(&too_small_path, b"missing newline").unwrap();

let mut fingerprinter = Fingerprinter::new(
FingerprintStrategy::FirstLinesChecksum {
ignored_header_bytes: 0,
lines: 1,
},
1024,
false,
);

let emitter = RecordingEmitter::default();
let mut small_files = HashMap::new();

assert!(
fingerprinter
.fingerprint_or_emit(&empty_path, &mut small_files, &emitter)
.await
.is_none()
);
assert_eq!(emitter.checksum_failed_count(), 0);
assert!(small_files.contains_key(&empty_path));

assert!(
fingerprinter
.fingerprint_or_emit(&too_small_path, &mut small_files, &emitter)
.await
.is_none()
);
assert_eq!(emitter.checksum_failed_count(), 1);
assert!(small_files.contains_key(&too_small_path));
}

#[test]
fn test_monotonic_compression_algorithms() {
// This test is necessary to handle an edge case where when assessing the magic header
Expand All @@ -657,6 +712,47 @@ mod test {
#[derive(Clone)]
struct NoErrors;

#[derive(Clone, Default)]
struct RecordingEmitter {
checksum_failed: Arc<AtomicUsize>,
}

impl RecordingEmitter {
fn checksum_failed_count(&self) -> usize {
self.checksum_failed.load(Ordering::SeqCst)
}
}

impl FileSourceInternalEvents for RecordingEmitter {
fn emit_file_added(&self, _: &Path) {}

fn emit_file_resumed(&self, _: &Path, _: u64) {}

fn emit_file_watch_error(&self, _: &Path, _: Error) {}

fn emit_file_unwatched(&self, _: &Path, _: bool) {}

fn emit_file_deleted(&self, _: &Path) {}

fn emit_file_delete_error(&self, _: &Path, _: Error) {}

fn emit_file_fingerprint_read_error(&self, _: &Path, _: Error) {}

fn emit_file_checkpointed(&self, _: usize, _: Duration) {}

fn emit_file_checksum_failed(&self, _: &Path) {
self.checksum_failed.fetch_add(1, Ordering::SeqCst);
}

fn emit_file_checkpoint_write_error(&self, _: Error) {}

fn emit_files_open(&self, _: usize) {}

fn emit_path_globbing_failed(&self, _: &Path, _: &Error) {}

fn emit_file_line_too_long(&self, _: &BytesMut, _: usize, _: usize) {}
}

impl FileSourceInternalEvents for NoErrors {
fn emit_file_added(&self, _: &Path) {}

Expand Down
Loading