Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

152 changes: 0 additions & 152 deletions datafusion/core/tests/fuzz_cases/merge_fuzz.rs

This file was deleted.

1 change: 0 additions & 1 deletion datafusion/core/tests/fuzz_cases/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ mod aggregate_fuzz;
mod distinct_count_string_fuzz;
#[expect(clippy::needless_pass_by_value)]
mod join_fuzz;
mod merge_fuzz;
mod null_aware_join_fuzz;
#[expect(clippy::needless_pass_by_value)]
mod sort_fuzz;
Expand Down
1 change: 1 addition & 0 deletions datafusion/fuzz/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ bytes = { workspace = true }
chrono = { workspace = true }
datafusion = { workspace = true }
datafusion-common = { workspace = true }
datafusion-datasource = { workspace = true }
datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-physical-plan = { workspace = true }
Expand Down
14 changes: 14 additions & 0 deletions datafusion/fuzz/tests/fuzz.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,20 @@ async fn timestamp_arithmetic() -> Result<()> {
.await
}

#[tokio::test]
async fn merge() -> Result<()> {
run_suite(
&suites::merge::MergeFuzzer,
RunConfig {
run_seed: None,
max_cases: None,
time_limit: Some(Duration::from_secs(4)),
cases_per_dataset: 7,
},
)
.await
}

async fn run_suite<T: FuzzTest>(test: &T, config: RunConfig) -> Result<()> {
eprintln!("Running fuzz suite: {}", test.name());
let stats = run(test, config).await?;
Expand Down
114 changes: 114 additions & 0 deletions datafusion/fuzz/tests/suites/merge.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Checks sorted-stream merging against sorting the input values, with seeded
//! input layouts, batch boundaries, and empty batches.

use std::future::Future;
use std::sync::Arc;

use arrow::array::{ArrayRef, Int32Array};
use arrow::compute::SortOptions;
use arrow::record_batch::RecordBatch;
use datafusion_common::Result;
use datafusion_datasource::memory::MemorySourceConfig;
use datafusion_execution::TaskContext;
use datafusion_execution::config::SessionConfig;
use datafusion_physical_plan::collect;
use datafusion_physical_plan::expressions::{PhysicalSortExpr, col};
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use test_utils::fuzz::FuzzTest;
use test_utils::{batches_to_vec, partitions_to_sorted_vec, stagger_batch_with_seed};

type MergeLayout = (&'static str, &'static [&'static [(i32, i32)]]);

// (name, partitions), where each partition lists [start, end) value ranges.
static MERGE_LAYOUTS: &[MergeLayout] = &[
("two", &[&[(0, 100)], &[(0, 100)]]),
("two_no_overlap", &[&[(0, 20)], &[(20, 40)]]),
("three", &[&[(0, 100)], &[(0, 100)], &[(0, 51)]]),
(
"three_gaps",
&[&[(0, 50), (50, 100)], &[(0, 33), (50, 123)], &[(0, 51)]],
),
];

pub(crate) struct MergeFuzzer;

pub(crate) struct MergeDataset {
name: &'static str,
input: Vec<Vec<RecordBatch>>,
expected: Vec<Option<i32>>,
}

impl FuzzTest for MergeFuzzer {
type Dataset = MergeDataset;

fn name(&self) -> &str {
"merge"
}

fn prepare_dataset(&self, seed: u64) -> impl Future<Output = Result<Self::Dataset>> {
let mut rng = StdRng::seed_from_u64(seed);
let (name, ranges) = MERGE_LAYOUTS[rng.random_range(0..MERGE_LAYOUTS.len())];
let input: Vec<Vec<RecordBatch>> = ranges
.iter()
.map(|partition| {
partition
.iter()
.flat_map(|&(low, high)| {
let array: ArrayRef =
Arc::new(Int32Array::from_iter_values(low..high));
let batch = RecordBatch::try_from_iter([("x", array)]).unwrap();
stagger_batch_with_seed(batch, rng.random())
})
.collect()
})
.collect();
let expected = partitions_to_sorted_vec(&input);
std::future::ready(Ok(MergeDataset {
name,
input,
expected,
}))
}

async fn run_case(&self, dataset: &Self::Dataset, seed: u64) -> Result<()> {
// Seven consecutive case seeds cover every output batch size on one dataset.
let batch_size = [1, 2, 7, 49, 50, 51, 100][(seed % 7) as usize];
let name = dataset.name;
eprintln!("Merge layout: {name}, batch size: {batch_size}");
let schema = dataset.input[0][0].schema();
let sort = [PhysicalSortExpr::new(
col("x", &schema)?,
SortOptions::default(),
)]
.into();
let input = MemorySourceConfig::try_new_exec(&dataset.input, schema, None)?;
let merge = Arc::new(SortPreservingMergeExec::new(sort, input));
let task_ctx = Arc::new(
TaskContext::default()
.with_session_config(SessionConfig::new().with_batch_size(batch_size)),
);
let collected = collect(merge, task_ctx).await?;

assert_eq!(
dataset.expected,
batches_to_vec(&collected),
"{name}, batch size {batch_size}"
);
if let Some((_, full_batches)) = collected.split_last() {
for batch in full_batches {
assert_eq!(
batch.num_rows(),
batch_size,
"{name}, batch size {batch_size}"
);
}
}
Ok(())
}
}
1 change: 1 addition & 0 deletions datafusion/fuzz/tests/suites/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ pub(crate) mod key_range_bitmap;
pub(crate) mod limit;
pub(crate) mod lru_queue;
pub(crate) mod memory;
pub(crate) mod merge;
pub(crate) mod spill_pool;
pub(crate) mod timestamp;
Loading