Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 3 additions & 60 deletions datafusion/common/src/utils/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -819,10 +819,9 @@ mod tests {
mod record_batch_tests {
use super::*;
use arrow::array::{
ArrayData, ArrayRef, BinaryViewArray, BooleanArray, DictionaryArray,
Float64Array, Int16Array, Int32Array, Int64Array, LargeListViewArray, ListArray,
ListViewArray, MapArray, RunArray, StringArray, StringViewArray, StructArray,
UnionArray, new_null_array,
ArrayData, ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int16Array,
Int32Array, Int64Array, LargeListViewArray, ListArray, ListViewArray, MapArray,
RunArray, StringArray, StringViewArray, StructArray, UnionArray, new_null_array,
};
use arrow::buffer::OffsetBuffer;
use arrow::datatypes::{
Expand Down Expand Up @@ -1592,60 +1591,4 @@ mod record_batch_tests {
assert_eq!(tracker.memory_usage(), full);
assert_eq!(tracker.uncount_batch_with_array_overhead(&b2), full);
}

/// A random sequence of counts and uncounts, checked against counting the
/// arrays still held from scratch.
#[test]
fn test_randomized_count_uncount_sequence() {
let int32 = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
let string = Arc::new(StringArray::from(vec!["a", "bc", "def"])) as ArrayRef;
let string_view = Arc::new(StringViewArray::from_iter_values([
"short",
"another long string that also spills to data buffers for sharing",
])) as ArrayRef;
let dictionary = Arc::new(
DictionaryArray::<Int32Type>::try_new(
Int32Array::from(vec![0, 1, 2]),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
)
.unwrap(),
) as ArrayRef;
let arrays = [
int32.slice(0, 2),
int32.slice(1, 2),
Arc::clone(&string),
string.slice(1, 2),
string_view.slice(0, 1),
string_view.slice(1, 1),
dictionary.slice(0, 2),
dictionary,
Arc::new(BooleanArray::from(vec![true, false])) as ArrayRef,
];

let mut seed: u64 = 42;
let mut next_rand = || {
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
(seed >> 33) as usize
};
let mut tracker = RecordBatchMemoryTracker::new();
let mut held = vec![0usize; arrays.len()];
for op in 0..300 {
let idx = next_rand() % arrays.len();
if held[idx] > 0 && next_rand() % 2 == 0 {
tracker.uncount_array(arrays[idx].as_ref());
held[idx] -= 1;
} else {
tracker.count_array(arrays[idx].as_ref());
held[idx] += 1;
}

let mut expected = RecordBatchMemoryCounter::new();
for (array, &count) in arrays.iter().zip(&held) {
if count > 0 {
expected.count_array(array.as_ref());
}
}
assert_eq!(tracker.memory_usage(), expected.memory_usage(), "op {op}");
}
}
}
14 changes: 14 additions & 0 deletions datafusion/fuzz/tests/fuzz.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,20 @@ async fn sort_topk() -> Result<()> {
.await
}

#[tokio::test]
async fn record_batch_memory_tracker() -> Result<()> {
run_suite(
&suites::memory::RecordBatchMemoryFuzzer,
RunConfig {
run_seed: None,
max_cases: None,
time_limit: Some(Duration::from_secs(1)),
cases_per_dataset: 100,
},
)
.await
}

async fn run_suite<T: FuzzTest>(test: &T, config: RunConfig) -> Result<()> {
eprintln!("Running fuzz suite: {}", test.name());
let stats = run(test, config).await?;
Expand Down
89 changes: 89 additions & 0 deletions datafusion/fuzz/tests/suites/memory.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Checks incremental memory accounting against recounting the retained arrays
//! after each operation, including arrays that share buffers.

use std::future::Future;
use std::sync::Arc;

use arrow::array::{
ArrayRef, BooleanArray, DictionaryArray, Int32Array, StringArray, StringViewArray,
};
use arrow::datatypes::Int32Type;
use datafusion_common::Result;
use datafusion_common::utils::memory::{
RecordBatchMemoryCounter, RecordBatchMemoryTracker,
};
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use test_utils::fuzz::FuzzTest;

pub(crate) struct RecordBatchMemoryFuzzer;

impl FuzzTest for RecordBatchMemoryFuzzer {
type Dataset = Vec<ArrayRef>;

fn name(&self) -> &str {
"record_batch_memory_tracker"
}

fn prepare_dataset(&self, _seed: u64) -> impl Future<Output = Result<Self::Dataset>> {
let int32: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
let string: ArrayRef = Arc::new(StringArray::from(vec!["a", "bc", "def"]));
let string_view: ArrayRef = Arc::new(StringViewArray::from_iter_values([
"short",
"another long string that also spills to data buffers for sharing",
]));
let dictionary: ArrayRef = Arc::new(
DictionaryArray::<Int32Type>::try_new(
Int32Array::from(vec![0, 1, 2]),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
)
.unwrap(),
);

std::future::ready(Ok(vec![
int32.slice(0, 2),
int32.slice(1, 2),
Arc::clone(&string),
string.slice(1, 2),
string_view.slice(0, 1),
string_view.slice(1, 1),
dictionary.slice(0, 2),
dictionary,
Arc::new(BooleanArray::from(vec![true, false])),
]))
}

fn run_case(
&self,
arrays: &Self::Dataset,
seed: u64,
) -> impl Future<Output = Result<()>> {
let mut rng = StdRng::seed_from_u64(seed);
let mut tracker = RecordBatchMemoryTracker::new();
let mut held = vec![0usize; arrays.len()];
for op in 0..300 {
let idx = rng.random_range(0..arrays.len());
if held[idx] > 0 && rng.random_bool(0.5) {
tracker.uncount_array(arrays[idx].as_ref());
held[idx] -= 1;
} else {
tracker.count_array(arrays[idx].as_ref());
held[idx] += 1;
}

let mut expected = RecordBatchMemoryCounter::new();
for (array, &count) in arrays.iter().zip(&held) {
if count > 0 {
expected.count_array(array.as_ref());
}
}
assert_eq!(tracker.memory_usage(), expected.memory_usage(), "op {op}");
}

std::future::ready(Ok(()))
}
}
1 change: 1 addition & 0 deletions datafusion/fuzz/tests/suites/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@
pub(crate) mod key_range_bitmap;
pub(crate) mod limit;
pub(crate) mod lru_queue;
pub(crate) mod memory;
pub(crate) mod spill_pool;
Loading