Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
1f9c729
5777 Enable backtrace for DataFusion
patchwork01 Oct 9, 2025
fcb6d4f
5777 Remove disabled tag
patchwork01 Oct 9, 2025
83106e9
5777 Set RUST_BACKTRACE on compaction tasks
patchwork01 Oct 9, 2025
7c6fb44
5777 Log more error details
patchwork01 Oct 9, 2025
468e4d7
5777 Use if let
patchwork01 Oct 10, 2025
d5b338b
5777 Log debug output separately
patchwork01 Oct 10, 2025
3e7d2f9
5777 Collapse pattern matching
patchwork01 Oct 10, 2025
4bf2e9f
5777 Remove unused import
patchwork01 Oct 10, 2025
1964f6a
5777 Log DataFusion root error
patchwork01 Oct 10, 2025
a066d55
5777 Remove unused imports
patchwork01 Oct 10, 2025
e9dccc9
5777 Disable readahead store
patchwork01 Oct 10, 2025
1f2b9c4
5777 Allow not specifying region min/max in CLI compaction
patchwork01 Oct 13, 2025
15289d9
5777 Re-enable readahead store
patchwork01 Oct 14, 2025
a4f7baa
5777 Disable sorted rows check temporarily
patchwork01 Oct 20, 2025
3ab49ad
5777 Set reqwest log level to debug
patchwork01 Oct 20, 2025
a1aebdf
5777 Disable Rust logging
patchwork01 Oct 21, 2025
26bdb2d
5777 Remove DataFusion backtrace feature
patchwork01 Oct 21, 2025
2788869
Merge branch 'develop' into 5777-large-compaction-datafusion-error
patchwork01 Oct 24, 2025
5c6ff03
Merge branch 'develop' into 5777-large-compaction-datafusion-error
patchwork01 Oct 28, 2025
bdbab48
5777 Enable logging for reqwest library
patchwork01 Oct 28, 2025
f9fe3e6
Merge branch 'develop' into 5777-large-compaction-datafusion-error
m09526 Oct 28, 2025
2725a8a
WIP Add debug
m09526 Oct 28, 2025
22bc76c
Merge branch 'develop' into 5777-large-compaction-datafusion-error
m09526 Oct 29, 2025
e363525
Apply debug backtrace
m09526 Nov 3, 2025
18ea61a
Merge branch 'develop' into 5777-large-compaction-datafusion-error
m09526 Nov 3, 2025
725cb38
Object_store crate patched in
m09526 Nov 3, 2025
7d36d9b
Added change message
m09526 Nov 3, 2025
3e5c4d4
Update depracted method
rtjd6554 Oct 29, 2025
82bef06
Update depracated method in compact_cli
rtjd6554 Oct 29, 2025
9010b4c
Remove unused imports
rtjd6554 Oct 29, 2025
287aea7
Update toml file
rtjd6554 Oct 29, 2025
c3cd40f
5898: Initial framework
rtjd6554 Oct 29, 2025
d3013ce
5898: Update method execution
rtjd6554 Oct 31, 2025
462afe8
5898: Rust corrections
rtjd6554 Oct 31, 2025
45cc03b
5898: Missing comment
rtjd6554 Oct 31, 2025
38d1b7e
5898: Correction for driver usage
rtjd6554 Nov 3, 2025
b1f4e0e
5898: Adjust DSL for base usage
rtjd6554 Nov 3, 2025
78f49ce
5898: Test improvement
rtjd6554 Nov 3, 2025
43242f5
5898: Improve test clarity
rtjd6554 Nov 3, 2025
9701762
5898: Align test design with other elements of application
rtjd6554 Nov 3, 2025
fe5bde7
5898: Simplify test
rtjd6554 Nov 3, 2025
599add9
5389 Add SleeperArtefactsCdkApp
patchwork01 Nov 3, 2025
b423b31
5389 Add jars bucket
patchwork01 Nov 3, 2025
a221596
5389 Rename LambdaJar.all
patchwork01 Nov 3, 2025
2f05ded
5389 Deploy ECR repositories
patchwork01 Nov 3, 2025
1341b18
5389 Add whitespace
patchwork01 Nov 3, 2025
5e14c41
5389 Make bucket name lower case
patchwork01 Nov 3, 2025
be2ab11
5389 Make repository name lower case
patchwork01 Nov 3, 2025
aec2cd7
5389 Set deployment ID in context
patchwork01 Nov 3, 2025
e6e505b
5389 Adjust handling deployment ID
patchwork01 Nov 3, 2025
9313d72
5389 Add missing Javadoc
patchwork01 Nov 3, 2025
8d17c37
5389 Add missing Javadoc
patchwork01 Nov 3, 2025
4239f4b
5389 Add missing private constructor
patchwork01 Nov 3, 2025
bd8aca1
5898: Improve method legibility
rtjd6554 Nov 4, 2025
3792cf5
5927: Removed EMR and remnants of EMR Serverless
Rob9786 Nov 3, 2025
702eb35
5927: Re-added PersistentEMRStepCount
Rob9786 Nov 4, 2025
6099249
5927: Added newline to bottom of class
Rob9786 Nov 4, 2025
eae5476
5927: Removed TerminateEMRClusters
Rob9786 Nov 4, 2025
3775b71
Merge branch 'develop' into 5777-large-compaction-datafusion-error
m09526 Nov 4, 2025
a1614b8
Fix merge issues
m09526 Nov 4, 2025
55db963
Remove debug line
m09526 Nov 4, 2025
31f0ee6
Re-enable system test
m09526 Nov 4, 2025
492ec83
Quiet build
m09526 Nov 4, 2025
8aeb492
Merge branch 'develop' into 5777-large-compaction-datafusion-error
m09526 Nov 5, 2025
379e820
Added more debug and source local reqwest module
m09526 Nov 20, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions NOTICES
Original file line number Diff line number Diff line change
Expand Up @@ -616,7 +616,9 @@ test-log (https://github.com/d-e-s-o/test-log)

- Apache 2 License

tracing-subscriber (https://github.com/tokio-rs/tracing)

- MIT License

The build pipeline uses the following GitHub Actions from the marketplace.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import static sleeper.core.properties.instance.IngestProperty.INGEST_TRACKER_ENABLED;
import static sleeper.core.properties.instance.IngestProperty.MAXIMUM_CONCURRENT_INGEST_TASKS;
import static sleeper.core.properties.instance.LoggingLevelsProperty.LOGGING_LEVEL;
import static sleeper.core.properties.instance.LoggingLevelsProperty.RUST_BACKTRACE;
import static sleeper.core.properties.instance.MetricsProperty.METRICS_TABLE_BATCH_SIZE;
import static sleeper.core.properties.instance.NonPersistentEMRProperty.DEFAULT_BULK_IMPORT_EMR_EXECUTOR_X86_INSTANCE_TYPES;
import static sleeper.core.properties.instance.NonPersistentEMRProperty.DEFAULT_BULK_IMPORT_EMR_INSTANCE_ARCHITECTURE;
Expand Down Expand Up @@ -205,6 +206,7 @@ private static DeployInstanceConfiguration createCompactionPerformanceOnDataFusi
properties.set(COMPACTION_TASK_ARM_MEMORY, "8192");
properties.set(MAXIMUM_CONCURRENT_COMPACTION_TASKS, "10");
properties.set(DEFAULT_COMPACTION_FILES_BATCH_SIZE, "11");
properties.set(RUST_BACKTRACE, "1");
setSystemTestTags(properties, "compactionOnDataFusion", "Sleeper Maven system test compaction performance on DataFusion");
return createInstanceConfiguration(properties);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,7 @@ void shouldRunVeryLargeCompaction(SleeperSystemTest sleeper) {
TABLE_ONLINE, "false",
DATA_ENGINE, DataEngine.DATAFUSION.toString(),
COMPACTION_FILES_BATCH_SIZE, "40",
// We've disabled readahead temporarily until the following bug is resolved:
// https://github.com/gchq/sleeper/issues/5777
DATAFUSION_S3_READAHEAD_ENABLED, "false"));
DATAFUSION_S3_READAHEAD_ENABLED, "true"));
// And 40 input files
sleeper.systemTestCluster().runDataGenerationJobs(40,
builder -> builder.ingestMode(DIRECT).rowsPerIngest(50_000_000),
Expand Down
Loading