diff --git a/datafusion/functions/src/datetime/date_trunc.rs b/datafusion/functions/src/datetime/date_trunc.rs index d5cf186bc14de..db2b8eae3c160 100644 --- a/datafusion/functions/src/datetime/date_trunc.rs +++ b/datafusion/functions/src/datetime/date_trunc.rs @@ -547,10 +547,21 @@ impl ScalarUDFImpl for DateTruncFunc { let precision = &input[0]; let date_value = &input[1]; - let order_safe_input = matches!( - date_value.range.data_type(), - Timestamp(_, None) | Time32(_) | Time64(_) - ); + let order_safe_input = match date_value.range.data_type() { + Timestamp(_, None) | Time32(_) | Time64(_) => true, + // A timezone transition can reverse the order of timestamps + // truncated in local time, but not of those truncated on the + // epoch value. + Timestamp(_, Some(tz)) => { + match (constant_granularity(precision), parse_tz(Some(&tz))) { + (Some(granularity), Ok(tz)) => { + truncates_in_input_unit(granularity, tz.as_ref()) + } + _ => false, + } + } + _ => false, + }; if precision.sort_properties == SortProperties::Singleton && order_safe_input { Ok(date_value.sort_properties) @@ -710,6 +721,16 @@ where 1 + 3 * ((date.month() - 1) / 3) } +/// The granularity `precision` always evaluates to, if that is known. +fn constant_granularity(precision: &ExprProperties) -> Option { + let range = &precision.range; + if range.lower() != range.upper() { + return None; + } + let granularity = range.lower().try_as_str().flatten()?; + parse_granularity(&granularity.to_lowercase()).ok() +} + /// Returns true if `tz` explicitly names UTC (`"UTC"` or `"+00:00"`) fn is_utc(tz: &str) -> bool { tz.eq_ignore_ascii_case("utc") || tz == "+00:00" @@ -1233,6 +1254,9 @@ fn general_date_trunc_array_fine_granularity( /// Both datafusion and arrow are ignoring historical timezone's non-minute granularity /// bias (e.g., Asia/Kathmandu before 1919 is UTC+05:41:16). /// In UTC, "hour" and "day" have uniform durations and can be truncated with simple arithmetic +/// +/// Rounding down never reorders values, whatever the timezone, which +/// [`DateTruncFunc::output_ordering`] relies on. fn truncates_in_input_unit(granularity: DatePart, tz: Option<&Tz>) -> bool { granularity.is_fine_granularity() || (tz.is_none() && granularity.is_fine_granularity_utc()) @@ -1400,6 +1424,74 @@ mod tests { ); } + #[test] + fn output_ordering_with_timezone_depends_on_granularity() { + let constant = |value: ScalarValue| { + ExprProperties::new_unknown() + .with_order(SortProperties::Singleton) + .with_range(Interval::try_new(value.clone(), value).unwrap()) + }; + let ordered = SortProperties::Ordered(SortOptions::default()); + let function = DateTruncFunc::new(); + let output_ordering = |precision: ExprProperties, unit| { + let data_type = DataType::Timestamp(unit, Some("America/Goose_Bay".into())); + let timestamp = ExprProperties::new_unknown() + .with_order(ordered) + .with_range(Interval::make_unbounded(&data_type).unwrap()); + function.output_ordering(&[precision, timestamp]).unwrap() + }; + + for unit in [ + TimeUnit::Second, + TimeUnit::Millisecond, + TimeUnit::Microsecond, + TimeUnit::Nanosecond, + ] { + for granularity in ["microsecond", "millisecond", "second", "minute"] { + assert_eq!( + output_ordering(constant(ScalarValue::from(granularity)), unit), + ordered, + "{granularity} {unit:?}" + ); + } + for granularity in ["hour", "day", "week", "month", "quarter", "year"] { + assert_eq!( + output_ordering(constant(ScalarValue::from(granularity)), unit), + SortProperties::Unordered, + "{granularity} {unit:?}" + ); + } + } + + for precision in [ + ScalarValue::Utf8(Some("MINUTE".into())), + ScalarValue::Utf8View(Some("minute".into())), + ScalarValue::LargeUtf8(Some("minute".into())), + ] { + assert_eq!( + output_ordering(constant(precision), TimeUnit::Nanosecond), + ordered + ); + } + + // A constant whose value is not known, such as a column that holds a + // single value, may name any granularity. + let unknown_constant = ExprProperties::new_unknown() + .with_order(SortProperties::Singleton) + .with_range(Interval::make_unbounded(&DataType::Utf8).unwrap()); + assert_eq!( + output_ordering(unknown_constant, TimeUnit::Nanosecond), + SortProperties::Unordered + ); + assert_eq!( + output_ordering( + constant(ScalarValue::from("not a granularity")), + TimeUnit::Nanosecond + ), + SortProperties::Unordered + ); + } + #[test] fn range_key_locality_analysis_requires_timezone_less_timestamp() { let date_trunc = DateTruncFunc::new(); diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index ba2def1c055a6..ec1bdbbe7e9f9 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -1973,6 +1973,46 @@ physical_plan 04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true 05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/timestamps/goose_bay_epochs.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true +# Minute and finer granularities are truncated on the epoch value, so the +# same transition does not reorder them and no sort is needed. +query II +SELECT + c, + arrow_cast( + date_trunc('minute', from_unixtime(c, 'America/Goose_Bay')), + 'Int64' + ) AS truncated +FROM goose_bay_epochs +ORDER BY date_trunc('minute', from_unixtime(c, 'America/Goose_Bay')); +---- +562129259 562129200 +562129260 562129260 + +query TT +EXPLAIN SELECT date_trunc('minute', from_unixtime(c, 'America/Goose_Bay')) AS bucket +FROM goose_bay_epochs +ORDER BY bucket; +---- +physical_plan +01)SortPreservingMergeExec: [bucket@0 ASC NULLS LAST] +02)--ProjectionExec: expr=[date_trunc(minute, from_unixtime(c@0, America/Goose_Bay)) as bucket] +03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/timestamps/goose_bay_epochs.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true + +# The aggregation can then emit each bucket as soon as the next one starts. +query TT +EXPLAIN SELECT date_trunc('minute', from_unixtime(c, 'America/Goose_Bay')) AS bucket, count(*) +FROM goose_bay_epochs +GROUP BY bucket; +---- +physical_plan +01)ProjectionExec: expr=[date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0 as bucket, count(Int64(1))@1 as count(*)] +02)--AggregateExec: mode=FinalPartitioned, gby=[date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0 as date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))], aggr=[count(Int64(1))], group_clustering_mode=Full +03)----RepartitionExec: partitioning=Hash([date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0], 4), input_partitions=4, preserve_order=true, sort_exprs=date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0 ASC NULLS LAST +04)------AggregateExec: mode=Partial, gby=[date_trunc(minute, from_unixtime(c@0, America/Goose_Bay)) as date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))], aggr=[count(Int64(1))], group_clustering_mode=Full +05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true +06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/timestamps/goose_bay_epochs.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true + statement ok DROP TABLE goose_bay_epochs;