Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 96 additions & 4 deletions datafusion/functions/src/datetime/date_trunc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -547,10 +547,21 @@ impl ScalarUDFImpl for DateTruncFunc {
let precision = &input[0];
let date_value = &input[1];

let order_safe_input = matches!(
date_value.range.data_type(),
Timestamp(_, None) | Time32(_) | Time64(_)
);
let order_safe_input = match date_value.range.data_type() {
Timestamp(_, None) | Time32(_) | Time64(_) => true,
// A timezone transition can reverse the order of timestamps
// truncated in local time, but not of those truncated on the
// epoch value.
Timestamp(_, Some(tz)) => {
match (constant_granularity(precision), parse_tz(Some(&tz))) {
(Some(granularity), Ok(tz)) => {
truncates_in_input_unit(granularity, tz.as_ref())
}
_ => false,
}
}
_ => false,
};

if precision.sort_properties == SortProperties::Singleton && order_safe_input {
Ok(date_value.sort_properties)
Expand Down Expand Up @@ -710,6 +721,16 @@ where
1 + 3 * ((date.month() - 1) / 3)
}

/// The granularity `precision` always evaluates to, if that is known.
fn constant_granularity(precision: &ExprProperties) -> Option<DatePart> {
let range = &precision.range;
if range.lower() != range.upper() {
return None;
}
let granularity = range.lower().try_as_str().flatten()?;
parse_granularity(&granularity.to_lowercase()).ok()
}

/// Returns true if `tz` explicitly names UTC (`"UTC"` or `"+00:00"`)
fn is_utc(tz: &str) -> bool {
tz.eq_ignore_ascii_case("utc") || tz == "+00:00"
Expand Down Expand Up @@ -1233,6 +1254,9 @@ fn general_date_trunc_array_fine_granularity<T: ArrowTimestampType>(
/// Both datafusion and arrow are ignoring historical timezone's non-minute granularity
/// bias (e.g., Asia/Kathmandu before 1919 is UTC+05:41:16).
/// In UTC, "hour" and "day" have uniform durations and can be truncated with simple arithmetic
///
/// Rounding down never reorders values, whatever the timezone, which
/// [`DateTruncFunc::output_ordering`] relies on.
fn truncates_in_input_unit(granularity: DatePart, tz: Option<&Tz>) -> bool {
granularity.is_fine_granularity()
|| (tz.is_none() && granularity.is_fine_granularity_utc())
Expand Down Expand Up @@ -1400,6 +1424,74 @@ mod tests {
);
}

#[test]
fn output_ordering_with_timezone_depends_on_granularity() {
let constant = |value: ScalarValue| {
ExprProperties::new_unknown()
.with_order(SortProperties::Singleton)
.with_range(Interval::try_new(value.clone(), value).unwrap())
};
let ordered = SortProperties::Ordered(SortOptions::default());
let function = DateTruncFunc::new();
let output_ordering = |precision: ExprProperties, unit| {
let data_type = DataType::Timestamp(unit, Some("America/Goose_Bay".into()));
let timestamp = ExprProperties::new_unknown()
.with_order(ordered)
.with_range(Interval::make_unbounded(&data_type).unwrap());
function.output_ordering(&[precision, timestamp]).unwrap()
};

for unit in [
TimeUnit::Second,
TimeUnit::Millisecond,
TimeUnit::Microsecond,
TimeUnit::Nanosecond,
] {
for granularity in ["microsecond", "millisecond", "second", "minute"] {
assert_eq!(
output_ordering(constant(ScalarValue::from(granularity)), unit),
ordered,
"{granularity} {unit:?}"
);
}
for granularity in ["hour", "day", "week", "month", "quarter", "year"] {
assert_eq!(
output_ordering(constant(ScalarValue::from(granularity)), unit),
SortProperties::Unordered,
"{granularity} {unit:?}"
);
}
}

for precision in [
ScalarValue::Utf8(Some("MINUTE".into())),
ScalarValue::Utf8View(Some("minute".into())),
ScalarValue::LargeUtf8(Some("minute".into())),
] {
assert_eq!(
output_ordering(constant(precision), TimeUnit::Nanosecond),
ordered
);
}

// A constant whose value is not known, such as a column that holds a
// single value, may name any granularity.
let unknown_constant = ExprProperties::new_unknown()
.with_order(SortProperties::Singleton)
.with_range(Interval::make_unbounded(&DataType::Utf8).unwrap());
assert_eq!(
output_ordering(unknown_constant, TimeUnit::Nanosecond),
SortProperties::Unordered
);
assert_eq!(
output_ordering(
constant(ScalarValue::from("not a granularity")),
TimeUnit::Nanosecond
),
SortProperties::Unordered
);
}

#[test]
fn range_key_locality_analysis_requires_timezone_less_timestamp() {
let date_trunc = DateTruncFunc::new();
Expand Down
40 changes: 40 additions & 0 deletions datafusion/sqllogictest/test_files/datetime/timestamps.slt
Original file line number Diff line number Diff line change
Expand Up @@ -1973,6 +1973,46 @@ physical_plan
04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/timestamps/goose_bay_epochs.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true

# Minute and finer granularities are truncated on the epoch value, so the
# same transition does not reorder them and no sort is needed.
query II
SELECT
c,
arrow_cast(
date_trunc('minute', from_unixtime(c, 'America/Goose_Bay')),
'Int64'
) AS truncated
FROM goose_bay_epochs
ORDER BY date_trunc('minute', from_unixtime(c, 'America/Goose_Bay'));
----
562129259 562129200
562129260 562129260

query TT
EXPLAIN SELECT date_trunc('minute', from_unixtime(c, 'America/Goose_Bay')) AS bucket
FROM goose_bay_epochs
ORDER BY bucket;
----
physical_plan
01)SortPreservingMergeExec: [bucket@0 ASC NULLS LAST]
02)--ProjectionExec: expr=[date_trunc(minute, from_unixtime(c@0, America/Goose_Bay)) as bucket]
03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/timestamps/goose_bay_epochs.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true

# The aggregation can then emit each bucket as soon as the next one starts.
query TT
EXPLAIN SELECT date_trunc('minute', from_unixtime(c, 'America/Goose_Bay')) AS bucket, count(*)
FROM goose_bay_epochs
GROUP BY bucket;
----
physical_plan
01)ProjectionExec: expr=[date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0 as bucket, count(Int64(1))@1 as count(*)]
02)--AggregateExec: mode=FinalPartitioned, gby=[date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0 as date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))], aggr=[count(Int64(1))], group_clustering_mode=Full
03)----RepartitionExec: partitioning=Hash([date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0], 4), input_partitions=4, preserve_order=true, sort_exprs=date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))@0 ASC NULLS LAST
04)------AggregateExec: mode=Partial, gby=[date_trunc(minute, from_unixtime(c@0, America/Goose_Bay)) as date_trunc(Utf8("minute"),from_unixtime(goose_bay_epochs.c,Utf8("America/Goose_Bay")))], aggr=[count(Int64(1))], group_clustering_mode=Full
05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/timestamps/goose_bay_epochs.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true

statement ok
DROP TABLE goose_bay_epochs;

Expand Down
Loading