diff --git a/crates/paimon/src/arrow/evolution_string.rs b/crates/paimon/src/arrow/evolution_string.rs new file mode 100644 index 000000000..ee5207d9a --- /dev/null +++ b/crates/paimon/src/arrow/evolution_string.rs @@ -0,0 +1,429 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Temporal character casts: Java formatting, local zone and CHAR/VARCHAR length. + +use std::sync::Arc; + +use arrow_array::{cast::AsArray, types::Int64Type, ArrayRef, StringArray, Time32MillisecondArray}; +use arrow_schema::{DataType as ArrowType, TimeUnit}; +use jiff::tz::TimeZone; + +use crate::common::date_time::{format_epoch_timestamp, format_time}; +use crate::spec::DataType; + +/// Return None when this is not a temporal-to-character conversion. +pub(super) fn temporal_to_string( + source: &ArrayRef, + source_type: &DataType, + target_type: &DataType, +) -> crate::Result> { + let (length, pad) = match target_type { + DataType::Char(t) => (t.length(), true), + DataType::VarChar(t) => (t.length() as usize, false), + _ => return Ok(None), + }; + let values = match source_type { + DataType::Time(t) => { + let millis = arrow_cast::cast(source, &ArrowType::Time32(TimeUnit::Millisecond)) + .map_err(|error| invalid(error.to_string()))?; + let millis = millis + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid("Expected TIME array"))?; + millis + .iter() + .map(|value| value.map(|value| format_time(value, t.precision()))) + .collect::>() + } + DataType::Timestamp(_) | DataType::LocalZonedTimestamp(_) => { + let (precision, local) = match source_type { + DataType::Timestamp(t) => (t.precision(), false), + DataType::LocalZonedTimestamp(t) => (t.precision(), true), + _ => unreachable!(), + }; + let ArrowType::Timestamp(unit, _) = source.data_type() else { + return Err(invalid("Expected TIMESTAMP array")); + }; + let zone = local.then(TimeZone::system); + let raw = arrow_cast::cast(source, &ArrowType::Int64) + .map_err(|error| invalid(error.to_string()))?; + raw.as_primitive::() + .iter() + .map(|value| { + value.map(|value| render_timestamp(value, unit, precision, zone.as_ref())) + }) + .collect::>() + } + _ => return Ok(None), + }; + // The temporal representation is ASCII. Java toCharacterString counts + // characters, truncates both string kinds, and pads only CHAR. + let strings = values.into_iter().map(|value| { + value.map(|mut value| { + value.truncate(length); + if pad && value.len() < length { + value.extend(std::iter::repeat_n(' ', length - value.len())); + } + value + }) + }); + Ok(Some(Arc::new(StringArray::from_iter(strings)))) +} + +fn render_timestamp( + value: i64, + unit: &TimeUnit, + precision: u32, + zone: Option<&TimeZone>, +) -> String { + let millis = match unit { + TimeUnit::Second => value.wrapping_mul(1000), + TimeUnit::Millisecond => value, + TimeUnit::Microsecond => value.div_euclid(1000), + TimeUnit::Nanosecond => value.div_euclid(1_000_000), + }; + let sub_milli = match unit { + TimeUnit::Microsecond => value.rem_euclid(1000) * 1000, + TimeUnit::Nanosecond => value.rem_euclid(1_000_000), + _ => 0, + }; + let millis = match zone { + Some(zone) => { + let offset = local_offset(millis.div_euclid(1000), zone); + // Timestamp.fromLocalDateTime uses signed long millisecond storage. + // Applying an offset can wrap and change the millisecond fraction. + millis.wrapping_add(i64::from(offset) * 1000) + } + None => millis, + }; + let nanos = (millis.rem_euclid(1000) * 1_000_000 + sub_milli) as u32; + format_epoch_timestamp(millis.div_euclid(1000), nanos, precision) +} + +fn local_offset(seconds: i64, zone: &TimeZone) -> i32 { + let timestamp = jiff::Timestamp::from_second(seconds).unwrap_or_else(|_| { + if seconds < 0 { + // Java ZoneRules uses its first wall offset before recorded history. + jiff::Timestamp::MIN + } else { + // Java's final transition rules repeat with the Gregorian calendar: + // 400 years contain 146097 days, a whole number of weeks. Look up + // the same month/day/weekday/time in 2370..2770, after historical + // transitions, while retaining the original millis for formatting. + const CYCLE_SECONDS: i64 = 146_097 * 86_400; + jiff::Timestamp::from_second(seconds.rem_euclid(CYCLE_SECONDS) + CYCLE_SECONDS) + .expect("a projected Gregorian cycle is within jiff's timestamp range") + } + }); + zone.to_offset(timestamp).seconds() +} + +fn invalid(message: impl Into) -> crate::Error { + crate::Error::DataInvalid { + message: message.into(), + source: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::arrow::nested_evolution::evolve_column; + use crate::spec::{CharType, TimeType, TimestampType, VarCharType}; + use arrow_array::{Array, Int64Array, TimestampNanosecondArray}; + + fn strings(source: ArrayRef, source_type: DataType, target: DataType) -> StringArray { + evolve_column(&source, &source_type, &target) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .clone() + } + + #[test] + fn time_string_matches_java_fraction_loop_and_negative_normalization() { + let source: ArrayRef = Arc::new(Time32MillisecondArray::from(vec![ + None, + Some(0), + Some(100), + Some(101), + Some(-1), + None, + Some(0), + ])); + for (precision, expected) in [ + (0, ["00:00:00", "00:00:00", "00:00:00", "23:59:59"]), + (1, ["00:00:00.0", "00:00:00.1", "00:00:00.1", "23:59:59.9"]), + ( + 2, + ["00:00:00.0", "00:00:00.1", "00:00:00.10", "23:59:59.99"], + ), + ( + 3, + ["00:00:00.0", "00:00:00.1", "00:00:00.101", "23:59:59.999"], + ), + ( + 9, + ["00:00:00.0", "00:00:00.1", "00:00:00.101", "23:59:59.999"], + ), + ] { + let actual = strings( + source.slice(1, 5), + DataType::Time(TimeType::new(precision).unwrap()), + DataType::VarChar(VarCharType::string_type()), + ); + assert_eq!( + actual, + StringArray::from_iter(expected.into_iter().map(Some).chain([None])) + ); + } + } + + #[test] + fn timestamp_string_keeps_nanos_and_minimum_fraction_width() { + let source: ArrayRef = Arc::new(TimestampNanosecondArray::from(vec![ + None, + Some(0), + Some(123_456_789), + Some(-1), + None, + Some(0), + ])); + for (precision, whole) in [ + (0, "1970-01-01 00:00:00"), + (1, "1970-01-01 00:00:00.0"), + (3, "1970-01-01 00:00:00.000"), + (6, "1970-01-01 00:00:00.000000"), + (9, "1970-01-01 00:00:00.000000000"), + ] { + let actual = strings( + source.slice(1, 4), + DataType::Timestamp(TimestampType::new(precision).unwrap()), + DataType::VarChar(VarCharType::string_type()), + ); + assert_eq!( + actual, + StringArray::from(vec![ + Some(whole), + Some("1970-01-01 00:00:00.123456789"), + Some("1969-12-31 23:59:59.999999999"), + None + ]) + ); + } + } + + #[test] + fn timestamp_string_handles_all_physical_units_and_character_lengths() { + for (unit, ticks) in [ + (TimeUnit::Second, 1), + (TimeUnit::Millisecond, 1_000), + (TimeUnit::Microsecond, 1_000_000), + (TimeUnit::Nanosecond, 1_000_000_000), + ] { + let raw: ArrayRef = Arc::new(Int64Array::from(vec![Some(ticks), Some(-ticks), None])); + let source = arrow_cast::cast(&raw, &ArrowType::Timestamp(unit, None)).unwrap(); + for (target, first, second) in [ + ( + DataType::VarChar(VarCharType::string_type()), + "1970-01-01 00:00:01.000", + "1969-12-31 23:59:59.000", + ), + ( + DataType::VarChar(VarCharType::new(12).unwrap()), + "1970-01-01 0", + "1969-12-31 2", + ), + ( + DataType::Char(CharType::new(25).unwrap()), + "1970-01-01 00:00:01.000 ", + "1969-12-31 23:59:59.000 ", + ), + (DataType::Char(CharType::new(4).unwrap()), "1970", "1969"), + ] { + let result = strings( + source.clone(), + DataType::Timestamp(TimestampType::new(3).unwrap()), + target, + ); + assert_eq!( + result, + StringArray::from(vec![Some(first), Some(second), None]) + ); + } + } + } + + #[test] + fn timestamp_string_uses_java_four_digits_outside_datetime_year_range() { + use arrow_array::TimestampMillisecondArray; + let source: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![ + Some(253_402_300_800_000), + Some(-62_167_219_200_000), + Some(-62_198_755_200_000), + Some(i64::MAX), + Some(i64::MIN), + None, + ])); + let actual = strings( + source, + DataType::Timestamp(TimestampType::new(3).unwrap()), + DataType::VarChar(VarCharType::string_type()), + ); + assert_eq!( + actual, + StringArray::from(vec![ + Some("0000-01-01 00:00:00.000"), + Some("0000-01-01 00:00:00.000"), + Some("000/-01-01 00:00:00.000"), + Some("/+-.-04-06 07:12:55.807"), + Some("5471-09-26 16:47:04.192"), + None + ]) + ); + } + + #[test] + fn timestamp_seconds_wrapping_keeps_wrapped_millisecond_fraction() { + use arrow_array::TimestampSecondArray; + let source: ArrayRef = Arc::new(TimestampSecondArray::from(vec![ + Some(9_223_372_036_854_776), + None, + ])); + let actual = strings( + source, + DataType::Timestamp(TimestampType::new(3).unwrap()), + DataType::VarChar(VarCharType::string_type()), + ); + assert_eq!( + actual, + StringArray::from(vec![Some("5471-09-26 16:47:04.384"), None]) + ); + } + + #[test] + fn local_timestamp_string_extreme_millis_does_not_fail_the_batch() { + use crate::spec::LocalZonedTimestampType; + use arrow_array::TimestampMillisecondArray; + let source: ArrayRef = Arc::new( + TimestampMillisecondArray::from(vec![Some(0), Some(i64::MAX), None, Some(i64::MIN)]) + .with_timezone("UTC"), + ); + let result = evolve_column( + &source, + &DataType::LocalZonedTimestamp(LocalZonedTimestampType::new(3).unwrap()), + &DataType::VarChar(VarCharType::string_type()), + ); + let result = result.expect("LTZ conversion must not fail on a non-NULL extreme value"); + assert_eq!(result.len(), 4); + assert_eq!(result.null_count(), 1); + assert!(result.is_null(2)); + assert!(result.is_valid(0) && result.is_valid(1) && result.is_valid(3)); + } + + #[test] + fn local_timestamp_strings_match_java_extremes_and_zone_rules() { + // Golden strings from Java CastExecutors in these explicitly selected + // zones, including far-future DST and historical offsets with seconds. + let values = [ + i64::MAX, + i64::MIN, + i64::MAX - 181 * 86_400_000, + 1_704_067_200_000, + 1_719_792_000_000, + ]; + for (name, expected) in [ + ( + "UTC", + [ + "/+-.-04-06 07:12:55.807", + "5471-09-26 16:47:04.192", + "/+---10-08 07:12:55.807", + "2024-01-01 00:00:00.000", + "2024-07-01 00:00:00.000", + ], + ), + ( + "Asia/Shanghai", + [ + "5471-09-27 00:47:04.191", + "5471-09-27 00:52:47.192", + "/+---10-08 15:12:55.807", + "2024-01-01 08:00:00.000", + "2024-07-01 08:00:00.000", + ], + ), + ( + "America/New_York", + [ + "/+-.-04-06 03:12:55.807", + "/+-.-04-06 02:16:53.808", + "/+---10-08 02:12:55.807", + "2023-12-31 19:00:00.000", + "2024-06-30 20:00:00.000", + ], + ), + ( + "Australia/Sydney", + [ + "5471-09-27 02:47:04.191", + "5471-09-27 02:51:56.192", + "/+---10-08 18:12:55.807", + "2024-01-01 11:00:00.000", + "2024-07-01 10:00:00.000", + ], + ), + ] { + let zone = TimeZone::get(name).unwrap(); + for (value, expected) in values.into_iter().zip(expected) { + assert_eq!( + render_timestamp(value, &TimeUnit::Millisecond, 3, Some(&zone)), + expected, + "{value} in {name}" + ); + } + } + } + + #[test] + fn local_timestamp_preserves_sub_millisecond_digits_in_all_ranges() { + let zone = TimeZone::get("America/New_York").unwrap(); + for (value, unit, precision, expected) in [ + ( + i64::MAX, + TimeUnit::Microsecond, + 6, + "4247-01-09 23:00:54.775807", + ), + (-1, TimeUnit::Microsecond, 6, "1969-12-31 18:59:59.999999"), + ( + i64::MAX, + TimeUnit::Nanosecond, + 9, + "2262-04-11 19:47:16.854775807", + ), + (-1, TimeUnit::Nanosecond, 9, "1969-12-31 18:59:59.999999999"), + ] { + assert_eq!( + render_timestamp(value, &unit, precision, Some(&zone)), + expected + ); + } + } +} diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index 2ef79cedb..06f9d4f76 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -16,6 +16,7 @@ // under the License. mod evolution_cast; +mod evolution_string; pub(crate) mod filtering; pub(crate) mod format; pub(crate) mod nested_evolution; diff --git a/crates/paimon/src/arrow/nested_evolution.rs b/crates/paimon/src/arrow/nested_evolution.rs index 883b89c35..2a79f6e3c 100644 --- a/crates/paimon/src/arrow/nested_evolution.rs +++ b/crates/paimon/src/arrow/nested_evolution.rs @@ -204,6 +204,12 @@ pub(crate) fn evolve_column( _ => {} } + if let Some(result) = + super::evolution_string::temporal_to_string(source, source_type, target_type)? + { + return Ok(result); + } + // Apply the epoch-seconds rule at every leaf, including nested containers. // Numeric -> LTZ depends on Java's system time zone and retains its existing // path; ordinary TIMESTAMP has no such runtime dependency. @@ -344,14 +350,17 @@ fn render_string_values( .collect()) } _ => { - let casted = cast(source, &ArrowDataType::Utf8).map_err(|error| { - crate::Error::UnexpectedError { - message: format!( - "failed to render {:?} as string during schema evolution", - source.data_type() - ), - source: Some(Box::new(error)), - } + let casted = evolve_column( + source, + source_type, + &DataType::VarChar(crate::spec::VarCharType::string_type()), + ) + .map_err(|error| crate::Error::UnexpectedError { + message: format!( + "failed to render {:?} as string during schema evolution", + source.data_type() + ), + source: Some(Box::new(error)), })?; let strings = casted .as_any() @@ -623,6 +632,29 @@ mod tests { use arrow_buffer::NullBuffer; use arrow_schema::{DataType as ArrowDataType, Fields}; + #[test] + fn timestamp_string_evolution_uses_java_separator_and_precision() { + let source: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![ + Some(0), + Some(-1), + None, + ])); + let result = evolve_column( + &source, + &DataType::Timestamp(TimestampType::new(3).unwrap()), + &DataType::VarChar(VarCharType::string_type()), + ) + .unwrap(); + assert_eq!( + result.as_any().downcast_ref::().unwrap(), + &StringArray::from(vec![ + Some("1970-01-01 00:00:00.000"), + Some("1969-12-31 23:59:59.999"), + None + ]) + ); + } + #[test] fn selected_map_read_type_uses_key_metadata_not_child_names() { use arrow_array::builder::{Int32Builder, MapBuilder, StringBuilder}; diff --git a/crates/paimon/src/common/date_time.rs b/crates/paimon/src/common/date_time.rs new file mode 100644 index 000000000..a30f5cc58 --- /dev/null +++ b/crates/paimon/src/common/date_time.rs @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Java temporal string formatting shared by casts and partition paths. + +use chrono::{Datelike, NaiveDateTime, Timelike}; + +/// Format millis-since-midnight to `HH:mm:ss[.f...]` using Java `DateTimeUtils` semantics. +/// +/// Matches Java `DateTimeUtils.formatTimestampMillis(time, precision)`: +/// outputs up to `precision` fractional digits, stopping after the first digit whose remaining fraction is zero. +pub(crate) fn format_time(millis_of_day: i32, precision: u32) -> String { + let mut ms = millis_of_day; + // Handle negative millis (same guard as Java) + while ms < 0 { + ms += 86_400_000; + } + let ms = ms as u32; + let h = ms / 3_600_000 % 100; + let m = (ms % 3_600_000) / 60_000; + let s = (ms % 60_000) / 1_000; + let mut frac_ms = ms % 1_000; + + let hms = format!("{h:02}:{m:02}:{s:02}"); + if precision == 0 { + return hms; + } + + // Emit up to `precision` digits, each extracted from the ms value. + // Matches Java's digit-by-digit loop with trailing-zero break. + let mut frac = String::with_capacity(precision as usize); + let mut remaining = precision; + while remaining > 0 { + frac.push((b'0' + (frac_ms / 100) as u8) as char); + frac_ms = (frac_ms % 100) * 10; + if frac_ms == 0 { + break; + } + remaining -= 1; + } + + format!("{hms}.{frac}") +} + +/// Format a timestamp using non-legacy `DateTimeUtils.formatTimestamp()` semantics. +/// +/// Always uses space separator: `yyyy-MM-dd HH:mm:ss[.fraction]`. +/// Fraction: pad nano to 9 digits, strip trailing zeros while keeping at least `precision` digits. +pub(crate) fn format_timestamp(dt: NaiveDateTime, precision: u32) -> String { + format_timestamp_parts( + (dt.year(), dt.month(), dt.day()), + (dt.hour(), dt.minute(), dt.second()), + dt.nanosecond(), + precision, + ) +} + +/// Java Timestamp.toLocalDateTime stores its epoch day in a signed int. +/// Use Gregorian arithmetic so that this also covers dates beyond chrono's range. +pub(crate) fn format_epoch_timestamp(seconds: i64, nanos: u32, precision: u32) -> String { + let (year, month, day) = epoch_day_to_date(seconds.div_euclid(86_400) as i32); + let time = seconds.rem_euclid(86_400) as u32; + format_timestamp_parts( + (year, month, day), + (time / 3600, time / 60 % 60, time % 60), + nanos, + precision, + ) +} + +fn epoch_day_to_date(day: i32) -> (i32, u32, u32) { + // Gregorian eras start on March 1, so the leap day is at the end of a year. + let days = i64::from(day) + 719_468; + let era = days.div_euclid(146_097); + let day_of_era = days - era * 146_097; + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let mut year = year_of_era + era * 400; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let march_month = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * march_month + 2) / 5 + 1; + let month = march_month + if march_month < 10 { 3 } else { -9 }; + year += i64::from(month <= 2); + (year as i32, month as u32, day as u32) +} + +fn format_timestamp_parts( + date: (i32, u32, u32), + time: (u32, u32, u32), + nano: u32, + precision: u32, +) -> String { + let (year, month, day) = date; + let (hour, minute, second) = time; + // DateTimeUtils.int4 writes exactly four digits using signed division and + // remainder, including years outside 0000..9999. + let year = [1000, 100, 10, 1] + .into_iter() + .map(|divisor| (i32::from(b'0') + year / divisor % 10) as u8 as char) + .collect::(); + let ymdhms = format!( + "{year}-{:02}-{:02} {:02}:{:02}:{:02}", + month, day, hour, minute, second + ); + + // Whole seconds keep the declared precision; nonzero fractions are never truncated. + let nano_str = format!("{nano:09}"); + let mut fraction = &nano_str[..]; + + // Strip trailing zeros, but don't go below the precision boundary. + while fraction.len() > precision as usize && fraction.ends_with('0') { + fraction = &fraction[..fraction.len() - 1]; + } + + if fraction.is_empty() { + ymdhms + } else { + format!("{ymdhms}.{fraction}") + } +} diff --git a/crates/paimon/src/common/mod.rs b/crates/paimon/src/common/mod.rs index 792cbfe5a..5376e76d4 100644 --- a/crates/paimon/src/common/mod.rs +++ b/crates/paimon/src/common/mod.rs @@ -17,6 +17,7 @@ //! Common utilities for Paimon. +pub(crate) mod date_time; pub mod options; pub use options::{CatalogOptions, Options}; diff --git a/crates/paimon/src/spec/partition_utils.rs b/crates/paimon/src/spec/partition_utils.rs index b0466eaa7..3785e6192 100644 --- a/crates/paimon/src/spec/partition_utils.rs +++ b/crates/paimon/src/spec/partition_utils.rs @@ -23,6 +23,7 @@ //! - Java `InternalRowPartitionComputer` for value formatting //! - Java `PartitionPathUtils` for path escaping and assembly +use crate::common::date_time::{format_time, format_timestamp}; use crate::error::*; use crate::spec::types::DataType; use crate::spec::BinaryRow; @@ -436,7 +437,7 @@ fn format_partition_value( if legacy { format_timestamp_legacy(dt) } else { - format_timestamp_non_legacy(dt, t.precision()) + format_timestamp(dt, t.precision()) } } @@ -452,7 +453,7 @@ fn format_partition_value( if legacy { format_timestamp_legacy(dt) } else { - format_timestamp_non_legacy(dt, t.precision()) + format_timestamp(dt, t.precision()) } } @@ -642,43 +643,6 @@ fn format_date(epoch_days: i32) -> String { date.format("%Y-%m-%d").to_string() } -/// Format millis-since-midnight to `HH:mm:ss[.f...]` for non-legacy TIME partitions. -/// -/// Matches Java `DateTimeUtils.formatTimestampMillis(time, precision)`: -/// outputs up to `precision` fractional digits, stripping trailing zeros. -fn format_time(millis_of_day: i32, precision: u32) -> String { - let mut ms = millis_of_day; - // Handle negative millis (same guard as Java) - while ms < 0 { - ms += 86_400_000; - } - let ms = ms as u32; - let h = ms / 3_600_000; - let m = (ms % 3_600_000) / 60_000; - let s = (ms % 60_000) / 1_000; - let mut frac_ms = ms % 1_000; - - let hms = format!("{h:02}:{m:02}:{s:02}"); - if precision == 0 || frac_ms == 0 { - return hms; - } - - // Emit up to `precision` digits, each extracted from the ms value. - // Matches Java's digit-by-digit loop with trailing-zero break. - let mut frac = String::with_capacity(precision as usize); - let mut remaining = precision; - while remaining > 0 { - frac.push((b'0' + (frac_ms / 100) as u8) as char); - frac_ms = (frac_ms % 100) * 10; - if frac_ms == 0 { - break; - } - remaining -= 1; - } - - format!("{hms}.{frac}") -} - /// Format an unscaled i128 value with the given scale to a plain decimal string. /// /// Matches Java `BigDecimal.toPlainString()` semantics: no scientific notation, @@ -792,30 +756,6 @@ fn format_timestamp_legacy(dt: NaiveDateTime) -> String { result } -/// Format a timestamp using non-legacy `DateTimeUtils.formatTimestamp()` semantics. -/// -/// Always uses space separator: `yyyy-MM-dd HH:mm:ss[.fraction]`. -/// Fraction: pad nano to 9 digits, strip trailing zeros while keeping at least `precision` digits. -fn format_timestamp_non_legacy(dt: NaiveDateTime, precision: u32) -> String { - let nano = dt.nanosecond(); - let ymdhms = dt.format("%Y-%m-%d %H:%M:%S").to_string(); - - // Whole seconds keep the declared precision; nonzero fractions are never truncated. - let nano_str = format!("{nano:09}"); - let mut fraction = &nano_str[..]; - - // Strip trailing zeros, but don't go below the precision boundary. - while fraction.len() > precision as usize && fraction.ends_with('0') { - fraction = &fraction[..fraction.len() - 1]; - } - - if fraction.is_empty() { - ymdhms - } else { - format!("{ymdhms}.{fraction}") - } -} - /// Escape a path component following Java `PartitionPathUtils.escapePathName`. /// /// Characters that need escaping are encoded as `%XX` (uppercase hex). @@ -1418,20 +1358,11 @@ mod tests { .and_hms_nano_opt(12, 0, 0, 123_000_000) .unwrap(); // Precision is a minimum fraction width, not a truncation boundary. - assert_eq!( - format_timestamp_non_legacy(dt, 0), - "2024-01-01 12:00:00.123" - ); + assert_eq!(format_timestamp(dt, 0), "2024-01-01 12:00:00.123"); // precision=3 → strip trailing zeros down to 3 digits - assert_eq!( - format_timestamp_non_legacy(dt, 3), - "2024-01-01 12:00:00.123" - ); + assert_eq!(format_timestamp(dt, 3), "2024-01-01 12:00:00.123"); // precision=9 → preserve all 9 digits (no stripping since len == precision) - assert_eq!( - format_timestamp_non_legacy(dt, 9), - "2024-01-01 12:00:00.123000000" - ); + assert_eq!(format_timestamp(dt, 9), "2024-01-01 12:00:00.123000000"); } #[test] @@ -1447,7 +1378,7 @@ mod tests { (6, "2024-01-01 12:00:00.000000"), (9, "2024-01-01 12:00:00.000000000"), ] { - assert_eq!(format_timestamp_non_legacy(dt, precision), expected); + assert_eq!(format_timestamp(dt, precision), expected); } } @@ -1464,7 +1395,7 @@ mod tests { .unwrap() .and_hms_nano_opt(12, 0, 0, nanos) .unwrap(); - assert_eq!(format_timestamp_non_legacy(dt, precision), expected); + assert_eq!(format_timestamp(dt, precision), expected); } } @@ -1734,7 +1665,7 @@ mod tests { #[test] fn test_format_time() { // precision=3, 12:34:56 = 45_296_000 ms - assert_eq!(format_time(45_296_000, 3), "12:34:56"); + assert_eq!(format_time(45_296_000, 3), "12:34:56.0"); // precision=3, 12:34:56.123 = 45_296_123 ms assert_eq!(format_time(45_296_123, 3), "12:34:56.123"); // precision=3, 00:00:00.100 = 100 ms → trailing zero stripped to .1 @@ -1769,6 +1700,13 @@ mod tests { "t=12%3A34%3A56.123/", false, ); + assert_single_partition( + "t", + DataType::Time(TimeType::new(3).unwrap()), + |b| b.write_int(0, 0), + "t=00%3A00%3A00.0/", + false, + ); } // ======================== Corrupted row tests ========================