From 38b3ff2c71d3559b56aecdb40de2319d13ebf1bb Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Fri, 11 Sep 2026 14:39:54 +0800 Subject: [PATCH] Re-enable CORRECTED Parquet timestamp round trips Signed-off-by: Allen Xu --- docs/compatibility.md | 11 +++++++---- integration_tests/src/main/python/parquet_test.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index 610a6f0b489..4ff3015cbd5 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -312,10 +312,13 @@ satisfy the query, the ORC read falls back to the CPU as it is a metadata-only q ## Parquet The Parquet format has more configs because there are multiple versions with some compatibility -issues between them. Dates and timestamps are where the known issues exist. For reads when -`spark.sql.legacy.parquet.datetimeRebaseModeInWrite` is set to `CORRECTED` -[timestamps](https://github.com/NVIDIA/cudf-spark/issues/132) before the transition between the -Julian and Gregorian calendars are wrong, but dates are fine. When +issues between them. Dates and timestamps are where the known issues exist. The +[CORRECTED timestamp discrepancy](https://github.com/NVIDIA/cudf-spark/issues/132) was reported +on Spark 3.0. For files written by the CPU on supported Spark versions with both +`spark.sql.parquet.datetimeRebaseModeInWrite` and `spark.sql.parquet.int96RebaseModeInWrite` +set to `CORRECTED`, GPU reads support timestamps before the transition between the Julian +and Gregorian calendars. This does not change LEGACY rebasing or INT96 timestamp-conversion +limitations. When `spark.sql.legacy.parquet.datetimeRebaseModeInWrite` is set to `LEGACY`, the read may fail for values occurring before the transition between the Julian and Gregorian calendars, i.e.: date <= 1582-10-04. diff --git a/integration_tests/src/main/python/parquet_test.py b/integration_tests/src/main/python/parquet_test.py index c092847b4dd..c286b704b16 100644 --- a/integration_tests/src/main/python/parquet_test.py +++ b/integration_tests/src/main/python/parquet_test.py @@ -76,7 +76,7 @@ def read_parquet_sql(data_path): StructGen([['child0', ArrayGen(byte_gen)], ['child1', byte_gen], ['child2', float_gen], ['child3', decimal_gen_64bit]]), ArrayGen(StructGen([['child0', string_gen], ['child1', double_gen], ['child2', int_gen]]))] + parquet_map_gens + decimal_gens, - pytest.param([timestamp_gen], marks=pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/132'))] + [timestamp_gen]] # test with original parquet file reader, the multi-file parallel reader for cloud, and coalesce file reader for # non-cloud