From 0218828d7cc692ed597230c45116303bd5849996 Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Tue, 8 Oct 2024 06:49:42 +0100 Subject: [PATCH 1/2] Fix test case unix_timestamp(col, 'yyyyMMdd') failed for Africa/Casablanca timezone and LEGACY mode Signed-off-by: Chong Gao --- docs/compatibility.md | 2 +- integration_tests/src/main/python/date_time_test.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/compatibility.md b/docs/compatibility.md index 6be6607b04f..71a3927df13 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -661,7 +661,7 @@ LEGACY timeParserPolicy support has the following limitations when running on th - The proleptic Gregorian calendar is used instead of the hybrid Julian+Gregorian calendar that Spark uses in legacy mode - When format is `yyyyMMdd`, GPU only supports 8 digit strings. Spark supports like 7 digit - `2024101` string while GPU does not support. + `2024101` string while GPU does not support. Only tested `UTC` and `Asia/Shanghai` timezones. ## Formatting dates and timestamps as strings diff --git a/integration_tests/src/main/python/date_time_test.py b/integration_tests/src/main/python/date_time_test.py index 0129e216a23..da917da0831 100644 --- a/integration_tests/src/main/python/date_time_test.py +++ b/integration_tests/src/main/python/date_time_test.py @@ -461,6 +461,7 @@ def test_to_timestamp(parser_policy): @pytest.mark.skipif(not is_supported_time_zone(), reason="not all time zones are supported now, refer to https://github.com/NVIDIA/spark-rapids/issues/6839, please update after all time zones are supported") # Test years after 1900, refer to issues: https://github.com/NVIDIA/spark-rapids/issues/11543, https://github.com/NVIDIA/spark-rapids/issues/11539 +@pytest.mark.skipif(get_test_tz() != "Asia/Shanghai" or get_test_tz() != "UTC", reason="https://github.com/NVIDIA/spark-rapids/issues/11562") def test_yyyyMMdd_format_for_legacy_mode(): gen = StringGen('(19[0-9]{2}|[2-9][0-9]{3})([0-9]{4})') assert_gpu_and_cpu_are_equal_sql( From 1c2902e715684bcef8318d253f97ee416ce6896a Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Tue, 8 Oct 2024 06:39:29 +0000 Subject: [PATCH 2/2] Fix logic error --- integration_tests/src/main/python/date_time_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/main/python/date_time_test.py b/integration_tests/src/main/python/date_time_test.py index da917da0831..f83c238a70c 100644 --- a/integration_tests/src/main/python/date_time_test.py +++ b/integration_tests/src/main/python/date_time_test.py @@ -461,7 +461,7 @@ def test_to_timestamp(parser_policy): @pytest.mark.skipif(not is_supported_time_zone(), reason="not all time zones are supported now, refer to https://github.com/NVIDIA/spark-rapids/issues/6839, please update after all time zones are supported") # Test years after 1900, refer to issues: https://github.com/NVIDIA/spark-rapids/issues/11543, https://github.com/NVIDIA/spark-rapids/issues/11539 -@pytest.mark.skipif(get_test_tz() != "Asia/Shanghai" or get_test_tz() != "UTC", reason="https://github.com/NVIDIA/spark-rapids/issues/11562") +@pytest.mark.skipif(get_test_tz() != "Asia/Shanghai" and get_test_tz() != "UTC", reason="https://github.com/NVIDIA/spark-rapids/issues/11562") def test_yyyyMMdd_format_for_legacy_mode(): gen = StringGen('(19[0-9]{2}|[2-9][0-9]{3})([0-9]{4})') assert_gpu_and_cpu_are_equal_sql(