diff --git a/ibis/backends/polars/compiler.py b/ibis/backends/polars/compiler.py index f836b6bf9119..7c47e505cde8 100644 --- a/ibis/backends/polars/compiler.py +++ b/ibis/backends/polars/compiler.py @@ -146,32 +146,40 @@ def _cast(op, strict=True, **kw): elif to.is_date(): if not strict: raise NotImplementedError(f"Unsupported try_cast to type: {to!r}") - if dtype.is_string(): - return arg.str.strptime(pl.Date, "%Y-%m-%d") elif to.is_timestamp(): if not strict: raise NotImplementedError(f"Unsupported try_cast to type: {to!r}") - time_zone = to.timezone - time_unit = _TIMESTAMP_SCALE_TO_UNITS.get(to.scale, "us") - if dtype.is_numeric(): - typ = pl.Datetime(time_unit="us", time_zone=time_zone) + time_unit = _TIMESTAMP_SCALE_TO_UNITS.get(to.scale, "us") + typ = pl.Datetime(time_unit="us", time_zone=to.timezone) arg = (arg * 1_000_000).cast(typ) if time_unit != "us": arg = arg.dt.truncate(f"1{time_unit}") return arg.alias(op.name) - elif dtype.is_string(): - typ = pl.Datetime(time_unit=time_unit, time_zone=time_zone) - arg = arg.str.strptime(typ) - if time_unit == "s": - return arg.dt.truncate("1s") - return arg + + if to.is_temporal() and dtype.is_string(): + return _cast_string_to_temporal(arg, to, strict=strict) typ = PolarsType.from_ibis(to) return arg.cast(typ, strict=strict) +def _cast_string_to_temporal(arg, to, strict=True): + # polars doesn't support casting strings to temporal types + if to.is_date(): + return arg.str.strptime(pl.Date, "%Y-%m-%d", strict=strict) + elif to.is_timestamp(): + arg = arg.str.strptime(PolarsType.from_ibis(to), strict=strict) + # polars has no second-precision datetime type + if to.scale == 0: + return arg.dt.truncate("1s") + return arg + elif to.is_time(): + return arg.str.strptime(pl.Time, "%H:%M:%S%.f", strict=strict) + raise com.OperationNotDefinedError(f"Unsupported cast from string to {to!r}") + + @translate.register(ops.Field) def column(op, **_): return pl.col(op.name) @@ -1099,17 +1107,15 @@ def array_collect(op, in_group_by=False, **kw): @translate.register(ops.ArrayFlatten) def array_flatten(op, **kw): result = translate(op.arg, **kw) - return ( - pl.when(result.is_null()) - .then(None) - .when(result.list.len() == 0) - .then([]) - # polars doesn't have an efficient API (yet?) for removing one level of - # nesting from an array so we use elementwise evaluation - # - # https://github.com/ibis-project/ibis/issues/10135 - .otherwise(result.list.eval(pl.element().flatten())) - ) + # polars doesn't have an efficient API (yet?) for removing one level of + # nesting from an array so we use elementwise evaluation + # + # https://github.com/ibis-project/ibis/issues/10135 + # + # Null and empty inner arrays are filtered out before exploding, because + # `explode`'s handling of them differs between polars versions + element = pl.element() + return result.list.eval(element.filter(element.list.len() > 0).explode()) _date_methods = { @@ -1173,10 +1179,9 @@ def extract_epoch_seconds(op, **kw): @translate.register(ops.DayOfWeekName) def day_of_week_name(op, **kw): index = translate(op.arg, **kw).dt.weekday() - 1 - arg = None - for i, name in enumerate(calendar.day_name): - arg = pl.when(index == i).then(pl.lit(name)).otherwise(arg) - return arg + return index.replace_strict( + dict(enumerate(calendar.day_name)), return_dtype=pl.String + ) @translate.register(ops.Unary) @@ -1217,18 +1222,16 @@ def between(op, **kw): dtype = PolarsType.from_ibis(arg_dtype) - lower_bound = op.lower_bound - lower = translate(lower_bound, **kw) - - if lower_bound.dtype != arg_dtype: - lower = lower.cast(dtype) - - upper_bound = op.upper_bound - upper = translate(upper_bound, **kw) - - if upper_bound.dtype != arg_dtype: - upper = upper.cast(dtype) + def cast_bound(bound): + value = translate(bound, **kw) + if bound.dtype == arg_dtype: + return value + if bound.dtype.is_string() and arg_dtype.is_temporal(): + return _cast_string_to_temporal(value, arg_dtype) + return value.cast(dtype) + lower = cast_bound(op.lower_bound) + upper = cast_bound(op.upper_bound) return arg.is_between(lower, upper, closed="both") diff --git a/ibis/backends/polars/tests/test_client.py b/ibis/backends/polars/tests/test_client.py index 6fdc0abd4fcf..ad74cf83cf51 100644 --- a/ibis/backends/polars/tests/test_client.py +++ b/ibis/backends/polars/tests/test_client.py @@ -82,3 +82,19 @@ def test_compile_with_memtable(con): t = ibis.memtable({"a": [1, 2, 3], "b": [4, 5, 6]}) result = con.compile(t) assert isinstance(result, pl.LazyFrame) + + +def test_cast_string_to_timestamp_seconds(con): + t = ibis.memtable({"s": ["2023-01-07 13:20:05.561"]}) + result = con.execute(t.s.cast("timestamp(0)")) + assert result.iat[0] == pd.Timestamp("2023-01-07 13:20:05") + + +@pytest.mark.parametrize("to_type", ["timestamp", "timestamp(0)", "timestamp(9)"]) +def test_between_string_timestamp_bounds(con, to_type): + # bounds should be parsed at the same precision as the column, so a value + # is always between itself + value = "2023-01-07 13:20:05.561000231" + t = ibis.memtable({"s": [value]}) + result = con.execute(t.s.cast(to_type).between(value, value)) + assert result.iat[0] diff --git a/ibis/backends/tests/test_array.py b/ibis/backends/tests/test_array.py index 9fa758809b57..bd5720ec7a08 100644 --- a/ibis/backends/tests/test_array.py +++ b/ibis/backends/tests/test_array.py @@ -1259,11 +1259,6 @@ def flatten_data(): reason="Arrays are never nullable", raises=AssertionError, ), - pytest.mark.notyet( - ["polars"], - reason="flattened empty arrays incorrectly insert a null", - raises=AssertionError, - ), ], ), ], @@ -1349,10 +1344,10 @@ def test_range_start_stop_step_zero(con, start, stop): assert list(result) == [] -@pytest.mark.notimpl( - ["polars"], +@pytest.mark.xfail_version( + polars=["polars<2"], raises=AssertionError, - reason="ibis hasn't implemented this behavior yet", + reason="explode turns empty lists into nulls", ) @pytest.mark.notyet( ["flink"], diff --git a/ibis/backends/tests/test_generic.py b/ibis/backends/tests/test_generic.py index a6e6d2c43687..500223f4337e 100644 --- a/ibis/backends/tests/test_generic.py +++ b/ibis/backends/tests/test_generic.py @@ -1854,6 +1854,32 @@ def hash_256(col): param("float", "int", 0.0, 0, id="float_to_int"), param("string", "int", "0", 0, id="string_to_int"), param("string", "float", "0", 0.0, id="string_to_float"), + param( + "string", + "time", + "01:02:03", + datetime.time(1, 2, 3), + marks=[ + pytest.mark.notimpl( + ["sqlite"], + raises=AssertionError, + reason="SQLite returns Timedelta from execution", + ), + pytest.mark.notyet( + ["clickhouse", "impala"], + raises=AssertionError, + reason="backend doesn't have a time datatype", + ), + pytest.mark.notyet(["druid"], raises=PyDruidProgrammingError), + pytest.mark.notyet(["exasol"], raises=ExaQueryError), + pytest.mark.notimpl(["pyspark"], raises=AttributeError), + pytest.mark.notyet(["oracle"], raises=OracleDatabaseError), + pytest.mark.notyet( + ["athena"], raises=PyAthenaOperationalError, reason="no time type" + ), + ], + id="string_to_time", + ), param( "array", "array", @@ -1915,6 +1941,24 @@ def test_cast(con, from_type, to_type, from_val, expected): param(0.0, "int", 0), param("0", "int", 0), param("0.0", "float", 0.0), + param( + "01:02:03", + "time", + datetime.time(1, 2, 3), + marks=[ + pytest.mark.notyet( + ["clickhouse", "impala"], + raises=AssertionError, + reason="backend doesn't have a time datatype", + ), + pytest.mark.notyet(["druid"], raises=PyDruidProgrammingError), + pytest.mark.notyet(["exasol"], raises=ExaQueryError), + pytest.mark.notimpl(["pyspark"], raises=AttributeError), + pytest.mark.notyet( + ["athena"], raises=PyAthenaOperationalError, reason="no time type" + ), + ], + ), param( datetime.datetime(2023, 1, 1), "int", @@ -1955,7 +1999,6 @@ def test_try_cast(con, from_val, to_type, expected): "datafusion", "druid", "exasol", - "mysql", "singlestoredb", "oracle", "postgres", @@ -1971,11 +2014,19 @@ def test_try_cast(con, from_val, to_type, expected): @pytest.mark.parametrize( ("from_val", "to_type"), [ - param("a", "int"), + param("a", "int", marks=pytest.mark.notimpl(["mysql"])), + param( + "a", + "time", + marks=pytest.mark.notyet( + ["athena"], raises=PyAthenaOperationalError, reason="no time type" + ), + ), param( datetime.datetime(2023, 1, 1), "int", marks=[ + pytest.mark.notimpl(["mysql"]), pytest.mark.never( ["clickhouse", "pyspark", "flink", "databricks"], reason="casts to 1672531200", diff --git a/requirements-dev.txt b/requirements-dev.txt index 4885ff57e809..a9782d312283 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -178,8 +178,8 @@ plotly==6.5.2 plotnine==0.15.3 pluggy==1.6.0 plum-dispatch==2.6.1 -polars==1.37.1 -polars-runtime-32==1.37.1 +polars==2.0.0 +polars-runtime-32==2.0.0 pprintpp==0.4.0 pre-commit==4.5.1 prometheus-client==0.24.1 diff --git a/uv.lock b/uv.lock index 88f703f1c7bc..20e042592c0b 100644 --- a/uv.lock +++ b/uv.lock @@ -4822,30 +4822,30 @@ wheels = [ [[package]] name = "polars" -version = "1.37.1" +version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "polars-runtime-32" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/84/ae/dfebf31b9988c20998140b54d5b521f64ce08879f2c13d9b4d44d7c87e32/polars-1.37.1.tar.gz", hash = "sha256:0309e2a4633e712513401964b4d95452f124ceabf7aec6db50affb9ced4a274e", size = 715572, upload-time = "2026-01-12T23:27:03.267Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/e9/001f371ec6a1bb54893f599ceebd56e6144fed4091f09f09fec0021a9276/polars-2.0.0.tar.gz", hash = "sha256:62da109e27a19a9d36657ee25dc035c9d3f87e7bd610526fe467dc37ea7dc115", size = 778215, upload-time = "2026-10-06T11:51:29.679Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/08/75/ec73e38812bca7c2240aff481b9ddff20d1ad2f10dee4b3353f5eeaacdab/polars-1.37.1-py3-none-any.whl", hash = "sha256:377fed8939a2f1223c1563cfabdc7b4a3d6ff846efa1f2ddeb8644fafd9b1aff", size = 805749, upload-time = "2026-01-12T23:25:48.595Z" }, + { url = "https://files.pythonhosted.org/packages/ac/09/cc33bbd5463749c116b62c204d88bed6c02a6cb901eac7adab0d38651b07/polars-2.0.0-py3-none-any.whl", hash = "sha256:35d62f3541b7a6d4c360a2e2f07fccc0c2bcbd33b0ea51c83a25417a47a3f3ad", size = 876611, upload-time = "2026-10-06T11:44:04.327Z" }, ] [[package]] name = "polars-runtime-32" -version = "1.37.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/40/0b/addabe5e8d28a5a4c9887a08907be7ddc3fce892dc38f37d14b055438a57/polars_runtime_32-1.37.1.tar.gz", hash = "sha256:68779d4a691da20a5eb767d74165a8f80a2bdfbde4b54acf59af43f7fa028d8f", size = 2818945, upload-time = "2026-01-12T23:27:04.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/a2/e828ea9f845796de02d923edb790e408ca0b560cd68dbd74bb99a1b3c461/polars_runtime_32-1.37.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0b8d4d73ea9977d3731927740e59d814647c5198bdbe359bcf6a8bfce2e79771", size = 43499912, upload-time = "2026-01-12T23:25:51.182Z" }, - { url = "https://files.pythonhosted.org/packages/7e/46/81b71b7aa9e3703ee6e4ef1f69a87e40f58ea7c99212bf49a95071e99c8c/polars_runtime_32-1.37.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c682bf83f5f352e5e02f5c16c652c48ca40442f07b236f30662b22217320ce76", size = 39695707, upload-time = "2026-01-12T23:25:54.289Z" }, - { url = "https://files.pythonhosted.org/packages/81/2e/20009d1fde7ee919e24040f5c87cb9d0e4f8e3f109b74ba06bc10c02459c/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc82b5bbe70ca1a4b764eed1419f6336752d6ba9fc1245388d7f8b12438afa2c", size = 41467034, upload-time = "2026-01-12T23:25:56.925Z" }, - { url = "https://files.pythonhosted.org/packages/eb/21/9b55bea940524324625b1e8fd96233290303eb1bf2c23b54573487bbbc25/polars_runtime_32-1.37.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8362d11ac5193b994c7e9048ffe22ccfb976699cfbf6e128ce0302e06728894", size = 45142711, upload-time = "2026-01-12T23:26:00.817Z" }, - { url = "https://files.pythonhosted.org/packages/8c/25/c5f64461aeccdac6834a89f826d051ccd3b4ce204075e562c87a06ed2619/polars_runtime_32-1.37.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:04f5d5a2f013dca7391b7d8e7672fa6d37573a87f1d45d3dd5f0d9b5565a4b0f", size = 41638564, upload-time = "2026-01-12T23:26:04.186Z" }, - { url = "https://files.pythonhosted.org/packages/35/af/509d3cf6c45e764ccf856beaae26fc34352f16f10f94a7839b1042920a73/polars_runtime_32-1.37.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fbfde7c0ca8209eeaed546e4a32cca1319189aa61c5f0f9a2b4494262bd0c689", size = 44721136, upload-time = "2026-01-12T23:26:07.088Z" }, - { url = "https://files.pythonhosted.org/packages/af/d1/5c0a83a625f72beef59394bebc57d12637997632a4f9d3ab2ffc2cc62bbf/polars_runtime_32-1.37.1-cp310-abi3-win_amd64.whl", hash = "sha256:da3d3642ae944e18dd17109d2a3036cb94ce50e5495c5023c77b1599d4c861bc", size = 44948288, upload-time = "2026-01-12T23:26:10.214Z" }, - { url = "https://files.pythonhosted.org/packages/10/f3/061bb702465904b6502f7c9081daee34b09ccbaa4f8c94cf43a2a3b6dd6f/polars_runtime_32-1.37.1-cp310-abi3-win_arm64.whl", hash = "sha256:55f2c4847a8d2e267612f564de7b753a4bde3902eaabe7b436a0a4abf75949a0", size = 41001914, upload-time = "2026-01-12T23:26:12.997Z" }, +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/ad/dbb6f6d7070867951532bcfe5e6a648d8777b416b18cddabc07030404e8c/polars_runtime_32-2.0.0.tar.gz", hash = "sha256:b5f9afcc742b4a67eabd2c680ff0f12eb02ede9b4bf807bffabd6dbb9a58d5c7", size = 3591339, upload-time = "2026-10-06T11:51:31.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/88/d35dec6c8928dfbaa1cccf9b626a1067da906e792c92d9f994ca825ab2b5/polars_runtime_32-2.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ffb7ac6cf4e8c4a652df1951e3c3840c7c23a033603d5a9efd422fa8dd699d82", size = 52494314, upload-time = "2026-10-06T11:44:07.768Z" }, + { url = "https://files.pythonhosted.org/packages/5f/fd/2237bf53ffaff47cdf1edc6c10587a7a6444d4951150eeb08d84f3493ff8/polars_runtime_32-2.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7012d8a0201bd95638545ce8f256c0efe2c5cab0f806eb043021dddde5a9498b", size = 47930083, upload-time = "2026-10-06T11:44:11.592Z" }, + { url = "https://files.pythonhosted.org/packages/0d/0d/85e3ed90417996fc09770be91b39979074fe2978fc15b431bf8a9459760d/polars_runtime_32-2.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b85bb42e6009acc9629afcc70a83473fd468694d6a30ffb0ab376c8dd1a0a17", size = 50417889, upload-time = "2026-10-06T11:50:20.774Z" }, + { url = "https://files.pythonhosted.org/packages/83/88/e9fecfd49159da92f54ff2445883577a0f1bc195da53ecc9535c458d55dd/polars_runtime_32-2.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d6ac584ea2b38913784db943879412380d92e28ab9cb88e20a77ba71ba3f911", size = 54475036, upload-time = "2026-10-06T11:50:24.411Z" }, + { url = "https://files.pythonhosted.org/packages/48/ad/b2abf732697b21467aaaeaac0f3bf7eee0d89c59ce8125f1ed41b28a2d97/polars_runtime_32-2.0.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a6bf5e260e0a6f00d0f9181438fe9e45776df8c66cee9cba16e3675cc3888488", size = 50579474, upload-time = "2026-10-06T11:50:28.377Z" }, + { url = "https://files.pythonhosted.org/packages/7f/05/304deee59a95865e1b5e9ec7b066069b49093b81b768f473d9d3b165c686/polars_runtime_32-2.0.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55c26eef325b6840584d91aac232e9cf3ac19e1b904594b9b54131be1edeab4d", size = 54413293, upload-time = "2026-10-06T11:50:31.828Z" }, + { url = "https://files.pythonhosted.org/packages/61/59/8c9fd7199f7c4eb1b64e640306a946a2e4a46337b3bbb33b840972c7d84b/polars_runtime_32-2.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:7da1caf3c7b4f397fb213c984013a0c755557619a2d511899a1ff74392484078", size = 54229989, upload-time = "2026-10-06T11:50:35.206Z" }, + { url = "https://files.pythonhosted.org/packages/e2/93/43608026f38aa6ed4d22da8597706a61682ee403caef0021ce8e6dc73227/polars_runtime_32-2.0.0-cp310-abi3-win_arm64.whl", hash = "sha256:c30ba698c8904048df4a9bc3d6c5033cc2d0a7cbb0e13f4fd2de5a1947b61994", size = 48730655, upload-time = "2026-10-06T11:50:38.756Z" }, ] [[package]]