From 7d9467fc459bc7ac97d856e4aea3ff4ca4d1eccb Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sat, 10 Oct 2026 00:19:33 -0600 Subject: [PATCH 01/13] test(sql): add recursive CTE coverage across backends Verification-only commit for #12041: exercise WITH RECURSIVE through con.sql(), a filter on top of con.sql(), Table.sql() with a recursive query, and Table.sql() over a recursive parent, on every backend. The tests deliberately carry no xfail markers so CI reports the raw state. Also add compile-only BigQuery unit tests (the dialect from #11949) and temporarily run pytest with -v so per-test outcomes appear in CI logs even if a job times out. Assisted-by: Claude Opus 5.5 --- .../bigquery/tests/unit/test_compiler.py | 43 ++++++++++++++ ibis/backends/tests/test_dot_sql.py | 58 +++++++++++++++++++ pyproject.toml | 1 + 3 files changed, 102 insertions(+) diff --git a/ibis/backends/bigquery/tests/unit/test_compiler.py b/ibis/backends/bigquery/tests/unit/test_compiler.py index 7f7a3ec43875..e658541e9b37 100644 --- a/ibis/backends/bigquery/tests/unit/test_compiler.py +++ b/ibis/backends/bigquery/tests/unit/test_compiler.py @@ -711,3 +711,46 @@ def test_unreasonably_long_name(): match="BigQuery does not allow column names longer than 300 characters", ): ibis.to_sql(expr, dialect="bigquery") + + +RECURSIVE_COUNTER = """ +WITH RECURSIVE counter AS ( + SELECT 1 AS n + UNION ALL + SELECT n + 1 AS n FROM counter WHERE n < 3 +) +SELECT n FROM counter +""" + + +def test_recursive_cte_con_sql(): + expr = ops.SQLQueryResult( + RECURSIVE_COUNTER, ibis.schema({"n": "int64"}), ibis.backends.bigquery.Backend() + ).to_expr() + assert "WITH RECURSIVE" in to_sql(expr) + + +@pytest.mark.parametrize( + ("make_table", "query"), + [ + param( + lambda: ibis.table({"x": "int64"}, name="src"), + RECURSIVE_COUNTER.replace("SELECT 1 AS n", "SELECT x AS n FROM t"), + id="recursive_query", + ), + param( + lambda: ops.SQLQueryResult( + RECURSIVE_COUNTER, + ibis.schema({"n": "int64"}), + ibis.backends.bigquery.Backend(), + ).to_expr(), + "SELECT n FROM t", + id="recursive_parent", + ), + ], +) +def test_recursive_cte_table_sql(make_table, query): + sql = BigQueryCompiler().add_query_to_expr( + name="t", table=make_table(), query=query + ) + assert "WITH RECURSIVE" in sql diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index e5bdfe120d90..7938755db874 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -386,3 +386,61 @@ def test_scalar_dot_sql(con): sql = sg.select(sge.convert(1).as_("a")).sql(con.dialect) expr = con.sql(sql).as_scalar() assert expr.type().is_numeric() + + +RECURSIVE_COUNTER = """ +WITH RECURSIVE "counter" AS ( + SELECT 1 AS "n" + UNION ALL + SELECT "n" + 1 AS "n" FROM "counter" WHERE "n" < 3 +) +SELECT "n" FROM "counter" +""" + +RECURSIVE_COUNTER_FROM_T = """ +WITH RECURSIVE "counter" AS ( + SELECT "x" AS "n" FROM "t" + UNION ALL + SELECT "n" + 1 AS "n" FROM "counter" WHERE "n" < 3 +) +SELECT "n" FROM "counter" +""" + + +@pytest.mark.parametrize( + ("make_expr", "expected"), + [ + param( + lambda con: con.sql(RECURSIVE_COUNTER, dialect="duckdb"), + [1, 2, 3], + id="con_sql", + ), + param( + lambda con: con.sql(RECURSIVE_COUNTER, dialect="duckdb").filter(_.n > 1), + [2, 3], + id="con_sql_filtered", + ), + param( + lambda con: ( + con.sql('SELECT 1 AS "x"', dialect="duckdb") + .alias("t") + .sql(RECURSIVE_COUNTER_FROM_T, dialect="duckdb") + ), + [1, 2, 3], + id="table_sql_recursive_query", + ), + param( + lambda con: ( + con.sql(RECURSIVE_COUNTER, dialect="duckdb") + .alias("t") + .sql('SELECT "n" FROM "t"', dialect="duckdb") + ), + [1, 2, 3], + id="table_sql_recursive_parent", + ), + ], +) +def test_dot_sql_recursive_cte(con, make_expr, expected): + expr = make_expr(con) + result = expr.execute() + assert result["n"].astype("int64").sort_values().tolist() == expected diff --git a/pyproject.toml b/pyproject.toml index 983305ee9931..31b5efcaa231 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,6 +383,7 @@ xfail_strict = true addopts = [ "--strict-markers", "--strict-config", + "-v", "--benchmark-disable", "--benchmark-group-by=name", "--benchmark-sort=name", From 08492755b909c807f5e45a33b1a57456855d47b6 Mon Sep 17 00:00:00 2001 From: Bhargavi Sriram Date: Sat, 10 Oct 2026 00:20:26 -0600 Subject: [PATCH 02/13] fix(sql): preserve RECURSIVE keyword when rebuilding WITH clause The translate() and add_query_to_expr() methods in SQLGlotCompiler discarded the With node's recursive flag when reconstructing CTEs. Capture the flag before the pop and restore it after the reduce. Cherry-picked from #12041 (code change only; the tests come from the previous commit). Fixes #11949 --- ibis/backends/sql/compilers/base.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ibis/backends/sql/compilers/base.py b/ibis/backends/sql/compilers/base.py index 2e7333ce1bc4..2ecf2be02edf 100644 --- a/ibis/backends/sql/compilers/base.py +++ b/ibis/backends/sql/compilers/base.py @@ -682,6 +682,11 @@ def fn(node, __unused__, **kwargs): alias=sg.to_identifier(aliases[cte], quoted=self.quoted), this=this ) merged_ctes.append(modified_cte) + + # Capture the recursive flag before the With node is discarded + with_node = out.args.get(WITH_ARG) + is_recursive = with_node is not None and bool(with_node.args.get("recursive")) + merged_ctes.extend(out.ctes) out.args.pop(WITH_ARG, None) @@ -696,6 +701,10 @@ def fn(node, __unused__, **kwargs): out, ) + # Restore the recursive flag on the rebuilt With node + if is_recursive and WITH_ARG in out.args: + out.args[WITH_ARG].set("recursive", True) + return out def visit_node(self, op: ops.Node, **kwargs): @@ -1617,6 +1626,10 @@ def add_query_to_expr(self, *, name: str, table: ir.Table, query: str) -> str: ), *compiled_query.ctes, ] + # Capture the recursive flag before the With node is discarded + with_node = compiled_query.args.get(WITH_ARG) + is_recursive = with_node is not None and bool(with_node.args.get("recursive")) + compiled_ibis_expr.args.pop(WITH_ARG, None) compiled_query.args.pop(WITH_ARG, None) @@ -1628,6 +1641,10 @@ def add_query_to_expr(self, *, name: str, table: ir.Table, query: str) -> str: compiled_query, ) + # Restore the recursive flag on the rebuilt With node + if is_recursive and WITH_ARG in parsed.args: + parsed.args[WITH_ARG].set("recursive", True) + # generate the SQL string return parsed.sql(dialect) From 35489f6586d3d0e896c7c6c12c96077935caa0f2 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sat, 10 Oct 2026 00:21:00 -0600 Subject: [PATCH 03/13] fix(sql): pass the recursive flag through when rebuilding WITH clauses Instead of saving and restoring the flag around the rebuild, hand it to sqlglot's with_(recursive=...) directly. In add_query_to_expr, also keep the flag from the compiled ibis expression, not just the user's query, so `con.sql(recursive_query).alias("t").sql("SELECT ... FROM t")` works. Build ibis's own CTE aliases as TableAlias rather than bare Identifier. With RECURSIVE set, sqlglot's Presto/Trino generator reads `cte.args["alias"].columns` on every CTE, which raised AttributeError on Trino and Athena. SingleStoreDB overrides add_query_to_expr with a copy of the same logic, so apply the same changes there. Assisted-by: Claude Opus 5.5 --- ibis/backends/sql/compilers/base.py | 40 +++++++++----------- ibis/backends/sql/compilers/singlestoredb.py | 14 +++++-- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/ibis/backends/sql/compilers/base.py b/ibis/backends/sql/compilers/base.py index 2ecf2be02edf..b139bd8323d8 100644 --- a/ibis/backends/sql/compilers/base.py +++ b/ibis/backends/sql/compilers/base.py @@ -679,21 +679,21 @@ def fn(node, __unused__, **kwargs): if "alias" in this.args: this = this.this modified_cte = sge.CTE( - alias=sg.to_identifier(aliases[cte], quoted=self.quoted), this=this + alias=sge.TableAlias( + this=sg.to_identifier(aliases[cte], quoted=self.quoted) + ), + this=this, ) merged_ctes.append(modified_cte) - - # Capture the recursive flag before the With node is discarded - with_node = out.args.get(WITH_ARG) - is_recursive = with_node is not None and bool(with_node.args.get("recursive")) - merged_ctes.extend(out.ctes) - out.args.pop(WITH_ARG, None) + with_ = out.args.pop(WITH_ARG, None) + recursive = with_ is not None and bool(with_.args.get("recursive")) out = reduce( lambda parsed, cte: parsed.with_( cte.args["alias"], as_=cte.args["this"], + recursive=recursive, dialect=self.dialect, copy=False, ), @@ -701,10 +701,6 @@ def fn(node, __unused__, **kwargs): out, ) - # Restore the recursive flag on the rebuilt With node - if is_recursive and WITH_ARG in out.args: - out.args[WITH_ARG].set("recursive", True) - return out def visit_node(self, op: ops.Node, **kwargs): @@ -1621,30 +1617,28 @@ def add_query_to_expr(self, *, name: str, table: ir.Table, query: str) -> str: ctes = [ *compiled_ibis_expr.ctes, sge.CTE( - alias=sg.to_identifier(name, quoted=self.quoted), + alias=sge.TableAlias(this=sg.to_identifier(name, quoted=self.quoted)), this=compiled_ibis_expr, ), *compiled_query.ctes, ] - # Capture the recursive flag before the With node is discarded - with_node = compiled_query.args.get(WITH_ARG) - is_recursive = with_node is not None and bool(with_node.args.get("recursive")) - - compiled_ibis_expr.args.pop(WITH_ARG, None) - compiled_query.args.pop(WITH_ARG, None) + # either side may contain a recursive CTE, and RECURSIVE applies to the + # whole WITH clause, so keep it if either one had it + recursive = False + for expr in (compiled_ibis_expr, compiled_query): + if (with_ := expr.args.pop(WITH_ARG, None)) is not None: + recursive |= bool(with_.args.get("recursive")) # pull existing CTEs from the compiled Ibis expression and combine them # with the new query parsed = reduce( - lambda parsed, cte: parsed.with_(cte.args["alias"], as_=cte.args["this"]), + lambda parsed, cte: parsed.with_( + cte.args["alias"], as_=cte.args["this"], recursive=recursive + ), ctes, compiled_query, ) - # Restore the recursive flag on the rebuilt With node - if is_recursive and WITH_ARG in parsed.args: - parsed.args[WITH_ARG].set("recursive", True) - # generate the SQL string return parsed.sql(dialect) diff --git a/ibis/backends/sql/compilers/singlestoredb.py b/ibis/backends/sql/compilers/singlestoredb.py index 8d429137deb2..f63dfb0fe205 100644 --- a/ibis/backends/sql/compilers/singlestoredb.py +++ b/ibis/backends/sql/compilers/singlestoredb.py @@ -825,18 +825,24 @@ def add_query_to_expr(self, *, name: str, table, query: str) -> str: ctes = [ *compiled_ibis_expr.ctes, sge.CTE( - alias=sg.to_identifier(name, quoted=self.quoted), + alias=sge.TableAlias(this=sg.to_identifier(name, quoted=self.quoted)), this=compiled_ibis_expr, ), *compiled_query.ctes, ] - compiled_ibis_expr.args.pop(WITH_ARG, None) - compiled_query.args.pop(WITH_ARG, None) + # either side may contain a recursive CTE, and RECURSIVE applies to the + # whole WITH clause, so keep it if either one had it + recursive = False + for expr in (compiled_ibis_expr, compiled_query): + if (with_ := expr.args.pop(WITH_ARG, None)) is not None: + recursive |= bool(with_.args.get("recursive")) # pull existing CTEs from the compiled Ibis expression and combine them # with the new query parsed = reduce( - lambda parsed, cte: parsed.with_(cte.args["alias"], as_=cte.args["this"]), + lambda parsed, cte: parsed.with_( + cte.args["alias"], as_=cte.args["this"], recursive=recursive + ), ctes, compiled_query, ) From 4f40d0bfcb2168258298cf2276a9cb5d643720b7 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sun, 11 Oct 2026 21:15:51 -0600 Subject: [PATCH 04/13] ci: run only dot-sql, CTE and recursive tests (verification only) Restrict pytest to the tests relevant to #12041 so the BigQuery job finishes within its 60-minute limit and prints its failure summary, and so the other backends turn around faster. Jobs that select none of these tests (doctests, core) will exit with "no tests collected". Assisted-by: Claude Opus 5.5 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 31b5efcaa231..71248f41a1b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -384,6 +384,8 @@ addopts = [ "--strict-markers", "--strict-config", "-v", + "-k", + "recursive_cte or test_dot_sql or test_sql", "--benchmark-disable", "--benchmark-group-by=name", "--benchmark-sort=name", From 523893305b23fdc4524ad8da7d5e5819942419fe Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sun, 11 Oct 2026 21:16:04 -0600 Subject: [PATCH 05/13] test(sql): don't alias the recursive step's output in recursive CTE tests ClickHouse lets WHERE refer to SELECT aliases, so in `SELECT "n" + 1 AS "n" FROM "counter" WHERE "n" < 3` the WHERE tests the incremented value and recursion stops one step early ([1, 2] instead of [1, 2, 3]). The recursive term's column names come from the anchor term anyway, so the alias was redundant. Assisted-by: Claude Opus 5.5 --- ibis/backends/bigquery/tests/unit/test_compiler.py | 2 +- ibis/backends/tests/test_dot_sql.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ibis/backends/bigquery/tests/unit/test_compiler.py b/ibis/backends/bigquery/tests/unit/test_compiler.py index e658541e9b37..80f5147da4ca 100644 --- a/ibis/backends/bigquery/tests/unit/test_compiler.py +++ b/ibis/backends/bigquery/tests/unit/test_compiler.py @@ -717,7 +717,7 @@ def test_unreasonably_long_name(): WITH RECURSIVE counter AS ( SELECT 1 AS n UNION ALL - SELECT n + 1 AS n FROM counter WHERE n < 3 + SELECT n + 1 FROM counter WHERE n < 3 ) SELECT n FROM counter """ diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index 7938755db874..0f877d7fbc54 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -392,7 +392,7 @@ def test_scalar_dot_sql(con): WITH RECURSIVE "counter" AS ( SELECT 1 AS "n" UNION ALL - SELECT "n" + 1 AS "n" FROM "counter" WHERE "n" < 3 + SELECT "n" + 1 FROM "counter" WHERE "n" < 3 ) SELECT "n" FROM "counter" """ @@ -401,7 +401,7 @@ def test_scalar_dot_sql(con): WITH RECURSIVE "counter" AS ( SELECT "x" AS "n" FROM "t" UNION ALL - SELECT "n" + 1 AS "n" FROM "counter" WHERE "n" < 3 + SELECT "n" + 1 FROM "counter" WHERE "n" < 3 ) SELECT "n" FROM "counter" """ From e2f585ede619fd5a9667ba2b35349be42b00d00b Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sun, 11 Oct 2026 21:17:08 -0600 Subject: [PATCH 06/13] test(sql): mark recursive CTE tests for backends that can't run them Markers follow the errors observed in CI on #12153. Polars, Exasol, RisingWave, Druid, Flink and Impala don't support recursive CTEs; Materialize only supports WITH MUTUALLY RECURSIVE; sqlglot's Oracle dialect emits a RECURSIVE keyword Oracle rejects; SingleStoreDB rejects the query shape; Spark < 4.1 fails to parse (3.5) or plan (4.0) them. BigQuery fails only when the recursive WITH ends up nested inside a subquery or CTE. Assisted-by: Claude Opus 5.5 --- ibis/backends/tests/test_dot_sql.py | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index 0f877d7fbc54..ba43510a5881 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -16,9 +16,16 @@ from ibis.backends.tests.errors import ( ExaQueryError, GoogleBadRequest, + ImpalaHiveServer2Error, OracleDatabaseError, + PolarsSQLInterfaceError, + PsycoPg2InternalError, + PsycoPgSyntaxError, + Py4JJavaError, PyAthenaOperationalError, + PyDruidProgrammingError, PyODBCProgrammingError, + SingleStoreDBOperationalError, ) pd = pytest.importorskip("pandas") @@ -418,6 +425,11 @@ def test_scalar_dot_sql(con): param( lambda con: con.sql(RECURSIVE_COUNTER, dialect="duckdb").filter(_.n > 1), [2, 3], + marks=pytest.mark.notyet( + ["bigquery"], + raises=GoogleBadRequest, + reason="BigQuery rejects WITH RECURSIVE nested in a subquery or CTE", + ), id="con_sql_filtered", ), param( @@ -436,10 +448,62 @@ def test_scalar_dot_sql(con): .sql('SELECT "n" FROM "t"', dialect="duckdb") ), [1, 2, 3], + marks=pytest.mark.notyet( + ["bigquery"], + raises=GoogleBadRequest, + reason="BigQuery rejects WITH RECURSIVE nested in a subquery or CTE", + ), id="table_sql_recursive_parent", ), ], ) +@pytest.mark.notyet( + ["polars"], + raises=PolarsSQLInterfaceError, + reason="Polars SQL doesn't support recursive CTEs", +) +@pytest.mark.notyet( + ["exasol"], raises=ExaQueryError, reason="Exasol doesn't support WITH RECURSIVE" +) +@pytest.mark.notyet( + ["risingwave"], + raises=PsycoPg2InternalError, + reason="RisingWave doesn't support recursive CTEs in batch queries", +) +@pytest.mark.notyet( + ["druid"], + raises=PyDruidProgrammingError, + reason="Druid doesn't support recursive CTEs", +) +@pytest.mark.notyet( + ["flink"], + raises=Py4JJavaError, + reason="Flink doesn't support recursive CTEs; sqlglot drops RECURSIVE", +) +@pytest.mark.notyet( + ["impala"], + raises=ImpalaHiveServer2Error, + reason="Impala doesn't support recursive CTEs; sqlglot drops RECURSIVE", +) +@pytest.mark.notyet( + ["materialize"], + raises=PsycoPgSyntaxError, + reason="Materialize only supports WITH MUTUALLY RECURSIVE", +) +@pytest.mark.notyet( + ["oracle"], + raises=OracleDatabaseError, + reason="sqlglot emits WITH RECURSIVE, which Oracle doesn't accept", +) +@pytest.mark.notyet( + ["singlestoredb"], + raises=SingleStoreDBOperationalError, + reason="unsupported recursive common table expression query shape", +) +@pytest.mark.notyet( + ["pyspark"], + reason="Spark < 4.1 doesn't support recursive CTEs (3.5 fails to parse, 4.0 to plan)", +) def test_dot_sql_recursive_cte(con, make_expr, expected): expr = make_expr(con) result = expr.execute() From c214cf2c6f4c6540bd665e88bf374c6973cefc1d Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sun, 11 Oct 2026 21:17:09 -0600 Subject: [PATCH 07/13] test(mssql): update CTE snapshot for TableAlias CTE aliases With CTE aliases built as TableAlias, sqlglot's T-SQL generator names CTE projections explicitly. The SQL is equivalent. Assisted-by: Claude Opus 5.5 --- .../test_sql/test_cte_refs_in_topo_order/mssql/out.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis/backends/tests/snapshots/test_sql/test_cte_refs_in_topo_order/mssql/out.sql b/ibis/backends/tests/snapshots/test_sql/test_cte_refs_in_topo_order/mssql/out.sql index 7a46232efae6..38448f048aef 100644 --- a/ibis/backends/tests/snapshots/test_sql/test_cte_refs_in_topo_order/mssql/out.sql +++ b/ibis/backends/tests/snapshots/test_sql/test_cte_refs_in_topo_order/mssql/out.sql @@ -1,6 +1,6 @@ WITH [t1] AS ( SELECT - [t0].[key] + [t0].[key] AS [key] FROM [leaf] AS [t0] WHERE (1 = 1) From 9f048b91485759391a5d479cd0ae42320e565007 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sun, 11 Oct 2026 22:02:02 -0600 Subject: [PATCH 08/13] test(bigquery): probe where BigQuery accepts WITH RECURSIVE (diagnostic) Temporary, verification-only: run raw recursive CTE placements against BigQuery so the failure messages explain the two xfailed nested cases. Assisted-by: Claude Opus 5.5 --- ibis/backends/tests/test_dot_sql.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index ba43510a5881..fe7593db03a9 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -508,3 +508,33 @@ def test_dot_sql_recursive_cte(con, make_expr, expected): expr = make_expr(con) result = expr.execute() assert result["n"].astype("int64").sort_values().tolist() == expected + + +_REC = ( + "WITH RECURSIVE c AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM c WHERE n < 3) " + "SELECT n FROM c" +) + + +@pytest.mark.parametrize( + "sql", + [ + param(_REC, id="top_level"), + param(f"SELECT * FROM ({_REC}) AS t0 WHERE n > 1", id="nested_in_subquery"), + param(f"WITH t AS ({_REC}) SELECT n FROM t", id="nested_in_cte"), + param( + "WITH RECURSIVE c AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM c " + "WHERE n < 3), t AS (SELECT n FROM c) SELECT n FROM t", + id="hoisted", + ), + param( + "SELECT * FROM (WITH c AS (SELECT 1 AS n) SELECT n FROM c) AS t0", + id="control_nonrecursive_nested", + ), + ], +) +def test_recursive_cte_bigquery_probe(con, sql): + """Temporary diagnostic: which placements of WITH RECURSIVE BigQuery accepts.""" + if con.name != "bigquery": + pytest.skip("BigQuery-only diagnostic") + assert len(list(con.raw_sql(sql))) > 0 From e766038ae7f018e051d3f003b4623148931fe196 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Tue, 13 Oct 2026 10:39:25 -0600 Subject: [PATCH 09/13] refactor(sql): check the `With.recursive` property sqlglot's `With` node has had a `recursive` property since before our 23.4 floor, so use it instead of reading the raw arg. Assisted-by: Claude Fable 5.1 --- ibis/backends/sql/compilers/base.py | 4 ++-- ibis/backends/sql/compilers/singlestoredb.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ibis/backends/sql/compilers/base.py b/ibis/backends/sql/compilers/base.py index b139bd8323d8..c49d4b918b83 100644 --- a/ibis/backends/sql/compilers/base.py +++ b/ibis/backends/sql/compilers/base.py @@ -687,7 +687,7 @@ def fn(node, __unused__, **kwargs): merged_ctes.append(modified_cte) merged_ctes.extend(out.ctes) with_ = out.args.pop(WITH_ARG, None) - recursive = with_ is not None and bool(with_.args.get("recursive")) + recursive = with_ is not None and with_.recursive out = reduce( lambda parsed, cte: parsed.with_( @@ -1627,7 +1627,7 @@ def add_query_to_expr(self, *, name: str, table: ir.Table, query: str) -> str: recursive = False for expr in (compiled_ibis_expr, compiled_query): if (with_ := expr.args.pop(WITH_ARG, None)) is not None: - recursive |= bool(with_.args.get("recursive")) + recursive |= with_.recursive # pull existing CTEs from the compiled Ibis expression and combine them # with the new query diff --git a/ibis/backends/sql/compilers/singlestoredb.py b/ibis/backends/sql/compilers/singlestoredb.py index f63dfb0fe205..c1baf4fafc3d 100644 --- a/ibis/backends/sql/compilers/singlestoredb.py +++ b/ibis/backends/sql/compilers/singlestoredb.py @@ -835,7 +835,7 @@ def add_query_to_expr(self, *, name: str, table, query: str) -> str: recursive = False for expr in (compiled_ibis_expr, compiled_query): if (with_ := expr.args.pop(WITH_ARG, None)) is not None: - recursive |= bool(with_.args.get("recursive")) + recursive |= with_.recursive # pull existing CTEs from the compiled Ibis expression and combine them # with the new query From 50666a48b6aa0f296dc4abf6ff29762ae4415caa Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Tue, 13 Oct 2026 10:39:25 -0600 Subject: [PATCH 10/13] test(sql): share recursive CTE compile-only checks The compile-only tests exercised base compiler behavior but lived in the BigQuery unit tests with their own copy of the recursive query. Move them next to the execution test, reuse its query, and run them for every SQL backend's dialect the way `test_sql.py` does, so they need no connection. Compilers are loaded by module path because `ibis.backends.sql.compilers` doesn't export `singlestoredb`. Assisted-by: Claude Fable 5.1 --- .../bigquery/tests/unit/test_compiler.py | 43 -------------- ibis/backends/tests/test_dot_sql.py | 59 +++++++++++++++++++ 2 files changed, 59 insertions(+), 43 deletions(-) diff --git a/ibis/backends/bigquery/tests/unit/test_compiler.py b/ibis/backends/bigquery/tests/unit/test_compiler.py index 80f5147da4ca..7f7a3ec43875 100644 --- a/ibis/backends/bigquery/tests/unit/test_compiler.py +++ b/ibis/backends/bigquery/tests/unit/test_compiler.py @@ -711,46 +711,3 @@ def test_unreasonably_long_name(): match="BigQuery does not allow column names longer than 300 characters", ): ibis.to_sql(expr, dialect="bigquery") - - -RECURSIVE_COUNTER = """ -WITH RECURSIVE counter AS ( - SELECT 1 AS n - UNION ALL - SELECT n + 1 FROM counter WHERE n < 3 -) -SELECT n FROM counter -""" - - -def test_recursive_cte_con_sql(): - expr = ops.SQLQueryResult( - RECURSIVE_COUNTER, ibis.schema({"n": "int64"}), ibis.backends.bigquery.Backend() - ).to_expr() - assert "WITH RECURSIVE" in to_sql(expr) - - -@pytest.mark.parametrize( - ("make_table", "query"), - [ - param( - lambda: ibis.table({"x": "int64"}, name="src"), - RECURSIVE_COUNTER.replace("SELECT 1 AS n", "SELECT x AS n FROM t"), - id="recursive_query", - ), - param( - lambda: ops.SQLQueryResult( - RECURSIVE_COUNTER, - ibis.schema({"n": "int64"}), - ibis.backends.bigquery.Backend(), - ).to_expr(), - "SELECT n FROM t", - id="recursive_parent", - ), - ], -) -def test_recursive_cte_table_sql(make_table, query): - sql = BigQueryCompiler().add_query_to_expr( - name="t", table=make_table(), query=query - ) - assert "WITH RECURSIVE" in sql diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index fe7593db03a9..a876f58981c7 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -1,6 +1,7 @@ from __future__ import annotations import getpass +import importlib import pytest import sqlglot as sg @@ -10,8 +11,10 @@ import ibis import ibis.backends.sql.dialects # to load dialects import ibis.common.exceptions as com +import ibis.expr.operations as ops from ibis import _ from ibis.backends import _get_backend_names +from ibis.backends.conftest import _get_backends_to_test from ibis.backends.tests.base import PYTHON_SHORT_VERSION from ibis.backends.tests.errors import ( ExaQueryError, @@ -510,6 +513,62 @@ def test_dot_sql_recursive_cte(con, make_expr, expected): assert result["n"].astype("int64").sort_values().tolist() == expected +def _sql_query_result(query, compiler): + """Build what `con.sql(query, dialect="duckdb")` builds, without a connection.""" + query = sg.transpile(query, read="duckdb", write=compiler.dialect)[0] + return ops.SQLQueryResult(query, ibis.schema({"n": "int64"}), None).to_expr() + + +@pytest.mark.never( + ["mssql"], raises=AssertionError, reason="T-SQL has no RECURSIVE keyword" +) +@pytest.mark.never( + ["flink", "impala"], + raises=AssertionError, + reason="recursive CTEs aren't supported, so sqlglot drops RECURSIVE", +) +@pytest.mark.parametrize("backend_name", _get_backends_to_test(discard=("polars",))) +def test_recursive_cte_con_sql_compiles(backend_name): + compiler = importlib.import_module( + f"ibis.backends.sql.compilers.{backend_name}" + ).compiler + expr = _sql_query_result(RECURSIVE_COUNTER, compiler) + assert "WITH RECURSIVE" in compiler.to_sqlglot(expr).sql(compiler.dialect) + + +@pytest.mark.never( + ["mssql"], raises=AssertionError, reason="T-SQL has no RECURSIVE keyword" +) +@pytest.mark.never( + ["flink", "impala"], + raises=AssertionError, + reason="recursive CTEs aren't supported, so sqlglot drops RECURSIVE", +) +@pytest.mark.parametrize( + ("make_table", "query"), + [ + param( + lambda _compiler: ibis.table({"x": "int64"}, name="src"), + RECURSIVE_COUNTER_FROM_T, + id="recursive_query", + ), + param( + lambda compiler: _sql_query_result(RECURSIVE_COUNTER, compiler), + 'SELECT "n" FROM "t"', + id="recursive_parent", + ), + ], +) +@pytest.mark.parametrize("backend_name", _get_backends_to_test(discard=("polars",))) +def test_recursive_cte_table_sql_compiles(backend_name, make_table, query): + compiler = importlib.import_module( + f"ibis.backends.sql.compilers.{backend_name}" + ).compiler + query = sg.transpile(query, read="duckdb", write=compiler.dialect)[0] + sql = compiler.add_query_to_expr(name="t", table=make_table(compiler), query=query) + assert "WITH RECURSIVE" in sql + + _REC = ( "WITH RECURSIVE c AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM c WHERE n < 3) " "SELECT n FROM c" From 1a64d5582d7577e5f75b621e074f80c0f8dab318 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Tue, 13 Oct 2026 16:59:49 -0600 Subject: [PATCH 11/13] test(sql): dedupe recursive CTE compile test marks Collect the marks shared by the two recursive CTE compile tests into a single composed decorator, following the `builtin_array` precedent in test_array.py, and move the per-backend compiler lookup into a helper. Flink and Impala move from `never` to `notyet`: neither supports recursive CTEs today, but that is not a permanent fact about the SQL dialect the way the missing RECURSIVE keyword is for T-SQL, and the execution test in this file already marks them `notyet`. Also gate PySpark and Databricks on `sqlglot<30`: sqlglot drops the RECURSIVE keyword for Spark up to and including 29.0.1 and keeps it from 30.0.0 (sqlglot#7214). The lock pins 30.18, so this only matters for the declared lower bound. Assisted-by: Claude Fable 5.1 --- ibis/backends/tests/test_dot_sql.py | 51 +++++++++++++++++------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index a876f58981c7..92751b7d6a35 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -6,6 +6,7 @@ import pytest import sqlglot as sg import sqlglot.expressions as sge +import toolz from pytest import param import ibis @@ -519,31 +520,41 @@ def _sql_query_result(query, compiler): return ops.SQLQueryResult(query, ibis.schema({"n": "int64"}), None).to_expr() -@pytest.mark.never( - ["mssql"], raises=AssertionError, reason="T-SQL has no RECURSIVE keyword" -) -@pytest.mark.never( - ["flink", "impala"], - raises=AssertionError, - reason="recursive CTEs aren't supported, so sqlglot drops RECURSIVE", +def _compiler(backend_name): + # not `getattr(sc, backend_name)`: the compilers package doesn't export + # singlestoredb (see #12164) + return importlib.import_module( + f"ibis.backends.sql.compilers.{backend_name}" + ).compiler + + +no_recursive_keyword = toolz.compose( + pytest.mark.never( + ["mssql"], raises=AssertionError, reason="T-SQL has no RECURSIVE keyword" + ), + pytest.mark.notyet( + ["flink", "impala"], + raises=AssertionError, + reason="recursive CTEs aren't supported, so sqlglot drops RECURSIVE", + ), + pytest.mark.xfail_version( + pyspark=["sqlglot<30"], + databricks=["sqlglot<30"], + raises=AssertionError, + reason="sqlglot drops RECURSIVE for Spark before 30.0", + ), ) + + +@no_recursive_keyword @pytest.mark.parametrize("backend_name", _get_backends_to_test(discard=("polars",))) def test_recursive_cte_con_sql_compiles(backend_name): - compiler = importlib.import_module( - f"ibis.backends.sql.compilers.{backend_name}" - ).compiler + compiler = _compiler(backend_name) expr = _sql_query_result(RECURSIVE_COUNTER, compiler) assert "WITH RECURSIVE" in compiler.to_sqlglot(expr).sql(compiler.dialect) -@pytest.mark.never( - ["mssql"], raises=AssertionError, reason="T-SQL has no RECURSIVE keyword" -) -@pytest.mark.never( - ["flink", "impala"], - raises=AssertionError, - reason="recursive CTEs aren't supported, so sqlglot drops RECURSIVE", -) +@no_recursive_keyword @pytest.mark.parametrize( ("make_table", "query"), [ @@ -561,9 +572,7 @@ def test_recursive_cte_con_sql_compiles(backend_name): ) @pytest.mark.parametrize("backend_name", _get_backends_to_test(discard=("polars",))) def test_recursive_cte_table_sql_compiles(backend_name, make_table, query): - compiler = importlib.import_module( - f"ibis.backends.sql.compilers.{backend_name}" - ).compiler + compiler = _compiler(backend_name) query = sg.transpile(query, read="duckdb", write=compiler.dialect)[0] sql = compiler.add_query_to_expr(name="t", table=make_table(compiler), query=query) assert "WITH RECURSIVE" in sql From 2af057b5825b255fe1525a8e93842860fdd65059 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Tue, 13 Oct 2026 10:43:21 -0600 Subject: [PATCH 12/13] fix(sql): export the SingleStoreDB compiler module `ibis.to_sql` resolves a dialect name by attribute lookup on `ibis.backends.sql.compilers`, which never imported the `singlestoredb` module. `ibis.to_sql(expr, dialect="singlestoredb")` therefore raised "Unknown dialect" unless the backend had already been imported. Add a test that compiles a trivial table for every SQL backend name. Assisted-by: Claude Fable 5.1 --- ibis/backends/sql/compilers/__init__.py | 2 ++ ibis/backends/sql/tests/test_compiler.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/ibis/backends/sql/compilers/__init__.py b/ibis/backends/sql/compilers/__init__.py index 4a85c54594cc..57d96cd6bf08 100644 --- a/ibis/backends/sql/compilers/__init__.py +++ b/ibis/backends/sql/compilers/__init__.py @@ -19,6 +19,7 @@ "PySparkCompiler", "RisingWaveCompiler", "SQLiteCompiler", + "SingleStoreDBCompiler", "SnowflakeCompiler", "TrinoCompiler", ] @@ -40,6 +41,7 @@ from ibis.backends.sql.compilers.postgres import PostgresCompiler from ibis.backends.sql.compilers.pyspark import PySparkCompiler from ibis.backends.sql.compilers.risingwave import RisingWaveCompiler +from ibis.backends.sql.compilers.singlestoredb import SingleStoreDBCompiler from ibis.backends.sql.compilers.snowflake import SnowflakeCompiler from ibis.backends.sql.compilers.sqlite import SQLiteCompiler from ibis.backends.sql.compilers.trino import TrinoCompiler diff --git a/ibis/backends/sql/tests/test_compiler.py b/ibis/backends/sql/tests/test_compiler.py index 3596a7287b8c..f9248b664f6b 100644 --- a/ibis/backends/sql/tests/test_compiler.py +++ b/ibis/backends/sql/tests/test_compiler.py @@ -1,9 +1,11 @@ from __future__ import annotations +import pytest import sqlglot as sg import ibis from ibis import _ +from ibis.backends import _get_backend_names from ibis.backends.sql.dialects import Trino @@ -26,3 +28,11 @@ def test_transpile_join(): "SELECT * FROM t1 JOIN t2 ON x = y", read="duckdb", write=Trino ) assert "CROSS JOIN" not in result + + +@pytest.mark.parametrize("dialect", sorted(_get_backend_names(exclude=("polars",)))) +def test_to_sql_knows_every_sql_backend(dialect): + # `ibis.to_sql` looks the dialect up on `ibis.backends.sql.compilers`, so + # every SQL backend's compiler module has to be imported there + expr = ibis.table({"a": "int64"}, name="t") + assert ibis.to_sql(expr, dialect=dialect) From 1c6bac446f22e39ca38dca720084d1130bd256f4 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Tue, 13 Oct 2026 17:29:44 -0600 Subject: [PATCH 13/13] test(sql): look up compilers like test_sql.py does With the SingleStoreDB compiler module exported from the compilers package, the recursive CTE compile tests can use `getattr(sc, backend_name)` the way test_sql.py does, instead of a helper that worked around the missing export via importlib. Assisted-by: Claude Fable 5.1 --- ibis/backends/tests/test_dot_sql.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index 92751b7d6a35..f8f2e3c9a20a 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -1,7 +1,6 @@ from __future__ import annotations import getpass -import importlib import pytest import sqlglot as sg @@ -10,6 +9,7 @@ from pytest import param import ibis +import ibis.backends.sql.compilers as sc import ibis.backends.sql.dialects # to load dialects import ibis.common.exceptions as com import ibis.expr.operations as ops @@ -520,14 +520,6 @@ def _sql_query_result(query, compiler): return ops.SQLQueryResult(query, ibis.schema({"n": "int64"}), None).to_expr() -def _compiler(backend_name): - # not `getattr(sc, backend_name)`: the compilers package doesn't export - # singlestoredb (see #12164) - return importlib.import_module( - f"ibis.backends.sql.compilers.{backend_name}" - ).compiler - - no_recursive_keyword = toolz.compose( pytest.mark.never( ["mssql"], raises=AssertionError, reason="T-SQL has no RECURSIVE keyword" @@ -549,7 +541,7 @@ def _compiler(backend_name): @no_recursive_keyword @pytest.mark.parametrize("backend_name", _get_backends_to_test(discard=("polars",))) def test_recursive_cte_con_sql_compiles(backend_name): - compiler = _compiler(backend_name) + compiler = getattr(sc, backend_name).compiler expr = _sql_query_result(RECURSIVE_COUNTER, compiler) assert "WITH RECURSIVE" in compiler.to_sqlglot(expr).sql(compiler.dialect) @@ -572,7 +564,7 @@ def test_recursive_cte_con_sql_compiles(backend_name): ) @pytest.mark.parametrize("backend_name", _get_backends_to_test(discard=("polars",))) def test_recursive_cte_table_sql_compiles(backend_name, make_table, query): - compiler = _compiler(backend_name) + compiler = getattr(sc, backend_name).compiler query = sg.transpile(query, read="duckdb", write=compiler.dialect)[0] sql = compiler.add_query_to_expr(name="t", table=make_table(compiler), query=query) assert "WITH RECURSIVE" in sql