Merge pull request #442 from Tmonster/no_inserts_to_v3_tables

Disable Inserts into V3 tables

Author: Tmonster

duckdb

@@ -12,8 +12,7 @@ duckdb_extension_load(icu)
 duckdb_extension_load(ducklake
         LOAD_TESTS
         GIT_URL https://github.com/duckdb/ducklake
-        GIT_TAG d2392c36f33151cf5cdd7d006375b0b669bd44ac
-	APPLY_PATCHES
+        GIT_TAG c1ebd032eb4c763910551c08f4b61bdb8168f209
 )
 
 duckdb_extension_load(avro

extension_config.cmake

@@ -65,7 +65,7 @@ def generate(self, con: IcebergConnection):
                     intermediate_data_path = os.path.join(intermediate_dir, snapshot_name, 'data.parquet')
                     df.write.mode("overwrite").parquet(intermediate_data_path)
 
-        if self.write_intermediates:
+        if self.write_intermediates and last_file:
             ### Finally, copy the latest results to a "last" dir for easy test writing
             shutil.copytree(
                 os.path.join(intermediate_dir, last_file, 'data.parquet'),

scripts/data_generators/tests/base.py

@@ -0,0 +1,22 @@
+from scripts.data_generators.tests.base import IcebergTest
+import pathlib
+import tempfile
+import duckdb
+
+
+@IcebergTest.register()
+class Test(IcebergTest):
+    def __init__(self):
+        path = pathlib.PurePath(__file__)
+        super().__init__(path.parent.name)
+
+        # Create a temporary directory
+        self.tempdir = pathlib.Path(tempfile.mkdtemp())
+        self.parquet_file = self.tempdir / "tmp.parquet"
+
+        duckdb_con = duckdb.connect()
+        duckdb_con.execute("call dbgen(sf=1)")
+        duckdb_con.execute(f"copy customer to '{self.parquet_file}' (FORMAT PARQUET)")
+
+    def setup(self, con):
+        con.con.read.parquet(self.parquet_file.as_posix()).createOrReplaceTempView('parquet_file_view')

scripts/data_generators/tests/simple_v3_table/__init__.py

@@ -0,0 +1,8 @@
+CREATE or REPLACE TABLE default.simple_v3_table
+TBLPROPERTIES (
+	'format-version' = '3',
+	'write.delete.mode' = 'merge-on-read',
+	'write.delete.format' = 'puffin',
+	'write.update.mode' = 'merge-on-read'
+)
+AS SELECT * FROM parquet_file_view;

scripts/data_generators/tests/simple_v3_table/q00.sql

@@ -0,0 +1 @@
+delete from default.simple_v3_table where c_custkey % 2 = 0;

scripts/data_generators/tests/simple_v3_table/q01.sql

@@ -376,6 +376,12 @@ PhysicalOperator &IRCatalog::PlanInsert(ClientContext &context, PhysicalPlanGene
 	}
 
 	auto &table_entry = op.table.Cast<ICTableEntry>();
+	// FIXME: Inserts into V3 tables is not yet supported since
+	// we need to keep track of row lineage, which we do not support
+	// https://iceberg.apache.org/spec/#row-lineage
+	if (table_entry.table_info.table_metadata.iceberg_version == 3) {
+		throw NotImplementedException("Insert into Iceberg V3 tables");
+	}
 	table_entry.PrepareIcebergScanFromEntry(context);
 	auto &table_info = table_entry.table_info;
 	auto &schema = table_info.table_metadata.GetLatestSchema();

src/storage/iceberg_insert.cpp

@@ -0,0 +1,65 @@
+# name: test/sql/local/irc/insert/test_insert_to_v3_tables.test
+# group: [insert]
+
+require-env DUCKDB_ICEBERG_HAVE_GENERATED_DATA
+
+require avro
+
+require parquet
+
+require iceberg
+
+require httpfs
+
+statement ok
+CREATE SECRET (
+    TYPE S3,
+    KEY_ID 'admin',
+    SECRET 'password',
+    ENDPOINT '127.0.0.1:9000',
+    URL_STYLE 'path',
+    USE_SSL 0
+);
+
+
+statement ok
+ATTACH '' AS my_datalake (
+    TYPE ICEBERG,
+    CLIENT_ID 'admin',
+    CLIENT_SECRET 'password',
+    ENDPOINT 'http://127.0.0.1:8181'
+);
+
+statement ok
+CALL enable_logging('Iceberg');
+
+query I
+select count(*) from my_datalake.default.simple_v3_table;
+----
+75000
+
+query I
+select count(*) from my_datalake.default.simple_v3_table where c_custkey > 75_000;
+----
+37500
+
+query I
+select count(*) from my_datalake.default.simple_v3_table where c_custkey > 50_000;
+----
+50_000
+
+query I
+select count(*) from my_datalake.default.simple_v3_table where c_custkey < 75_000;
+----
+37500
+
+query I
+select count(*) from my_datalake.default.simple_v3_table where c_custkey > 25_000 and c_custkey < 75_000;
+----
+25000
+
+statement error
+insert into my_datalake.default.simple_v3_table values (100000000, 'Customer#1000000000', 'jsdjsdffi', 4, 06-122-922-9374, 9281.24, 'FURNITURE', 'ducks are human beings');
+----
+<REGEX>:.*Not implemented Error: Insert into Iceberg V3 tables.*
+