Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion duckdb
Submodule duckdb updated 773 files
3 changes: 1 addition & 2 deletions extension_config.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ duckdb_extension_load(icu)
duckdb_extension_load(ducklake
LOAD_TESTS
GIT_URL https://github.com/duckdb/ducklake
GIT_TAG d2392c36f33151cf5cdd7d006375b0b669bd44ac
APPLY_PATCHES
GIT_TAG c1ebd032eb4c763910551c08f4b61bdb8168f209
)

duckdb_extension_load(avro
Expand Down
2 changes: 1 addition & 1 deletion scripts/data_generators/tests/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def generate(self, con: IcebergConnection):
intermediate_data_path = os.path.join(intermediate_dir, snapshot_name, 'data.parquet')
df.write.mode("overwrite").parquet(intermediate_data_path)

if self.write_intermediates:
if self.write_intermediates and last_file:
### Finally, copy the latest results to a "last" dir for easy test writing
shutil.copytree(
os.path.join(intermediate_dir, last_file, 'data.parquet'),
Expand Down
22 changes: 22 additions & 0 deletions scripts/data_generators/tests/simple_v3_table/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from scripts.data_generators.tests.base import IcebergTest
import pathlib
import tempfile
import duckdb


@IcebergTest.register()
class Test(IcebergTest):
def __init__(self):
path = pathlib.PurePath(__file__)
super().__init__(path.parent.name)

# Create a temporary directory
self.tempdir = pathlib.Path(tempfile.mkdtemp())
self.parquet_file = self.tempdir / "tmp.parquet"

duckdb_con = duckdb.connect()
duckdb_con.execute("call dbgen(sf=1)")
duckdb_con.execute(f"copy customer to '{self.parquet_file}' (FORMAT PARQUET)")

def setup(self, con):
con.con.read.parquet(self.parquet_file.as_posix()).createOrReplaceTempView('parquet_file_view')
8 changes: 8 additions & 0 deletions scripts/data_generators/tests/simple_v3_table/q00.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
CREATE or REPLACE TABLE default.simple_v3_table
TBLPROPERTIES (
'format-version' = '3',
'write.delete.mode' = 'merge-on-read',
'write.delete.format' = 'puffin',
'write.update.mode' = 'merge-on-read'
)
AS SELECT * FROM parquet_file_view;
1 change: 1 addition & 0 deletions scripts/data_generators/tests/simple_v3_table/q01.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
delete from default.simple_v3_table where c_custkey % 2 = 0;
6 changes: 6 additions & 0 deletions src/storage/iceberg_insert.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,12 @@ PhysicalOperator &IRCatalog::PlanInsert(ClientContext &context, PhysicalPlanGene
}

auto &table_entry = op.table.Cast<ICTableEntry>();
// FIXME: Inserts into V3 tables is not yet supported since
// we need to keep track of row lineage, which we do not support
// https://iceberg.apache.org/spec/#row-lineage
if (table_entry.table_info.table_metadata.iceberg_version == 3) {
throw NotImplementedException("Insert into Iceberg V3 tables");
}
table_entry.PrepareIcebergScanFromEntry(context);
auto &table_info = table_entry.table_info;
auto &schema = table_info.table_metadata.GetLatestSchema();
Expand Down
65 changes: 65 additions & 0 deletions test/sql/local/irc/insert/test_insert_to_v3_tables.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# name: test/sql/local/irc/insert/test_insert_to_v3_tables.test
# group: [insert]

require-env DUCKDB_ICEBERG_HAVE_GENERATED_DATA

require avro

require parquet

require iceberg

require httpfs

statement ok
CREATE SECRET (
TYPE S3,
KEY_ID 'admin',
SECRET 'password',
ENDPOINT '127.0.0.1:9000',
URL_STYLE 'path',
USE_SSL 0
);


statement ok
ATTACH '' AS my_datalake (
TYPE ICEBERG,
CLIENT_ID 'admin',
CLIENT_SECRET 'password',
ENDPOINT 'http://127.0.0.1:8181'
);

statement ok
CALL enable_logging('Iceberg');

query I
select count(*) from my_datalake.default.simple_v3_table;
----
75000

query I
select count(*) from my_datalake.default.simple_v3_table where c_custkey > 75_000;
----
37500

query I
select count(*) from my_datalake.default.simple_v3_table where c_custkey > 50_000;
----
50_000

query I
select count(*) from my_datalake.default.simple_v3_table where c_custkey < 75_000;
----
37500

query I
select count(*) from my_datalake.default.simple_v3_table where c_custkey > 25_000 and c_custkey < 75_000;
----
25000

statement error
insert into my_datalake.default.simple_v3_table values (100000000, 'Customer#1000000000', 'jsdjsdffi', 4, 06-122-922-9374, 9281.24, 'FURNITURE', 'ducks are human beings');
----
<REGEX>:.*Not implemented Error: Insert into Iceberg V3 tables.*

Loading