Merge pull request #435 from Tmonster/v1.3-and-main

Update v1.3 with main code

Author: Tmonster

.github/workflows/CloudTesting.yml

@@ -58,21 +58,27 @@ jobs:
           AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}}
           R2_TOKEN: ${{secrets.r2_token}}
         run: | 
-          python3 scripts/create_s3_insert_table.py --action=delete-and-create --catalogs=s3tables,r2
+          python3 scripts/create_s3_insert_table.py --action=delete-and-create --catalogs=s3tables,r2,glue
 
       - name: Test with rest catalog
         env:
           AWS_ACCESS_KEY_ID: ${{secrets.S3_ICEBERG_TEST_USER_KEY_ID}}
           AWS_SECRET_ACCESS_KEY: ${{secrets.S3_ICEBERG_TEST_USER_SECRET}}
           AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}}
+          SNOWFLAKE_KEY_ID_GCS: ${{secrets.SNOWFLAKE_KEY_ID_GCS}}
+          SNOWFLAKE_SECRET_KEY_GCS: ${{secrets.SNOWFLAKE_SECRET_KEY_GCS}}
+          SNOWFLAKE_KEY_ID_S3: ${{secrets.SNOWFLAKE_KEY_ID_S3}}
+          SNOWFLAKE_SECRET_KEY_S3: ${{secrets.SNOWFLAKE_SECRET_KEY_S3}}
+          SNOWFLAKE_CATALOG_URI_GCS: ${{secrets.SNOWFLAKE_CATALOG_URI_GCS}}
           R2_TOKEN: ${{secrets.r2_token}}
           ICEBERG_REMOTE_INSERT_READY: 1
           ICEBERG_AWS_REMOTE_AVAILABLE: 1
+          ICEBERG_SNOWFLAKE_REMOTE_AVAILABLE: 1
         run: |
           make test_release
 
       - name: File issue if error
-        if: failure()
+        if: ${{ contains(github.ref_name, 'main') && failure() }}
         env:
           GH_TOKEN: ${{ github.token }}
         run: |

duckdb

@@ -33,7 +33,8 @@ endif ()
 endif()
 
 duckdb_extension_load(httpfs
-        GIT_URL https://github.com/duckdb/duckdb-httpfs
-        GIT_TAG da2821906eb42f7255d969be3e073bc1b45a71a8
+	GIT_URL https://github.com/duckdb/duckdb-httpfs
+        GIT_TAG e9bb99189d93c8ce6e0755907c38d283c963ae61
         INCLUDE_DIR extension/httpfs/include
 )
+

extension_config.cmake

@@ -15,8 +15,30 @@ const string &IcebergTableInformation::BaseFilePath() const {
 	return load_table_result.metadata.location;
 }
 
-static void ParseConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options) {
-	//! Set of recognized config parameters and the duckdb secret option that matches it.
+static string DetectStorageType(const string &location) {
+	// Detect storage type from the location URL
+	if (StringUtil::StartsWith(location, "gs://") || StringUtil::Contains(location, "storage.googleapis.com")) {
+		return "gcs";
+	} else if (StringUtil::StartsWith(location, "s3://") || StringUtil::StartsWith(location, "s3a://")) {
+		return "s3";
+	} else if (StringUtil::StartsWith(location, "abfs://") || StringUtil::StartsWith(location, "az://")) {
+		return "azure";
+	}
+	// Default to s3 for backward compatibility
+	return "s3";
+}
+
+static void ParseGCSConfigOptions(const case_insensitive_map_t<string> &config,
+                                  case_insensitive_map_t<Value> &options) {
+	// Parse GCS-specific configuration.
+	auto token_it = config.find("gcs.oauth2.token");
+	if (token_it != config.end()) {
+		options["bearer_token"] = token_it->second;
+	}
+}
+
+static void ParseS3ConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options) {
+	// Set of recognized S3 config parameters and the duckdb secret option that matches it.
 	static const case_insensitive_map_t<string> config_to_option = {{"s3.access-key-id", "key_id"},
 	                                                                {"s3.secret-access-key", "secret"},
 	                                                                {"s3.session-token", "session_token"},
@@ -25,15 +47,27 @@ static void ParseConfigOptions(const case_insensitive_map_t<string> &config, cas
 	                                                                {"client.region", "region"},
 	                                                                {"s3.endpoint", "endpoint"}};
 
-	if (config.empty()) {
-		return;
-	}
 	for (auto &entry : config) {
 		auto it = config_to_option.find(entry.first);
 		if (it != config_to_option.end()) {
 			options[it->second] = entry.second;
 		}
 	}
+}
+
+static void ParseConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options,
+                               const string &storage_type = "s3") {
+	if (config.empty()) {
+		return;
+	}
+
+	// Parse storage-specific config options
+	if (storage_type == "gcs") {
+		ParseGCSConfigOptions(config, options);
+	} else {
+		// Default to S3 parsing for backward compatibility
+		ParseS3ConfigOptions(config, options);
+	}
 
 	auto it = config.find("s3.path-style-access");
 	if (it != config.end()) {
@@ -105,19 +139,20 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
 		}
 	}
 
-	// Mapping from config key to a duckdb secret option
+	// Detect storage type from metadata location
+	const auto &metadata_location = load_table_result.metadata.location;
+	string storage_type = DetectStorageType(metadata_location);
 
+	// Mapping from config key to a duckdb secret option
 	case_insensitive_map_t<Value> config_options;
 	//! TODO: apply the 'defaults' retrieved from the /v1/config endpoint
 	config_options.insert(user_defaults.begin(), user_defaults.end());
 
 	if (load_table_result.has_config) {
 		auto &config = load_table_result.config;
-		ParseConfigOptions(config, config_options);
+		ParseConfigOptions(config, config_options, storage_type);
 	}
 
-	const auto &metadata_location = load_table_result.metadata.location;
-
 	if (load_table_result.has_storage_credentials) {
 		auto &storage_credentials = load_table_result.storage_credentials;
 
@@ -133,12 +168,12 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
 			create_secret_input.scope.push_back(ignore_credential_prefix ? metadata_location : credential.prefix);
 			create_secret_input.name = StringUtil::Format("%s_%d_%s", secret_base_name, index, credential.prefix);
 
-			create_secret_input.type = "s3";
+			create_secret_input.type = storage_type;
 			create_secret_input.provider = "config";
 			create_secret_input.storage_type = "memory";
 			create_secret_input.options = config_options;
 
-			ParseConfigOptions(credential.config, create_secret_input.options);
+			ParseConfigOptions(credential.config, create_secret_input.options, storage_type);
 			//! TODO: apply the 'overrides' retrieved from the /v1/config endpoint
 			result.storage_credentials.push_back(create_secret_input);
 		}
@@ -154,7 +189,7 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
 		//! TODO: apply the 'overrides' retrieved from the /v1/config endpoint
 		config.options = config_options;
 		config.name = secret_base_name;
-		config.type = "s3";
+		config.type = storage_type;
 		config.provider = "config";
 		config.storage_type = "memory";
 	}

src/storage/iceberg_table_information.cpp

@@ -18,10 +18,6 @@ require httpfs
 
 require aws
 
-# credentials in CI cannot set up the environment for this test
-# need to give the crentials glue:DropTable priviledges
-mode skip
-
 statement ok
 CREATE SECRET (
   TYPE S3,

test/sql/cloud/glue/test_create_table_glue.test

@@ -18,11 +18,6 @@ require httpfs
 
 require aws
 
-# TODO: re-enable these tests once we know what account has these
-#  credentials, and we can grant them access to the glue catalog
-# test using keys directory
-mode skip
-
 statement ok
 CREATE SECRET s1 (
       TYPE S3,
@@ -37,10 +32,6 @@ attach '840140254803:s3tablescatalog/pyiceberg-blog-bucket' as my_datalake (
     ENDPOINT_TYPE 'GLUE'
 );
 
-query T nosort tables_1
-show all tables;
-----
-
 statement ok
 SELECT count(*) FROM my_datalake.myblognamespace.lineitem;
 

test/sql/cloud/glue/test_direct_keys_glue.test

@@ -18,10 +18,6 @@ require httpfs
 
 require aws
 
-# TODO: re-enable these tests once we know what account has these
-#  credentials, and we can grant them access to the glue catalog
-mode skip
-
 # test using keys directory
 statement ok
 CREATE SECRET s1 (
@@ -38,10 +34,6 @@ attach '840140254803:s3tablescatalog/pyiceberg-blog-bucket' as my_datalake (
 	ENDPOINT 'glue.us-east-1.amazonaws.com/iceberg'
 );
 
-query T nosort tables_1
-show all tables;
-----
-
 statement ok
 SELECT count(*) FROM my_datalake.myblognamespace.lineitem;
 
@@ -51,8 +43,6 @@ drop secret s1;
 statement ok
 detach my_datalake;
 
-mode unskip
-
 # test using assume role
 statement ok
 CREATE SECRET assume_role_secret (

test/sql/cloud/glue/test_direct_keys_glue_no_endpoint_type.test

@@ -20,10 +20,6 @@ require httpfs
 
 require aws
 
-# credentials in CI cannot set up the environment for this test
-# need to give the crentials glue:DropTable priviledges
-mode skip
-
 statement ok
 CREATE SECRET (
     TYPE S3,

test/sql/cloud/glue/test_insert_glue.test

@@ -0,0 +1,67 @@
+# name: test/sql/cloud/snowflake/test_snowflake.test
+# description: test integration with iceberg catalog read
+# group: [snowflake]
+
+require-env ICEBERG_SNOWFLAKE_REMOTE_AVAILABLE
+
+require-env SNOWFLAKE_KEY_ID_GCS
+
+require-env SNOWFLAKE_SECRET_KEY_GCS
+
+require-env SNOWFLAKE_CATALOG_URI_GCS
+
+require avro
+
+require parquet
+
+require iceberg
+
+require httpfs
+
+require aws
+
+
+# Do not ignore 'HTTP' error messages!
+set ignore_error_messages
+
+
+statement ok
+create secret polaris_secret (
+	TYPE ICEBERG,
+	CLIENT_ID '${SNOWFLAKE_KEY_ID_GCS}',
+	CLIENT_SECRET '${SNOWFLAKE_SECRET_KEY_GCS}',
+	ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+);
+
+
+statement ok
+attach 'GCS_catalog' as my_datalake (
+	type ICEBERG,
+	ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+);
+
+statement ok
+create schema if not exists my_datalake.test_create_schema;
+
+statement ok
+create table my_datalake.test_create_schema.table1 as select range a from range(10);
+
+query I
+select * from my_datalake.test_create_schema.table1;
+----
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+statement ok
+drop table if exists my_datalake.test_create_schema.table1;
+
+statement ok
+drop schema if exists my_datalake.test_create_schema;

test/sql/cloud/snowflake/test_snowflake.test

@@ -0,0 +1,64 @@
+# name: test/sql/cloud/snowflake/test_snowflake_s3.test
+# group: [snowflake]
+
+require-env SNOWFLAKE_KEY_ID_S3
+
+require-env SNOWFLAKE_SECRET_ID_S3
+
+require-env SNOWFLAKE_CATALOG_URI_GCS
+
+require avro
+
+require parquet
+
+require iceberg
+
+require httpfs
+
+require aws
+
+# Do not ignore 'HTTP' error messages!
+set ignore_error_messages
+
+statement ok
+create secret polaris_secret (
+        TYPE ICEBERG,
+        CLIENT_ID '${SNOWFLAKE_KEY_ID_S3}',
+        CLIENT_SECRET '${SNOWFLAKE_SECRET_KEY_S3}',
+        ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+    );
+
+
+statement ok
+attach 's3-catalog' as my_datalake (
+    type ICEBERG,
+    default_region 'eu-west-2',
+    ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+ );
+
+
+statement ok
+create schema if not exists my_datalake.test_create_schema;
+
+statement ok
+create table my_datalake.test_create_schema.table1 as select range a from range(10);
+
+query I
+select * from my_datalake.test_create_schema.table1;
+----
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+statement ok
+drop table if exists my_datalake.test_create_schema.table1;
+
+statement ok
+drop schema if exists my_datalake.test_create_schema;