Merge pull request #417 from Tmonster/add_snowflake_gcs_test

Add support for Polaris with GCS backed storage

Author: Tmonster

.github/workflows/CloudTesting.yml

@@ -65,9 +65,13 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{secrets.S3_ICEBERG_TEST_USER_KEY_ID}}
           AWS_SECRET_ACCESS_KEY: ${{secrets.S3_ICEBERG_TEST_USER_SECRET}}
           AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}}
+          SNOWFLAKE_KEY_ID_GCS: ${{secrets.SNOWFLAKE_KEY_ID_GCS}}
+          SNOWFLAKE_SECRET_KEY_GCS: ${{secrets.SNOWFLAKE_SECRET_KEY_GCS}}
+          SNOWFLAKE_CATALOG_URI_GCS: ${{secrets.SNOWFLAKE_CATALOG_URI}}
           R2_TOKEN: ${{secrets.r2_token}}
           ICEBERG_REMOTE_INSERT_READY: 1
           ICEBERG_AWS_REMOTE_AVAILABLE: 1
+          ICEBERG_SNOWFLAKE_REMOTE_AVAILABLE: 1
         run: |
           make test_release
 

extension_config.cmake

@@ -33,7 +33,8 @@ endif ()
 endif()
 
 duckdb_extension_load(httpfs
-        GIT_URL https://github.com/duckdb/duckdb-httpfs
-        GIT_TAG da2821906eb42f7255d969be3e073bc1b45a71a8
+	GIT_URL https://github.com/duckdb/duckdb-httpfs
+        GIT_TAG e9bb99189d93c8ce6e0755907c38d283c963ae61
         INCLUDE_DIR extension/httpfs/include
 )
+

src/storage/iceberg_table_information.cpp

@@ -15,8 +15,30 @@ const string &IcebergTableInformation::BaseFilePath() const {
 	return load_table_result.metadata.location;
 }
 
-static void ParseConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options) {
-	//! Set of recognized config parameters and the duckdb secret option that matches it.
+static string DetectStorageType(const string &location) {
+	// Detect storage type from the location URL
+	if (StringUtil::StartsWith(location, "gs://") || StringUtil::Contains(location, "storage.googleapis.com")) {
+		return "gcs";
+	} else if (StringUtil::StartsWith(location, "s3://") || StringUtil::StartsWith(location, "s3a://")) {
+		return "s3";
+	} else if (StringUtil::StartsWith(location, "abfs://") || StringUtil::StartsWith(location, "az://")) {
+		return "azure";
+	}
+	// Default to s3 for backward compatibility
+	return "s3";
+}
+
+static void ParseGCSConfigOptions(const case_insensitive_map_t<string> &config,
+                                  case_insensitive_map_t<Value> &options) {
+	// Parse GCS-specific configuration.
+	auto token_it = config.find("gcs.oauth2.token");
+	if (token_it != config.end()) {
+		options["bearer_token"] = token_it->second;
+	}
+}
+
+static void ParseS3ConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options) {
+	// Set of recognized S3 config parameters and the duckdb secret option that matches it.
 	static const case_insensitive_map_t<string> config_to_option = {{"s3.access-key-id", "key_id"},
 	                                                                {"s3.secret-access-key", "secret"},
 	                                                                {"s3.session-token", "session_token"},
@@ -25,15 +47,27 @@ static void ParseConfigOptions(const case_insensitive_map_t<string> &config, cas
 	                                                                {"client.region", "region"},
 	                                                                {"s3.endpoint", "endpoint"}};
 
-	if (config.empty()) {
-		return;
-	}
 	for (auto &entry : config) {
 		auto it = config_to_option.find(entry.first);
 		if (it != config_to_option.end()) {
 			options[it->second] = entry.second;
 		}
 	}
+}
+
+static void ParseConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options,
+                               const string &storage_type = "s3") {
+	if (config.empty()) {
+		return;
+	}
+
+	// Parse storage-specific config options
+	if (storage_type == "gcs") {
+		ParseGCSConfigOptions(config, options);
+	} else {
+		// Default to S3 parsing for backward compatibility
+		ParseS3ConfigOptions(config, options);
+	}
 
 	auto it = config.find("s3.path-style-access");
 	if (it != config.end()) {
@@ -105,19 +139,20 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
 		}
 	}
 
-	// Mapping from config key to a duckdb secret option
+	// Detect storage type from metadata location
+	const auto &metadata_location = load_table_result.metadata.location;
+	string storage_type = DetectStorageType(metadata_location);
 
+	// Mapping from config key to a duckdb secret option
 	case_insensitive_map_t<Value> config_options;
 	//! TODO: apply the 'defaults' retrieved from the /v1/config endpoint
 	config_options.insert(user_defaults.begin(), user_defaults.end());
 
 	if (load_table_result.has_config) {
 		auto &config = load_table_result.config;
-		ParseConfigOptions(config, config_options);
+		ParseConfigOptions(config, config_options, storage_type);
 	}
 
-	const auto &metadata_location = load_table_result.metadata.location;
-
 	if (load_table_result.has_storage_credentials) {
 		auto &storage_credentials = load_table_result.storage_credentials;
 
@@ -133,12 +168,12 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
 			create_secret_input.scope.push_back(ignore_credential_prefix ? metadata_location : credential.prefix);
 			create_secret_input.name = StringUtil::Format("%s_%d_%s", secret_base_name, index, credential.prefix);
 
-			create_secret_input.type = "s3";
+			create_secret_input.type = storage_type;
 			create_secret_input.provider = "config";
 			create_secret_input.storage_type = "memory";
 			create_secret_input.options = config_options;
 
-			ParseConfigOptions(credential.config, create_secret_input.options);
+			ParseConfigOptions(credential.config, create_secret_input.options, storage_type);
 			//! TODO: apply the 'overrides' retrieved from the /v1/config endpoint
 			result.storage_credentials.push_back(create_secret_input);
 		}
@@ -154,7 +189,7 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
 		//! TODO: apply the 'overrides' retrieved from the /v1/config endpoint
 		config.options = config_options;
 		config.name = secret_base_name;
-		config.type = "s3";
+		config.type = storage_type;
 		config.provider = "config";
 		config.storage_type = "memory";
 	}

test/sql/cloud/snowflake/test_snowflake.test

@@ -0,0 +1,56 @@
+# name: test/sql/cloud/snowflake/test_snowflake.test
+# description: test integration with iceberg catalog read
+# group: [snowflake]
+
+require-env ICEBERG_SNOWFLAKE_REMOTE_AVAILABLE
+
+require-env SNOWFLAKE_KEY_ID_GCS
+
+require-env SNOWFLAKE_SECRET_KEY_GCS
+
+require-env SNOWFLAKE_CATALOG_URI_GCS
+
+require avro
+
+require parquet
+
+require iceberg
+
+require httpfs
+
+require aws
+
+
+# Do not ignore 'HTTP' error messages!
+set ignore_error_messages
+
+
+statement ok
+create secret polaris_secret (
+	TYPE ICEBERG,
+	CLIENT_ID '${SNOWFLAKE_KEY_ID_GCS}',
+	CLIENT_SECRET '${SNOWFLAKE_SECRET_KEY_GCS}',
+	ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+);
+
+
+statement ok
+attach 'GCS_catalog' as my_datalake (
+	type ICEBERG,
+	ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
+);
+
+
+query I
+select * from my_datalake.default.duckdb_created_table;
+----
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9