Skip to content

Commit caca3ac

Browse files
TmonsterTmonster
authored
Merge pull request #431 from carlopi/merge_wasm_with_main
Merge back main into wasm branch
2 parents 5343bd6 + 74173a4 commit caca3ac

File tree

9 files changed

+122
-18
lines changed

9 files changed

+122
-18
lines changed

‎.github/workflows/CloudTesting.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,13 @@ jobs:
6565
AWS_ACCESS_KEY_ID: ${{secrets.S3_ICEBERG_TEST_USER_KEY_ID}}
6666
AWS_SECRET_ACCESS_KEY: ${{secrets.S3_ICEBERG_TEST_USER_SECRET}}
6767
AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}}
68+
SNOWFLAKE_KEY_ID_GCS: ${{secrets.SNOWFLAKE_KEY_ID_GCS}}
69+
SNOWFLAKE_SECRET_KEY_GCS: ${{secrets.SNOWFLAKE_SECRET_KEY_GCS}}
70+
SNOWFLAKE_CATALOG_URI_GCS: ${{secrets.SNOWFLAKE_CATALOG_URI_GCS}}
6871
R2_TOKEN: ${{secrets.r2_token}}
6972
ICEBERG_REMOTE_INSERT_READY: 1
7073
ICEBERG_AWS_REMOTE_AVAILABLE: 1
74+
ICEBERG_SNOWFLAKE_REMOTE_AVAILABLE: 1
7175
run: |
7276
make test_release
7377

‎extension_config.cmake‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,5 @@
44
duckdb_extension_load(iceberg
55
SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
66
LOAD_TESTS
7-
LINKED_LIBS "../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-mqtt.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-kinesis.a;../../vcpkg_installed/wasm32-emscripten/lib/liblzma.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-auth.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-s3.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-s3.a;../../vcpkg_installed/wasm32-emscripten/lib/libroaring.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-cal.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-sdkutils.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-sso.a;../../vcpkg_installed/wasm32-emscripten/lib/libs2n.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-common.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-checksums.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-sts.a;../../vcpkg_installed/wasm32-emscripten/lib/libsnappy.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-compression.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-cognito-identity.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-crt-cpp.a;../../vcpkg_installed/wasm32-emscripten/lib/libssl.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-event-stream.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-core.a;../../vcpkg_installed/wasm32-emscripten/lib/libcrypto.a;../../vcpkg_installed/wasm32-emscripten/lib/libz.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-http.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-dynamodb.a;../../vcpkg_installed/wasm32-emscripten/lib/libcurl.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-io.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-identity-management.a;../../vcpkg_installed/wasm32-emscripten/lib/libjansson.a;../../third_party/mbedtls/libduckdb_mbedtls.a"
7+
LINKED_LIBS "../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-mqtt.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-kinesis.a;../../vcpkg_installed/wasm32-emscripten/lib/liblzma.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-auth.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-s3.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-s3.a;../../vcpkg_installed/wasm32-emscripten/lib/libroaring.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-cal.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-sdkutils.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-sso.a;../../vcpkg_installed/wasm32-emscripten/lib/libs2n.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-common.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-checksums.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-sts.a;../../vcpkg_installed/wasm32-emscripten/lib/libsnappy.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-compression.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-cognito-identity.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-crt-cpp.a;../../vcpkg_installed/wasm32-emscripten/lib/libssl.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-event-stream.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-core.a;../../vcpkg_installed/wasm32-emscripten/lib/libcrypto.a;../../vcpkg_installed/wasm32-emscripten/lib/libz.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-http.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-dynamodb.a;../../vcpkg_installed/wasm32-emscripten/lib/libcurl.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-c-io.a;../../vcpkg_installed/wasm32-emscripten/lib/libaws-cpp-sdk-identity-management.a;../../vcpkg_installed/wasm32-emscripten/lib/libjansson.a;../../third_party/mbedtls/libduckdb_mbedtls.a"
88
)
9-

‎src/common/api_utils.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ unique_ptr<HTTPResponse> APIUtils::DeleteRequest(ClientContext &context, const I
5050
}
5151

5252
unique_ptr<HTTPResponse> APIUtils::PostRequest(ClientContext &context, const string &url, const string &post_data,
53+
const unordered_map<string, string> &additional_headers,
5354
const string &content_type, const string &token) {
5455
auto &db = DatabaseInstance::GetDatabase(context);
5556

@@ -58,6 +59,9 @@ unique_ptr<HTTPResponse> APIUtils::PostRequest(ClientContext &context, const str
5859
HTTPHeaders headers(db);
5960
headers.Insert("X-Iceberg-Access-Delegation", "vended-credentials");
6061
headers.Insert("Content-Type", StringUtil::Format("application/%s", content_type));
62+
for (auto it : additional_headers) {
63+
headers.Insert(it.first, it.second);
64+
}
6165

6266
if (!token.empty()) {
6367
headers.Insert("Authorization", StringUtil::Format("Bearer %s", token));

‎src/include/api_utils.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class APIUtils {
3838
static unique_ptr<HTTPResponse> DeleteRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder,
3939
const string &token = "");
4040
static unique_ptr<HTTPResponse> PostRequest(ClientContext &context, const string &url, const string &post_data,
41+
const unordered_map<string, string> &additional_headers,
4142
const string &content_type = "x-www-form-urlencoded",
4243
const string &token = "");
4344
//! We use a singleton here to store the path, set by SelectCurlCertPath

‎src/storage/authorization/none.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ unique_ptr<HTTPResponse> NoneAuthorization::DeleteRequest(ClientContext &context
2525
unique_ptr<HTTPResponse>
2626
NoneAuthorization::PostRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &body) {
2727
auto url = endpoint_builder.GetURL();
28-
return APIUtils::PostRequest(context, url, body, "json", "");
28+
unordered_map<string, string> empty_headers;
29+
return APIUtils::PostRequest(context, url, body, empty_headers, "json", "");
2930
}
3031

3132
} // namespace duckdb

‎src/storage/authorization/oauth2.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,13 @@ string OAuth2Authorization::GetToken(ClientContext &context, const string &grant
6666
string credentials = StringUtil::Format("%s:%s", client_id, client_secret);
6767
string_t credentials_blob(credentials.data(), credentials.size());
6868

69+
unordered_map<string, string> headers;
70+
headers.emplace("Authorization", StringUtil::Format("Basic %s", Blob::ToBase64(credentials_blob)));
71+
6972
string post_data = StringUtil::Format("%s", StringUtil::Join(parameters, "&"));
7073
std::unique_ptr<yyjson_doc, YyjsonDocDeleter> doc;
7174
try {
72-
auto response = APIUtils::PostRequest(context, uri, post_data);
75+
auto response = APIUtils::PostRequest(context, uri, post_data, headers);
7376
doc = std::unique_ptr<yyjson_doc, YyjsonDocDeleter>(ICUtils::api_result_to_doc(response->body));
7477
} catch (std::exception &ex) {
7578
ErrorData error(ex);
@@ -110,7 +113,8 @@ unique_ptr<HTTPResponse> OAuth2Authorization::PostRequest(ClientContext &context
110113
const IRCEndpointBuilder &endpoint_builder,
111114
const string &body) {
112115
auto url = endpoint_builder.GetURL();
113-
return APIUtils::PostRequest(context, url, body, "json", token);
116+
unordered_map<string, string> empty_headers;
117+
return APIUtils::PostRequest(context, url, body, empty_headers, "json", token);
114118
}
115119

116120
unique_ptr<OAuth2Authorization> OAuth2Authorization::FromAttachOptions(ClientContext &context,

‎src/storage/iceberg_table_information.cpp‎

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,30 @@ const string &IcebergTableInformation::BaseFilePath() const {
1515
return load_table_result.metadata.location;
1616
}
1717

18-
static void ParseConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options) {
19-
//! Set of recognized config parameters and the duckdb secret option that matches it.
18+
static string DetectStorageType(const string &location) {
19+
// Detect storage type from the location URL
20+
if (StringUtil::StartsWith(location, "gs://") || StringUtil::Contains(location, "storage.googleapis.com")) {
21+
return "gcs";
22+
} else if (StringUtil::StartsWith(location, "s3://") || StringUtil::StartsWith(location, "s3a://")) {
23+
return "s3";
24+
} else if (StringUtil::StartsWith(location, "abfs://") || StringUtil::StartsWith(location, "az://")) {
25+
return "azure";
26+
}
27+
// Default to s3 for backward compatibility
28+
return "s3";
29+
}
30+
31+
static void ParseGCSConfigOptions(const case_insensitive_map_t<string> &config,
32+
case_insensitive_map_t<Value> &options) {
33+
// Parse GCS-specific configuration.
34+
auto token_it = config.find("gcs.oauth2.token");
35+
if (token_it != config.end()) {
36+
options["bearer_token"] = token_it->second;
37+
}
38+
}
39+
40+
static void ParseS3ConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options) {
41+
// Set of recognized S3 config parameters and the duckdb secret option that matches it.
2042
static const case_insensitive_map_t<string> config_to_option = {{"s3.access-key-id", "key_id"},
2143
{"s3.secret-access-key", "secret"},
2244
{"s3.session-token", "session_token"},
@@ -25,15 +47,27 @@ static void ParseConfigOptions(const case_insensitive_map_t<string> &config, cas
2547
{"client.region", "region"},
2648
{"s3.endpoint", "endpoint"}};
2749

28-
if (config.empty()) {
29-
return;
30-
}
3150
for (auto &entry : config) {
3251
auto it = config_to_option.find(entry.first);
3352
if (it != config_to_option.end()) {
3453
options[it->second] = entry.second;
3554
}
3655
}
56+
}
57+
58+
static void ParseConfigOptions(const case_insensitive_map_t<string> &config, case_insensitive_map_t<Value> &options,
59+
const string &storage_type = "s3") {
60+
if (config.empty()) {
61+
return;
62+
}
63+
64+
// Parse storage-specific config options
65+
if (storage_type == "gcs") {
66+
ParseGCSConfigOptions(config, options);
67+
} else {
68+
// Default to S3 parsing for backward compatibility
69+
ParseS3ConfigOptions(config, options);
70+
}
3771

3872
auto it = config.find("s3.path-style-access");
3973
if (it != config.end()) {
@@ -105,19 +139,20 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
105139
}
106140
}
107141

108-
// Mapping from config key to a duckdb secret option
142+
// Detect storage type from metadata location
143+
const auto &metadata_location = load_table_result.metadata.location;
144+
string storage_type = DetectStorageType(metadata_location);
109145

146+
// Mapping from config key to a duckdb secret option
110147
case_insensitive_map_t<Value> config_options;
111148
//! TODO: apply the 'defaults' retrieved from the /v1/config endpoint
112149
config_options.insert(user_defaults.begin(), user_defaults.end());
113150

114151
if (load_table_result.has_config) {
115152
auto &config = load_table_result.config;
116-
ParseConfigOptions(config, config_options);
153+
ParseConfigOptions(config, config_options, storage_type);
117154
}
118155

119-
const auto &metadata_location = load_table_result.metadata.location;
120-
121156
if (load_table_result.has_storage_credentials) {
122157
auto &storage_credentials = load_table_result.storage_credentials;
123158

@@ -133,12 +168,12 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
133168
create_secret_input.scope.push_back(ignore_credential_prefix ? metadata_location : credential.prefix);
134169
create_secret_input.name = StringUtil::Format("%s_%d_%s", secret_base_name, index, credential.prefix);
135170

136-
create_secret_input.type = "s3";
171+
create_secret_input.type = storage_type;
137172
create_secret_input.provider = "config";
138173
create_secret_input.storage_type = "memory";
139174
create_secret_input.options = config_options;
140175

141-
ParseConfigOptions(credential.config, create_secret_input.options);
176+
ParseConfigOptions(credential.config, create_secret_input.options, storage_type);
142177
//! TODO: apply the 'overrides' retrieved from the /v1/config endpoint
143178
result.storage_credentials.push_back(create_secret_input);
144179
}
@@ -154,7 +189,7 @@ IRCAPITableCredentials IcebergTableInformation::GetVendedCredentials(ClientConte
154189
//! TODO: apply the 'overrides' retrieved from the /v1/config endpoint
155190
config.options = config_options;
156191
config.name = secret_base_name;
157-
config.type = "s3";
192+
config.type = storage_type;
158193
config.provider = "config";
159194
config.storage_type = "memory";
160195
}

‎test/sql/cloud/snowflake/test_snowflake.test‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# name: test/sql/cloud/snowflake/test_snowflake.test
2+
# description: test integration with iceberg catalog read
3+
# group: [snowflake]
4+
5+
require-env ICEBERG_SNOWFLAKE_REMOTE_AVAILABLE
6+
7+
require-env SNOWFLAKE_KEY_ID_GCS
8+
9+
require-env SNOWFLAKE_SECRET_KEY_GCS
10+
11+
require-env SNOWFLAKE_CATALOG_URI_GCS
12+
13+
require avro
14+
15+
require parquet
16+
17+
require iceberg
18+
19+
require httpfs
20+
21+
require aws
22+
23+
24+
# Do not ignore 'HTTP' error messages!
25+
set ignore_error_messages
26+
27+
28+
statement ok
29+
create secret polaris_secret (
30+
TYPE ICEBERG,
31+
CLIENT_ID '${SNOWFLAKE_KEY_ID_GCS}',
32+
CLIENT_SECRET '${SNOWFLAKE_SECRET_KEY_GCS}',
33+
ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
34+
);
35+
36+
37+
statement ok
38+
attach 'GCS_catalog' as my_datalake (
39+
type ICEBERG,
40+
ENDPOINT '${SNOWFLAKE_CATALOG_URI_GCS}'
41+
);
42+
43+
44+
query I
45+
select * from my_datalake.default.duckdb_created_table;
46+
----
47+
0
48+
1
49+
2
50+
3
51+
4
52+
5
53+
6
54+
7
55+
8
56+
9

‎vcpkg.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"dependencies": [
33
"vcpkg-cmake",
44
"avro-c",
5-
"curl","openssl",
5+
"curl",
66
"roaring",
77
{
88
"name": "aws-sdk-cpp",

0 commit comments

Comments
 (0)