From 406ec2e972db929207e8f065ffb414c103d1b9ef Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 25 Mar 2025 21:01:48 +0100 Subject: [PATCH 01/66] let the warehouse and scope be provided through the catalog parameters of the IRC --- src/catalog_api.cpp | 16 +++++++--- src/catalog_utils.cpp | 4 +-- src/common/url_utils.cpp | 38 +++++++++++++++------- src/iceberg_extension.cpp | 29 +++++++++++++---- src/include/catalog_api.hpp | 2 +- src/include/catalog_utils.hpp | 2 +- src/include/storage/irc_catalog.hpp | 12 ++++--- src/include/url_utils.hpp | 49 ++++++++++++++++++----------- src/storage/irc_catalog.cpp | 2 +- src/storage/irc_table_entry.cpp | 2 +- 10 files changed, 106 insertions(+), 50 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 4579a5b2..7a9e6b5d 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -365,10 +365,18 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat return result; } -string IRCAPI::GetToken(ClientContext &context, string id, string secret, string endpoint) { - string post_data = "grant_type=client_credentials&client_id=" + id + "&client_secret=" + secret + "&scope=PRINCIPAL_ROLE:ALL"; - string api_result = PostRequest(context, endpoint + "/v1/oauth/tokens", post_data); - std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); +string IRCAPI::GetToken(ClientContext &context, const string &id, const string &secret, const string &endpoint, const string &scope) { + vector parameters; + parameters.push_back("client_credentials"); + parameters.push_back(StringUtil::Format("%s=%s", "client_id", id)); + parameters.push_back(StringUtil::Format("%s=%s", "client_secret", secret)); + parameters.push_back(StringUtil::Format("%s=%s", "scope", scope)); + + string post_data = StringUtil::Format("grant_type=%s", StringUtil::Join(parameters, "&")); + string api_result = PostRequest(context, "http://localhost:30080/realms/iceberg/protocol/openid-connect/token", post_data); + //! FIXME: the oauth/tokens endpoint returns, on success; + // { 'access_token', 'token_type', 'expires_in', , 'refresh_token', 'scope'} + std::unique_ptr doc(ICUtils::api_result_to_doc(api_result, "access_token")); auto *root = yyjson_doc_get_root(doc.get()); return IcebergUtils::TryGetStrFromObject(root, "access_token"); } diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index 50e3dffe..be6d6b2b 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -234,12 +234,12 @@ LogicalType ICUtils::ToICType(const LogicalType &input) { } } -yyjson_doc *ICUtils::api_result_to_doc(const string &api_result) { +yyjson_doc *ICUtils::api_result_to_doc(const string &api_result, const string &variable) { auto *doc = yyjson_read(api_result.c_str(), api_result.size(), 0); auto *root = yyjson_doc_get_root(doc); auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { - string err_msg = IcebergUtils::TryGetStrFromObject(error, "message"); + string err_msg = IcebergUtils::TryGetStrFromObject(error, variable); throw std::runtime_error(err_msg); } return doc; diff --git a/src/common/url_utils.cpp b/src/common/url_utils.cpp index 32b26e60..bf7e83b1 100644 --- a/src/common/url_utils.cpp +++ b/src/common/url_utils.cpp @@ -1,46 +1,52 @@ #include "url_utils.hpp" +#include "duckdb/common/string_util.hpp" + namespace duckdb { -void IRCEndpointBuilder::AddPathComponent(std::string component) { +void IRCEndpointBuilder::AddPathComponent(const string &component) { path_components.push_back(component); } -void IRCEndpointBuilder::SetPrefix(std::string prefix_) { +void IRCEndpointBuilder::AddQueryParameter(const string &key, const string &value) { + query_parameters.emplace_back(key, value); +} + +void IRCEndpointBuilder::SetPrefix(const string &prefix_) { prefix = prefix_; } -std::string IRCEndpointBuilder::GetHost() const { +string IRCEndpointBuilder::GetHost() const { return host; } -void IRCEndpointBuilder::SetVersion(std::string version_) { +void IRCEndpointBuilder::SetVersion(const string &version_) { version = version_; } -std::string IRCEndpointBuilder::GetVersion() const { +string IRCEndpointBuilder::GetVersion() const { return version; } -void IRCEndpointBuilder::SetWarehouse(std::string warehouse_) { +void IRCEndpointBuilder::SetWarehouse(const string &warehouse_) { warehouse = warehouse_; } -std::string IRCEndpointBuilder::GetWarehouse() const { +string IRCEndpointBuilder::GetWarehouse() const { return warehouse; } -void IRCEndpointBuilder::SetHost(std::string host_) { +void IRCEndpointBuilder::SetHost(const string &host_) { host = host_; } -std::string IRCEndpointBuilder::GetPrefix() const { +string IRCEndpointBuilder::GetPrefix() const { return prefix; } -std::string IRCEndpointBuilder::GetURL() const { - std::string ret = host; +string IRCEndpointBuilder::GetURL() const { + string ret = host; if (!version.empty()) { ret = ret + "/" + version; } @@ -54,7 +60,15 @@ std::string IRCEndpointBuilder::GetURL() const { for (auto &component : path_components) { ret += "/" + component; } + if (!query_parameters.empty()) { + ret += "?"; + vector parameters; + for (auto &query_parameter : query_parameters) { + parameters.push_back(StringUtil::Format("%s=%s", query_parameter.key, query_parameter.value)); + ret += StringUtil::Join(parameters, "&"); + } + } return ret; } -} \ No newline at end of file +} diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 5e36c3cb..7e233d98 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -32,7 +32,8 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context if (lower_name == "key_id" || lower_name == "secret" || lower_name == "endpoint" || - lower_name == "aws_region") { + lower_name == "aws_region" || + lower_name == "scope") { result->secret_map[lower_name] = named_param.second.ToString(); } else { throw InternalException("Unknown named parameter passed to CreateIRCSecretFunction: " + lower_name); @@ -44,7 +45,9 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context context, result->secret_map["key_id"].ToString(), result->secret_map["secret"].ToString(), - result->secret_map["endpoint"].ToString()); + result->secret_map["endpoint"].ToString(), + result->secret_map["scope"].ToString() + ); //! Set redact keys result->redact_keys = {"token", "client_id", "client_secret"}; @@ -86,6 +89,9 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in string endpoint_type; string endpoint; + auto &warehouse = credentials.warehouse; + auto &scope = credentials.scope; + // check if we have a secret provided string secret_name; for (auto &entry : info.options) { @@ -99,11 +105,23 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in } else if (lower_name == "endpoint") { endpoint = StringUtil::Lower(entry.second.ToString()); StringUtil::RTrim(endpoint, "/"); + } else if (lower_name == "warehouse") { + warehouse = StringUtil::Lower(entry.second.ToString()); + } else if (lower_name == "scope") { + scope = StringUtil::Lower(entry.second.ToString()); } else { throw BinderException("Unrecognized option for PC attach: %s", entry.first); } } - auto warehouse = info.path; + if (warehouse.empty()) { + //! Default to the path of the catalog as the warehouse if none is provided + warehouse = info.path; + } + + if (scope.empty()) { + //! Default to the Polaris scope: 'PRINCIPAL_ROLE:ALL' + scope = "PRINCIPAL_ROLE:ALL"; + } if (endpoint_type == "glue" || endpoint_type == "s3_tables") { if (endpoint_type == "s3_tables") { @@ -126,11 +144,10 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in throw InvalidInputException("Could not parse S3 Tables arn warehouse value"); } region = Value::CreateValue(substrings[3]); - catalog->warehouse = warehouse; } else if (service == "glue") { SanityCheckGlueWarehouse(warehouse); - catalog->warehouse = StringUtil::Replace(warehouse, "/", ":"); + warehouse = StringUtil::Replace(warehouse, "/", ":"); } catalog->host = service + "." + region.ToString() + ".amazonaws.com"; @@ -162,6 +179,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in create_secret_input.options["key_id"] = key_val; create_secret_input.options["secret"] = secret_val; create_secret_input.options["endpoint"] = endpoint; + create_secret_input.options["scope"] = scope; auto new_secret = CreateCatalogSecretFunction(context, create_secret_input); auto &kv_secret_new = dynamic_cast(*new_secret); Value token = kv_secret_new.TryGetValue("token"); @@ -171,7 +189,6 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in credentials.token = token.ToString(); auto catalog = make_uniq(db, access_mode, credentials); catalog->host = endpoint; - catalog->warehouse = warehouse; catalog->version = "v1"; catalog->secret_name = secret_name; return std::move(catalog); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 76d720fe..6be93280 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -56,7 +56,7 @@ class IRCAPI { static IRCAPITable GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table_name, optional_ptr credentials = nullptr); static vector GetSchemas(ClientContext &context, IRCatalog &catalog, IRCCredentials credentials); static vector GetTablesInSchema(ClientContext &context, IRCatalog &catalog, const string &schema, IRCCredentials credentials); - static string GetToken(ClientContext &context, string id, string secret, string endpoint); + static string GetToken(ClientContext &context, const string &id, const string &secret, const string &endpoint, const string &scope); static IRCAPISchema CreateSchema(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials); static void DropSchema(ClientContext &context, const string &internal, const string &schema, IRCCredentials credentials); static IRCAPITable CreateTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials, CreateTableInfo *table_info); diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index e6475654..af5e6988 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -24,7 +24,7 @@ class ICUtils { static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); static string TypeToString(const LogicalType &input); static string LogicalToIcebergType(const LogicalType &input); - static yyjson_doc *api_result_to_doc(const string &api_result); + static yyjson_doc *api_result_to_doc(const string &api_result, const string &variable = "message"); }; struct YyjsonDocDeleter { diff --git a/src/include/storage/irc_catalog.hpp b/src/include/storage/irc_catalog.hpp index d39cff84..40b52874 100644 --- a/src/include/storage/irc_catalog.hpp +++ b/src/include/storage/irc_catalog.hpp @@ -17,10 +17,16 @@ class IRCSchemaEntry; struct IRCCredentials { string client_id; string client_secret; - // required to query s3 tables + //! required to query s3 tables string aws_region; - // Catalog generates the token using client id & secret + //! Catalog generates the token using client id & secret string token; + //! The scope of the OAuth token to request through the client_credentials flow + string scope; + //! OAuth endpoint + string oauth2_endpoint; + //! The warehouse where the catalog lives + string warehouse; }; class ICRClearCacheFunction : public TableFunction { @@ -56,8 +62,6 @@ class IRCatalog : public Catalog { string version; //! optional prefix string prefix; - //! warehouse - string warehouse; string secret_name; public: diff --git a/src/include/url_utils.hpp b/src/include/url_utils.hpp index ed64efc9..1b5d61a5 100644 --- a/src/include/url_utils.hpp +++ b/src/include/url_utils.hpp @@ -8,41 +8,54 @@ #pragma once -#include -#include +#include "duckdb/common/string.hpp" +#include "duckdb/common/vector.hpp" namespace duckdb { class IRCEndpointBuilder { +private: + struct QueryParameter { + public: + QueryParameter(const string &key, const string &value) : key(key), value(value) {} + public: + string key; + string value; + }; public: - void AddPathComponent(std::string component); + void AddPathComponent(const string &component); + + void SetPrefix(const string &prefix_); + string GetPrefix() const; - void SetPrefix(std::string prefix_); - std::string GetPrefix() const; + void SetHost(const string &host_); + string GetHost() const; - void SetHost(std::string host_); - std::string GetHost() const; + void SetWarehouse(const string &warehouse_); + string GetWarehouse() const; - void SetWarehouse(std::string warehouse_); - std::string GetWarehouse() const; + void SetVersion(const string &version_); + string GetVersion() const; - void SetVersion(std::string version_); - std::string GetVersion() const; + void AddQueryParameter(const string &key, const string &value); - std::string GetURL() const; + string GetURL() const; //! path components when querying. Like namespaces/tables etc. - std::vector path_components; + vector path_components; + + //! query parameters at the end of the url. + vector query_parameters; private: //! host of the endpoint, like `glue` or `polaris` - std::string host; + string host; //! version - std::string version; + string version; //! optional prefix - std::string prefix; + string prefix; //! warehouse - std::string warehouse; + string warehouse; }; -} // namespace duckdb \ No newline at end of file +} // namespace duckdb diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index 84efd08e..a77beb76 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -84,7 +84,7 @@ DatabaseSize IRCatalog::GetDatabaseSize(ClientContext &context) { IRCEndpointBuilder IRCatalog::GetBaseUrl() const { auto base_url = IRCEndpointBuilder(); base_url.SetPrefix(prefix); - base_url.SetWarehouse(warehouse); + base_url.SetWarehouse(credentials.warehouse); base_url.SetVersion(version); base_url.SetHost(host); return base_url; diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index 53bbab3f..3b10f26e 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -95,7 +95,7 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr Date: Wed, 26 Mar 2025 13:00:45 +0100 Subject: [PATCH 02/66] revert changes to api_result_to_doc, I misunderstood the intent there. Add 'oauth2_server_uri' as an optional secret key --- src/catalog_api.cpp | 7 +++---- src/catalog_utils.cpp | 4 ++-- src/iceberg_extension.cpp | 15 +++++++++++++-- src/include/catalog_api.hpp | 2 +- src/include/catalog_utils.hpp | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 7a9e6b5d..4c2e9d50 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -313,7 +313,6 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const } static string GetTableMetadataCached(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, const string &secret_name) { - struct curl_slist *extra_headers = NULL; auto url = catalog.GetBaseUrl(); url.AddPathComponent("namespaces"); url.AddPathComponent(schema); @@ -365,7 +364,7 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat return result; } -string IRCAPI::GetToken(ClientContext &context, const string &id, const string &secret, const string &endpoint, const string &scope) { +string IRCAPI::GetToken(ClientContext &context, const string &uri, const string &id, const string &secret, const string &endpoint, const string &scope) { vector parameters; parameters.push_back("client_credentials"); parameters.push_back(StringUtil::Format("%s=%s", "client_id", id)); @@ -373,10 +372,10 @@ string IRCAPI::GetToken(ClientContext &context, const string &id, const string & parameters.push_back(StringUtil::Format("%s=%s", "scope", scope)); string post_data = StringUtil::Format("grant_type=%s", StringUtil::Join(parameters, "&")); - string api_result = PostRequest(context, "http://localhost:30080/realms/iceberg/protocol/openid-connect/token", post_data); + string api_result = PostRequest(context, uri, post_data); //! FIXME: the oauth/tokens endpoint returns, on success; // { 'access_token', 'token_type', 'expires_in', , 'refresh_token', 'scope'} - std::unique_ptr doc(ICUtils::api_result_to_doc(api_result, "access_token")); + std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); return IcebergUtils::TryGetStrFromObject(root, "access_token"); } diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index be6d6b2b..50e3dffe 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -234,12 +234,12 @@ LogicalType ICUtils::ToICType(const LogicalType &input) { } } -yyjson_doc *ICUtils::api_result_to_doc(const string &api_result, const string &variable) { +yyjson_doc *ICUtils::api_result_to_doc(const string &api_result) { auto *doc = yyjson_read(api_result.c_str(), api_result.size(), 0); auto *root = yyjson_doc_get_root(doc); auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { - string err_msg = IcebergUtils::TryGetStrFromObject(error, variable); + string err_msg = IcebergUtils::TryGetStrFromObject(error, "message"); throw std::runtime_error(err_msg); } return doc; diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 7e233d98..9e43821b 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -33,7 +33,8 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context lower_name == "secret" || lower_name == "endpoint" || lower_name == "aws_region" || - lower_name == "scope") { + lower_name == "scope" || + lower_name == "oauth2_server_uri") { result->secret_map[lower_name] = named_param.second.ToString(); } else { throw InternalException("Unknown named parameter passed to CreateIRCSecretFunction: " + lower_name); @@ -43,12 +44,13 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context // Get token from catalog result->secret_map["token"] = IRCAPI::GetToken( context, + result->secret_map["oauth2_server_uri"].ToString(), result->secret_map["key_id"].ToString(), result->secret_map["secret"].ToString(), result->secret_map["endpoint"].ToString(), result->secret_map["scope"].ToString() ); - + //! Set redact keys result->redact_keys = {"token", "client_id", "client_secret"}; @@ -88,6 +90,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in string service; string endpoint_type; string endpoint; + string oauth2_server_uri; auto &warehouse = credentials.warehouse; auto &scope = credentials.scope; @@ -109,6 +112,8 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in warehouse = StringUtil::Lower(entry.second.ToString()); } else if (lower_name == "scope") { scope = StringUtil::Lower(entry.second.ToString()); + } else if (lower_name == "oauth2_server_uri") { + oauth2_server_uri = StringUtil::Lower(entry.second.ToString()); } else { throw BinderException("Unrecognized option for PC attach: %s", entry.first); } @@ -123,6 +128,11 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in scope = "PRINCIPAL_ROLE:ALL"; } + if (oauth2_server_uri.empty()) { + //! If no oauth2_server_uri is provided, default to the (deprecated) REST API endpoint for it + oauth2_server_uri = StringUtil::Format("%s/v1/oauth/tokens", endpoint); + } + if (endpoint_type == "glue" || endpoint_type == "s3_tables") { if (endpoint_type == "s3_tables") { service = "s3tables"; @@ -176,6 +186,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in Value key_val = kv_secret.TryGetValue("key_id"); Value secret_val = kv_secret.TryGetValue("secret"); CreateSecretInput create_secret_input; + create_secret_input.options["oauth2_server_uri"] = oauth2_server_uri; create_secret_input.options["key_id"] = key_val; create_secret_input.options["secret"] = secret_val; create_secret_input.options["endpoint"] = endpoint; diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 6be93280..c962711d 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -56,7 +56,7 @@ class IRCAPI { static IRCAPITable GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table_name, optional_ptr credentials = nullptr); static vector GetSchemas(ClientContext &context, IRCatalog &catalog, IRCCredentials credentials); static vector GetTablesInSchema(ClientContext &context, IRCatalog &catalog, const string &schema, IRCCredentials credentials); - static string GetToken(ClientContext &context, const string &id, const string &secret, const string &endpoint, const string &scope); + static string GetToken(ClientContext &context, const string &uri, const string &id, const string &secret, const string &endpoint, const string &scope); static IRCAPISchema CreateSchema(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials); static void DropSchema(ClientContext &context, const string &internal, const string &schema, IRCCredentials credentials); static IRCAPITable CreateTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials, CreateTableInfo *table_info); diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index af5e6988..e6475654 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -24,7 +24,7 @@ class ICUtils { static LogicalType TypeToLogicalType(ClientContext &context, const string &columnDefinition); static string TypeToString(const LogicalType &input); static string LogicalToIcebergType(const LogicalType &input); - static yyjson_doc *api_result_to_doc(const string &api_result, const string &variable = "message"); + static yyjson_doc *api_result_to_doc(const string &api_result); }; struct YyjsonDocDeleter { From 5c423a2a3f273ac7e1a674f0333474464b0f2bbe Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 27 Mar 2025 10:57:09 +0100 Subject: [PATCH 03/66] add support for additional secrets --- src/catalog_api.cpp | 97 +++++++++++++++++++++++++++++---- src/include/catalog_api.hpp | 2 + src/storage/irc_table_entry.cpp | 93 +++++++++++++------------------ 3 files changed, 127 insertions(+), 65 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 4c2e9d50..a606092c 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -207,6 +207,7 @@ static string GetRequestAws(ClientContext &context, IRCEndpointBuilder endpoint_ } static string GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &secret_name, const string &token = "", curl_slist *extra_headers = NULL) { + if (StringUtil::StartsWith(endpoint_builder.GetHost(), "glue." ) || StringUtil::StartsWith(endpoint_builder.GetHost(), "s3tables." )) { auto str = GetRequestAws(context, endpoint_builder, secret_name); return str; @@ -343,22 +344,97 @@ static IRCAPIColumnDefinition ParseColumnDefinition(yyjson_val *column_def) { return result; } +static void ParseConfigOptions(yyjson_val *config, case_insensitive_map_t &options) { + //! Set of recognized config parameters and the duckdb secret option that matches it. + static const case_insensitive_map_t config_to_option = { + {"s3.access-key-id", "key_id"}, + {"s3.secret-access-key", "secret"}, + {"s3.session-token", "session_token"}, + {"s3.region", "region"}, + {"s3.endpoint", "endpoint"} + }; + + auto config_size = yyjson_obj_size(config); + if (config && config_size > 0) { + for (auto &it : config_to_option) { + auto &key = it.first; + auto &option = it.second; + + auto *item = yyjson_obj_get(config, key.c_str()); + if (item) { + options[option] = yyjson_get_str(item); + } + } + auto *access_style = yyjson_obj_get(config, "s3.path-style-access"); + if (access_style) { + string value = yyjson_get_str(access_style); + bool use_ssl; + if (value == "true") { + use_ssl = false; + } else if (value == "false") { + use_ssl = true; + } else { + throw InternalException("Unexpected value ('%s') for 's3.path-style-access' in 'config' property", value); + } + options["use_ssl"] = Value(use_ssl); + } + } +} + IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, IRCCredentials credentials) { IRCAPITableCredentials result; string api_result = GetTableMetadataCached(context, catalog, schema, table, catalog.secret_name); std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); - auto *warehouse_credentials = yyjson_obj_get(root, "config"); - auto credential_size = yyjson_obj_size(warehouse_credentials); auto catalog_credentials = IRCatalog::GetSecret(context, catalog.secret_name); - if (warehouse_credentials && credential_size > 0) { - result.key_id = IcebergUtils::TryGetStrFromObject(warehouse_credentials, "s3.access-key-id", false); - result.secret = IcebergUtils::TryGetStrFromObject(warehouse_credentials, "s3.secret-access-key", false); - result.session_token = IcebergUtils::TryGetStrFromObject(warehouse_credentials, "s3.session-token", false); - if (catalog_credentials) { - auto kv_secret = dynamic_cast(*catalog_credentials->secret); - auto region = kv_secret.TryGetValue("region").ToString(); - result.region = region; + + // Mapping from config key to a duckdb secret option + + case_insensitive_map_t config_options; + if (catalog_credentials) { + auto kv_secret = dynamic_cast(*catalog_credentials->secret); + auto region = kv_secret.TryGetValue("region").ToString(); + config_options["region"] = region; + } + + auto *warehouse_credentials = yyjson_obj_get(root, "config"); + ParseConfigOptions(warehouse_credentials, config_options); + + if (!config_options.empty()) { + CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); + info.options = config_options; + info.name = "PLACEHOLDER"; + info.type = "s3"; + info.provider = "config"; + info.storage_type = "memory"; + result.storage_credentials.push_back(info); + } + + auto *storage_credentials = yyjson_obj_get(root, "storage-credentials"); + auto storage_credentials_size = yyjson_arr_size(storage_credentials); + if (storage_credentials && storage_credentials_size > 0) { + yyjson_val *storage_credential; + size_t index, max; + yyjson_arr_foreach(storage_credentials, index, max, storage_credential) { + auto *sc_prefix = yyjson_obj_get(storage_credential, "prefix"); + if (!sc_prefix) { + throw InternalException("required property 'prefix' is missing from the StorageCredential schema"); + } + + CreateSecretInfo create_secret_info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); + auto prefix_string = yyjson_get_str(sc_prefix); + if (!prefix_string) { + throw InternalException("property 'prefix' of StorageCredential is NULL"); + } + create_secret_info.scope.push_back(string(prefix_string)); + create_secret_info.type = "s3"; + create_secret_info.provider = "config"; + create_secret_info.storage_type = "memory"; + create_secret_info.options = config_options; + + auto *sc_config = yyjson_obj_get(storage_credential, "config"); + ParseConfigOptions(sc_config, create_secret_info.options); + result.storage_credentials.push_back(create_secret_info); } } return result; @@ -471,6 +547,7 @@ vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catal GetRequest(context, endpoint_builder, catalog.secret_name, catalog.credentials.token); std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); + //! 'ListNamespacesResponse' auto *schemas = yyjson_obj_get(root, "namespaces"); size_t idx, max; yyjson_val *schema; diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index c962711d..c3d42e8d 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -3,6 +3,7 @@ #include "duckdb/common/types.hpp" #include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/parsed_data/create_secret_info.hpp" //#include "storage/irc_catalog.hpp" namespace duckdb { @@ -41,6 +42,7 @@ struct IRCAPITableCredentials { string secret; string session_token; string region; + vector storage_credentials; }; class IRCAPI { diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index 3b10f26e..7d49110e 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -13,7 +13,6 @@ #include "duckdb/planner/logical_operator.hpp" #include "duckdb/planner/operator/logical_get.hpp" - namespace duckdb { ICTableEntry::ICTableEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info) @@ -54,67 +53,51 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrschema_name, table_data->name, ic_catalog.credentials); CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); // First check if table credentials are set (possible the IC catalog does not return credentials) - if (!table_credentials.key_id.empty()) { - // Inject secret into secret manager scoped to this path - info.name = "__internal_ic_" + table_data->table_id; - info.type = "s3"; - info.provider = "config"; - info.storage_type = "memory"; - info.options = { - {"key_id", table_credentials.key_id}, - {"secret", table_credentials.secret}, - {"session_token", table_credentials.session_token}, - {"region", table_credentials.region}, - }; + for (auto &info : table_credentials.storage_credentials) { + if (info.name == "PLACEHOLDER") { + //! Set a name for the initial secret created from the LoadTableResult's 'config' object + if (StringUtil::StartsWith(ic_catalog.host, "s3tables")) { + info.name = "__internal_ic_" + table_data->table_id + "__" + table_data->schema_name + "__" + table_data->name; + } else { + info.name = "__internal_ic_" + table_data->table_id; + } + } + + //! Limit the scope to the metadata location if no explicit scope was set + if (info.scope.empty()) { + std::string lc_storage_location; + lc_storage_location.resize(table_data->storage_location.size()); + std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); + size_t metadata_pos = lc_storage_location.find("metadata"); + if (metadata_pos != std::string::npos) { + info.scope = {lc_storage_location.substr(0, metadata_pos)}; + } else { + throw std::runtime_error("Substring not found"); + } + } if (StringUtil::StartsWith(ic_catalog.host, "glue")) { + //! Override the endpoint if 'glue' is the host of the catalog auto secret_entry = IRCatalog::GetSecret(context, ic_catalog.secret_name); auto kv_secret = dynamic_cast(*secret_entry->secret); auto region = kv_secret.TryGetValue("region").ToString(); auto endpoint = "s3." + region + ".amazonaws.com"; info.options["endpoint"] = endpoint; - } - - std::string lc_storage_location; - lc_storage_location.resize(table_data->storage_location.size()); - std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); - size_t metadata_pos = lc_storage_location.find("metadata"); - if (metadata_pos != std::string::npos) { - info.scope = {lc_storage_location.substr(0, metadata_pos)}; - } else { - throw std::runtime_error("Substring not found"); - } - auto my_secret = secret_manager.CreateSecret(context, info); - } else if (StringUtil::StartsWith(ic_catalog.host, "s3tables")) { - // Inject secret into secret manager with correct endpoint - CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); - auto secret_entry = IRCatalog::GetSecret(context, ic_catalog.secret_name); - auto kv_secret = dynamic_cast(*secret_entry->secret); - info.name = "__internal_ic_" + table_data->table_id + "__" + table_data->schema_name + "__" + table_data->name; - info.type = "s3"; - info.provider = "config"; - info.storage_type = "memory"; - - auto substrings = StringUtil::Split(ic_catalog.credentials.warehouse, ":"); - D_ASSERT(substrings.size() == 6); - auto region = substrings[3]; - auto endpoint = "s3." + region + ".amazonaws.com"; - info.options = { - {"key_id", kv_secret.TryGetValue("key_id").ToString()}, - {"secret", kv_secret.TryGetValue("secret").ToString()}, - {"session_token", kv_secret.TryGetValue("session_token").IsNull() ? "" : kv_secret.TryGetValue("session_token").ToString()}, - {"region", region}, - {"endpoint", endpoint} - }; - - std::string lc_storage_location; - lc_storage_location.resize(table_data->storage_location.size()); - std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); - size_t metadata_pos = lc_storage_location.find("metadata"); - if (metadata_pos != std::string::npos) { - info.scope = {lc_storage_location.substr(0, metadata_pos)}; - } else { - throw std::runtime_error("Substring not found"); + } else if (StringUtil::StartsWith(ic_catalog.host, "s3tables")) { + //! Override all the options if 's3tables' is the host of the catalog + auto secret_entry = IRCatalog::GetSecret(context, ic_catalog.secret_name); + auto kv_secret = dynamic_cast(*secret_entry->secret); + auto substrings = StringUtil::Split(ic_catalog.credentials.warehouse, ":"); + D_ASSERT(substrings.size() == 6); + auto region = substrings[3]; + auto endpoint = "s3." + region + ".amazonaws.com"; + info.options = { + {"key_id", kv_secret.TryGetValue("key_id").ToString()}, + {"secret", kv_secret.TryGetValue("secret").ToString()}, + {"session_token", kv_secret.TryGetValue("session_token").IsNull() ? "" : kv_secret.TryGetValue("session_token").ToString()}, + {"region", region}, + {"endpoint", endpoint} + }; } auto my_secret = secret_manager.CreateSecret(context, info); } From d47dbceb268bbdd9a56cce793e4530c6b71b021d Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 27 Mar 2025 11:40:07 +0100 Subject: [PATCH 04/66] now the file read actually succeeds --- src/catalog_api.cpp | 59 +++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index a606092c..3de60719 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -355,30 +355,47 @@ static void ParseConfigOptions(yyjson_val *config, case_insensitive_map_t }; auto config_size = yyjson_obj_size(config); - if (config && config_size > 0) { - for (auto &it : config_to_option) { - auto &key = it.first; - auto &option = it.second; - - auto *item = yyjson_obj_get(config, key.c_str()); - if (item) { - options[option] = yyjson_get_str(item); - } + if (!config || config_size == 0) { + return; + } + for (auto &it : config_to_option) { + auto &key = it.first; + auto &option = it.second; + + auto *item = yyjson_obj_get(config, key.c_str()); + if (item) { + options[option] = yyjson_get_str(item); } - auto *access_style = yyjson_obj_get(config, "s3.path-style-access"); - if (access_style) { - string value = yyjson_get_str(access_style); - bool use_ssl; - if (value == "true") { - use_ssl = false; - } else if (value == "false") { - use_ssl = true; - } else { - throw InternalException("Unexpected value ('%s') for 's3.path-style-access' in 'config' property", value); - } - options["use_ssl"] = Value(use_ssl); + } + auto *access_style = yyjson_obj_get(config, "s3.path-style-access"); + if (access_style) { + string value = yyjson_get_str(access_style); + bool path_style; + if (value == "true") { + path_style = true; + } else if (value == "false") { + path_style = false; + } else { + throw InternalException("Unexpected value ('%s') for 's3.path-style-access' in 'config' property", value); } + options["use_ssl"] = Value(!path_style); + if (path_style) { + options["url_style"] = "path"; + } + } + + auto endpoint_it = options.find("endpoint"); + if (endpoint_it == options.end()) { + return; + } + auto endpoint = endpoint_it->second.ToString(); + if (StringUtil::StartsWith(endpoint, "http://")) { + endpoint = endpoint.substr(7, std::string::npos); + } + if (StringUtil::EndsWith(endpoint, "/")) { + endpoint = endpoint.substr(0, endpoint.size() - 1); } + endpoint_it->second = endpoint; } IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, IRCCredentials credentials) { From ba8c4db17c285f5e6eaeea39337b75e38b3a7403 Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 27 Mar 2025 11:57:00 +0100 Subject: [PATCH 05/66] check if the path ends in 'gz.metadata.json', if it does - it's gzip-encoded --- src/common/iceberg.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp index bbc0902b..4b5d722d 100644 --- a/src/common/iceberg.cpp +++ b/src/common/iceberg.cpp @@ -238,7 +238,7 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa } string IcebergSnapshot::ReadMetaData(const string &path, FileSystem &fs, const string &metadata_compression_codec) { - if (metadata_compression_codec == "gzip") { + if (metadata_compression_codec == "gzip" || StringUtil::EndsWith(path, "gz.metadata.json")) { return IcebergUtils::GzFileToString(path, fs); } return IcebergUtils::FileToString(path, fs); From 6b8717e854f07b19baa39c2ac7c4d7badf8c3c4f Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 27 Mar 2025 13:07:07 +0100 Subject: [PATCH 06/66] remove the prefix from GetBaseUrl --- src/storage/irc_catalog.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index aef9d4cf..baa2de10 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -135,7 +135,6 @@ DatabaseSize IRCatalog::GetDatabaseSize(ClientContext &context) { IRCEndpointBuilder IRCatalog::GetBaseUrl() const { auto base_url = IRCEndpointBuilder(); - base_url.SetPrefix(prefix); base_url.SetWarehouse(credentials.warehouse); base_url.SetVersion(version); base_url.SetHost(host); From a8385040cf720c809b04b942731d7d6cfdd55f0d Mon Sep 17 00:00:00 2001 From: Tishj Date: Thu, 27 Mar 2025 13:15:24 +0100 Subject: [PATCH 07/66] remove the ability to provide 'warehouse' as an option, the warehouse should be the path of the catalog --- src/iceberg_extension.cpp | 8 +------- src/include/storage/irc_catalog.hpp | 2 -- src/storage/irc_catalog.cpp | 2 +- src/storage/irc_table_entry.cpp | 2 +- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 04702405..999f77d0 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -93,7 +93,6 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in string endpoint; string oauth2_server_uri; - auto &warehouse = credentials.warehouse; auto &scope = credentials.scope; // check if we have a secret provided @@ -109,8 +108,6 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in } else if (lower_name == "endpoint") { endpoint = StringUtil::Lower(entry.second.ToString()); StringUtil::RTrim(endpoint, "/"); - } else if (lower_name == "warehouse") { - warehouse = StringUtil::Lower(entry.second.ToString()); } else if (lower_name == "scope") { scope = StringUtil::Lower(entry.second.ToString()); } else if (lower_name == "oauth2_server_uri") { @@ -119,10 +116,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in throw BinderException("Unrecognized option for PC attach: %s", entry.first); } } - if (warehouse.empty()) { - //! Default to the path of the catalog as the warehouse if none is provided - warehouse = info.path; - } + auto warehouse = info.path; if (scope.empty()) { //! Default to the Polaris scope: 'PRINCIPAL_ROLE:ALL' diff --git a/src/include/storage/irc_catalog.hpp b/src/include/storage/irc_catalog.hpp index ea347192..6a03b0ed 100644 --- a/src/include/storage/irc_catalog.hpp +++ b/src/include/storage/irc_catalog.hpp @@ -23,8 +23,6 @@ struct IRCCredentials { string scope; //! OAuth endpoint string oauth2_endpoint; - //! The warehouse where the catalog lives - string warehouse; }; class ICRClearCacheFunction : public TableFunction { diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index baa2de10..80c616ca 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -135,7 +135,7 @@ DatabaseSize IRCatalog::GetDatabaseSize(ClientContext &context) { IRCEndpointBuilder IRCatalog::GetBaseUrl() const { auto base_url = IRCEndpointBuilder(); - base_url.SetWarehouse(credentials.warehouse); + base_url.SetWarehouse(warehouse); base_url.SetVersion(version); base_url.SetHost(host); switch (catalog_type) { diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index 7d49110e..3d4133c7 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -87,7 +87,7 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr(*secret_entry->secret); - auto substrings = StringUtil::Split(ic_catalog.credentials.warehouse, ":"); + auto substrings = StringUtil::Split(ic_catalog.warehouse, ":"); D_ASSERT(substrings.size() == 6); auto region = substrings[3]; auto endpoint = "s3." + region + ".amazonaws.com"; From b8cc79bf4fe72c6fba5f7294145c0316823e8340 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 13:33:06 +0100 Subject: [PATCH 08/66] opt for the more specific secret name. restore previous behavior of only setting the region from the catalog_credentials secret if a 'config' is present in the result --- src/catalog_api.cpp | 5 ++--- src/storage/irc_table_entry.cpp | 6 +----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index fcd42798..f9640900 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -136,13 +136,12 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat // Mapping from config key to a duckdb secret option case_insensitive_map_t config_options; - if (catalog_credentials) { + auto *warehouse_credentials = yyjson_obj_get(root, "config"); + if (warehouse_credentials && catalog_credentials) { auto kv_secret = dynamic_cast(*catalog_credentials->secret); auto region = kv_secret.TryGetValue("region").ToString(); config_options["region"] = region; } - - auto *warehouse_credentials = yyjson_obj_get(root, "config"); ParseConfigOptions(warehouse_credentials, config_options); if (!config_options.empty()) { diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index 3d4133c7..20be4785 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -56,11 +56,7 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrtable_id + "__" + table_data->schema_name + "__" + table_data->name; - } else { - info.name = "__internal_ic_" + table_data->table_id; - } + info.name = "__internal_ic_" + table_data->table_id + "__" + table_data->schema_name + "__" + table_data->name; } //! Limit the scope to the metadata location if no explicit scope was set From 9da8a89c266de7827d67f49f8de5e685b4fe51d2 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 13:37:02 +0100 Subject: [PATCH 09/66] remove dead code (GetTablesInSchema) --- src/catalog_api.cpp | 2 +- src/include/catalog_api.hpp | 3 +-- src/storage/irc_table_entry.cpp | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index f9640900..1022f67f 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -126,7 +126,7 @@ static void ParseConfigOptions(yyjson_val *config, case_insensitive_map_t endpoint_it->second = endpoint; } -IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, IRCCredentials credentials) { +IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table) { IRCAPITableCredentials result; string api_result = GetTableMetadataCached(context, catalog, schema, table, catalog.secret_name); std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index c3d42e8d..75c0fdfc 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -52,12 +52,11 @@ class IRCAPI { //! WARNING: not thread-safe. To be called once on extension initialization static void InitializeCurl(); - static IRCAPITableCredentials GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, IRCCredentials credentials); + static IRCAPITableCredentials GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table); static vector GetCatalogs(ClientContext &context, IRCatalog &catalog, IRCCredentials credentials); static vector GetTables(ClientContext &context, IRCatalog &catalog, const string &schema); static IRCAPITable GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table_name, optional_ptr credentials = nullptr); static vector GetSchemas(ClientContext &context, IRCatalog &catalog, IRCCredentials credentials); - static vector GetTablesInSchema(ClientContext &context, IRCatalog &catalog, const string &schema, IRCCredentials credentials); static string GetToken(ClientContext &context, const string &uri, const string &id, const string &secret, const string &endpoint, const string &scope); static IRCAPISchema CreateSchema(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials); static void DropSchema(ClientContext &context, const string &internal, const string &schema, IRCCredentials credentials); diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index 20be4785..df4ca557 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -50,7 +50,7 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrschema_name, table_data->name, ic_catalog.credentials); + auto table_credentials = IRCAPI::GetTableCredentials(context, ic_catalog, table_data->schema_name, table_data->name); CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); // First check if table credentials are set (possible the IC catalog does not return credentials) for (auto &info : table_credentials.storage_credentials) { From 2b1c7606b76ed022ed7b5273d791ffd9aa7aaa96 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 13:49:34 +0100 Subject: [PATCH 10/66] distinguish between the config secret and the storage-credential secrets, treat them differently --- src/catalog_api.cpp | 23 +++++++++++----------- src/include/catalog_api.hpp | 7 ++----- src/storage/irc_table_entry.cpp | 35 +++++++++++++++++---------------- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 1022f67f..28c158d3 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -126,7 +126,7 @@ static void ParseConfigOptions(yyjson_val *config, case_insensitive_map_t endpoint_it->second = endpoint; } -IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table) { +IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, const string &secret_base_name) { IRCAPITableCredentials result; string api_result = GetTableMetadataCached(context, catalog, schema, table, catalog.secret_name); std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); @@ -136,22 +136,22 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat // Mapping from config key to a duckdb secret option case_insensitive_map_t config_options; - auto *warehouse_credentials = yyjson_obj_get(root, "config"); - if (warehouse_credentials && catalog_credentials) { + auto *config_val = yyjson_obj_get(root, "config"); + if (config_val && catalog_credentials) { auto kv_secret = dynamic_cast(*catalog_credentials->secret); auto region = kv_secret.TryGetValue("region").ToString(); config_options["region"] = region; } - ParseConfigOptions(warehouse_credentials, config_options); + ParseConfigOptions(config_val, config_options); if (!config_options.empty()) { - CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); - info.options = config_options; - info.name = "PLACEHOLDER"; - info.type = "s3"; - info.provider = "config"; - info.storage_type = "memory"; - result.storage_credentials.push_back(info); + result.config = make_uniq(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); + auto &config = *result.config; + config.options = config_options; + config.name = secret_base_name; + config.type = "s3"; + config.provider = "config"; + config.storage_type = "memory"; } auto *storage_credentials = yyjson_obj_get(root, "storage-credentials"); @@ -171,6 +171,7 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat throw InternalException("property 'prefix' of StorageCredential is NULL"); } create_secret_info.scope.push_back(string(prefix_string)); + create_secret_info.name = StringUtil::Format("%s_%d_%s", secret_base_name, index, prefix_string); create_secret_info.type = "s3"; create_secret_info.provider = "config"; create_secret_info.storage_type = "memory"; diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 75c0fdfc..81a1df84 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -38,10 +38,7 @@ struct IRCAPISchema { }; struct IRCAPITableCredentials { - string key_id; - string secret; - string session_token; - string region; + unique_ptr config; vector storage_credentials; }; @@ -52,7 +49,7 @@ class IRCAPI { //! WARNING: not thread-safe. To be called once on extension initialization static void InitializeCurl(); - static IRCAPITableCredentials GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table); + static IRCAPITableCredentials GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, const string &secret_base_name); static vector GetCatalogs(ClientContext &context, IRCatalog &catalog, IRCCredentials credentials); static vector GetTables(ClientContext &context, IRCatalog &catalog, const string &schema); static IRCAPITable GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table_name, optional_ptr credentials = nullptr); diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index df4ca557..91cebaa2 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -50,26 +50,23 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrschema_name, table_data->name); + auto secret_base_name = StringUtil::Format("__internal_ic_%s__%s__%s", table_data->table_id, table_data->schema_name, table_data->name); + auto table_credentials = IRCAPI::GetTableCredentials(context, ic_catalog, table_data->schema_name, table_data->name, secret_base_name); CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); // First check if table credentials are set (possible the IC catalog does not return credentials) - for (auto &info : table_credentials.storage_credentials) { - if (info.name == "PLACEHOLDER") { - //! Set a name for the initial secret created from the LoadTableResult's 'config' object - info.name = "__internal_ic_" + table_data->table_id + "__" + table_data->schema_name + "__" + table_data->name; - } - //! Limit the scope to the metadata location if no explicit scope was set - if (info.scope.empty()) { - std::string lc_storage_location; - lc_storage_location.resize(table_data->storage_location.size()); - std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); - size_t metadata_pos = lc_storage_location.find("metadata"); - if (metadata_pos != std::string::npos) { - info.scope = {lc_storage_location.substr(0, metadata_pos)}; - } else { - throw std::runtime_error("Substring not found"); - } + if (table_credentials.config) { + auto &info = *table_credentials.config; + D_ASSERT(info.scope.empty()); + //! Limit the scope to the metadata location + std::string lc_storage_location; + lc_storage_location.resize(table_data->storage_location.size()); + std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); + size_t metadata_pos = lc_storage_location.find("metadata"); + if (metadata_pos != std::string::npos) { + info.scope = {lc_storage_location.substr(0, metadata_pos)}; + } else { + throw std::runtime_error("Substring not found"); } if (StringUtil::StartsWith(ic_catalog.host, "glue")) { @@ -98,6 +95,10 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr return_types; vector names; From 742d9c7aad432d1119de32462dbde62b89157e57 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 13:57:08 +0100 Subject: [PATCH 11/66] verify that the 'token_type' is 'bearer' --- src/catalog_api.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 28c158d3..464d93a2 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -198,7 +198,20 @@ string IRCAPI::GetToken(ClientContext &context, const string &uri, const string // { 'access_token', 'token_type', 'expires_in', , 'refresh_token', 'scope'} std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); - return IcebergUtils::TryGetStrFromObject(root, "access_token"); + auto access_token_val = yyjson_obj_get(root, "access_token"); + auto token_type_val = yyjson_obj_get(root, "token_type"); + if (!access_token_val) { + throw IOException("OAuthTokenResponse is missing required property 'access_token'"); + } + if (!token_type_val) { + throw IOException("OAuthTokenResponse is missing required property 'token_type'"); + } + string token_type = yyjson_get_str(token_type_val); + if (!StringUtil::CIEquals(token_type, "bearer")) { + throw NotImplementedException("token_type return value '%s' is not supported, only supports 'bearer' currently.", token_type); + } + string access_token = yyjson_get_str(access_token_val); + return access_token; } static void populateTableMetadata(IRCAPITable &table, yyjson_val *metadata_root) { From f24d61903e21be4a14d17d47d34e104786418329 Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 14:02:37 +0100 Subject: [PATCH 12/66] add warning for deprecated oath2 endpoint --- src/iceberg_extension.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 999f77d0..10f28606 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -125,6 +125,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in if (oauth2_server_uri.empty()) { //! If no oauth2_server_uri is provided, default to the (deprecated) REST API endpoint for it + DUCKDB_LOG_WARN(context, "iceberg", "'oauth2_server_uri' is not set, defaulting to deprecated '{endpoint}/v1/oauth/tokens' oauth2_server_uri"); oauth2_server_uri = StringUtil::Format("%s/v1/oauth/tokens", endpoint); } auto catalog_type = ICEBERG_CATALOG_TYPE::INVALID; From e1cb59e01c9b996efc4a4ddcac92e3beabb6455d Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 14:06:17 +0100 Subject: [PATCH 13/66] remove warehouse from endpoint builder + other clean up there --- src/common/url_utils.cpp | 12 ------------ src/include/storage/irc_catalog.hpp | 2 ++ src/include/url_utils.hpp | 17 ----------------- src/storage/irc_catalog.cpp | 1 - 4 files changed, 2 insertions(+), 30 deletions(-) diff --git a/src/common/url_utils.cpp b/src/common/url_utils.cpp index 11b1cba1..d67e4d4d 100644 --- a/src/common/url_utils.cpp +++ b/src/common/url_utils.cpp @@ -9,10 +9,6 @@ void IRCEndpointBuilder::AddPathComponent(const string &component) { } } -void IRCEndpointBuilder::AddQueryParameter(const string &key, const string &value) { - query_parameters.emplace_back(key, value); -} - void IRCEndpointBuilder::SetPrefix(const string &prefix_) { prefix = prefix_; } @@ -29,14 +25,6 @@ string IRCEndpointBuilder::GetVersion() const { return version; } -void IRCEndpointBuilder::SetWarehouse(const string &warehouse_) { - warehouse = warehouse_; -} - -string IRCEndpointBuilder::GetWarehouse() const { - return warehouse; -} - void IRCEndpointBuilder::SetHost(const string &host_) { host = host_; } diff --git a/src/include/storage/irc_catalog.hpp b/src/include/storage/irc_catalog.hpp index 6a03b0ed..ea347192 100644 --- a/src/include/storage/irc_catalog.hpp +++ b/src/include/storage/irc_catalog.hpp @@ -23,6 +23,8 @@ struct IRCCredentials { string scope; //! OAuth endpoint string oauth2_endpoint; + //! The warehouse where the catalog lives + string warehouse; }; class ICRClearCacheFunction : public TableFunction { diff --git a/src/include/url_utils.hpp b/src/include/url_utils.hpp index 9de840df..30fab578 100644 --- a/src/include/url_utils.hpp +++ b/src/include/url_utils.hpp @@ -15,14 +15,6 @@ namespace duckdb { class IRCEndpointBuilder { -private: - struct QueryParameter { - public: - QueryParameter(const string &key, const string &value) : key(key), value(value) {} - public: - string key; - string value; - }; public: void AddPathComponent(const string &component); void AddQueryParameter(const string &key, const string &value); @@ -33,9 +25,6 @@ class IRCEndpointBuilder { void SetHost(const string &host_); string GetHost() const; - void SetWarehouse(const string &warehouse_); - string GetWarehouse() const; - void SetVersion(const string &version_); string GetVersion() const; @@ -47,10 +36,6 @@ class IRCEndpointBuilder { //! path components when querying. Like namespaces/tables etc. vector path_components; - - //! query parameters at the end of the url. - vector query_parameters; - private: //! host of the endpoint, like `glue` or `polaris` string host; @@ -58,8 +43,6 @@ class IRCEndpointBuilder { string version; //! optional prefix string prefix; - //! warehouse - string warehouse; unordered_map params; }; diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index 80c616ca..adb1477e 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -135,7 +135,6 @@ DatabaseSize IRCatalog::GetDatabaseSize(ClientContext &context) { IRCEndpointBuilder IRCatalog::GetBaseUrl() const { auto base_url = IRCEndpointBuilder(); - base_url.SetWarehouse(warehouse); base_url.SetVersion(version); base_url.SetHost(host); switch (catalog_type) { From 733af4cc6e325c0aad2fe6300d95ea803e82778f Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 14:10:15 +0100 Subject: [PATCH 14/66] only create a secret out of the 'config' if there are no 'storage-credentials' --- src/catalog_api.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 464d93a2..8962e561 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -144,16 +144,6 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat } ParseConfigOptions(config_val, config_options); - if (!config_options.empty()) { - result.config = make_uniq(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); - auto &config = *result.config; - config.options = config_options; - config.name = secret_base_name; - config.type = "s3"; - config.provider = "config"; - config.storage_type = "memory"; - } - auto *storage_credentials = yyjson_obj_get(root, "storage-credentials"); auto storage_credentials_size = yyjson_arr_size(storage_credentials); if (storage_credentials && storage_credentials_size > 0) { @@ -182,6 +172,18 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat result.storage_credentials.push_back(create_secret_info); } } + + if (result.storage_credentials.empty() && !config_options.empty()) { + //! Only create a secret out of the 'config' if there are no 'storage-credentials' + result.config = make_uniq(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); + auto &config = *result.config; + config.options = config_options; + config.name = secret_base_name; + config.type = "s3"; + config.provider = "config"; + config.storage_type = "memory"; + } + return result; } From d5f1ba37cf6a44e7137fcb434ed345a1c40bcbbc Mon Sep 17 00:00:00 2001 From: Tishj Date: Fri, 28 Mar 2025 14:15:31 +0100 Subject: [PATCH 15/66] rename 'scope' to 'oauth2_scope' --- src/iceberg_extension.cpp | 16 ++++++++-------- src/include/storage/irc_catalog.hpp | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 10f28606..2ccf3376 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -34,7 +34,7 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context lower_name == "secret" || lower_name == "endpoint" || lower_name == "aws_region" || - lower_name == "scope" || + lower_name == "oauth2_scope" || lower_name == "oauth2_server_uri") { result->secret_map[lower_name] = named_param.second.ToString(); } else { @@ -49,7 +49,7 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context result->secret_map["key_id"].ToString(), result->secret_map["secret"].ToString(), result->secret_map["endpoint"].ToString(), - result->secret_map["scope"].ToString() + result->secret_map["oauth2_scope"].ToString() ); //! Set redact keys @@ -93,7 +93,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in string endpoint; string oauth2_server_uri; - auto &scope = credentials.scope; + auto &oauth2_scope = credentials.oauth2_scope; // check if we have a secret provided string secret_name; @@ -108,8 +108,8 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in } else if (lower_name == "endpoint") { endpoint = StringUtil::Lower(entry.second.ToString()); StringUtil::RTrim(endpoint, "/"); - } else if (lower_name == "scope") { - scope = StringUtil::Lower(entry.second.ToString()); + } else if (lower_name == "oauth2_scope") { + oauth2_scope = StringUtil::Lower(entry.second.ToString()); } else if (lower_name == "oauth2_server_uri") { oauth2_server_uri = StringUtil::Lower(entry.second.ToString()); } else { @@ -118,9 +118,9 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in } auto warehouse = info.path; - if (scope.empty()) { + if (oauth2_scope.empty()) { //! Default to the Polaris scope: 'PRINCIPAL_ROLE:ALL' - scope = "PRINCIPAL_ROLE:ALL"; + oauth2_scope = "PRINCIPAL_ROLE:ALL"; } if (oauth2_server_uri.empty()) { @@ -198,7 +198,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in create_secret_input.options["key_id"] = key_val; create_secret_input.options["secret"] = secret_val; create_secret_input.options["endpoint"] = endpoint; - create_secret_input.options["scope"] = scope; + create_secret_input.options["oauth2_scope"] = oauth2_scope; auto new_secret = CreateCatalogSecretFunction(context, create_secret_input); auto &kv_secret_new = dynamic_cast(*new_secret); Value token = kv_secret_new.TryGetValue("token"); diff --git a/src/include/storage/irc_catalog.hpp b/src/include/storage/irc_catalog.hpp index ea347192..2701d8f9 100644 --- a/src/include/storage/irc_catalog.hpp +++ b/src/include/storage/irc_catalog.hpp @@ -20,7 +20,7 @@ struct IRCCredentials { //! Catalog generates the token using client id & secret string token; //! The scope of the OAuth token to request through the client_credentials flow - string scope; + string oauth2_scope; //! OAuth endpoint string oauth2_endpoint; //! The warehouse where the catalog lives From 49ff59c906f64014e3d7ea573031366c64b1d399 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 29 Mar 2025 15:01:45 +0100 Subject: [PATCH 16/66] run make-format --- CMakeLists.txt | 16 ++- src/catalog_api.cpp | 72 ++++++----- src/common/api_utils.cpp | 120 +++++++++--------- src/common/iceberg.cpp | 66 ++++++---- src/common/url_utils.cpp | 2 +- src/common/utils.cpp | 24 ++-- src/iceberg_extension.cpp | 50 ++++---- src/iceberg_functions/iceberg_metadata.cpp | 12 +- .../iceberg_multi_file_reader.cpp | 119 +++++++++-------- src/iceberg_functions/iceberg_snapshots.cpp | 3 +- src/iceberg_manifest.cpp | 49 ++++--- src/include/api_utils.hpp | 39 +++--- src/include/catalog_api.hpp | 25 ++-- src/include/catalog_utils.hpp | 4 +- .../credentials/credential_provider.hpp | 7 +- src/include/iceberg_manifest.hpp | 27 ++-- src/include/iceberg_metadata.hpp | 16 ++- src/include/iceberg_multi_file_reader.hpp | 22 ++-- src/include/iceberg_options.hpp | 8 +- src/include/iceberg_types.hpp | 2 + src/include/iceberg_utils.hpp | 6 +- src/include/manifest_reader.hpp | 27 ++-- src/include/storage/irc_catalog.hpp | 14 +- src/include/storage/irc_table_set.hpp | 6 +- src/include/url_utils.hpp | 1 + src/manifest_reader.cpp | 10 +- src/storage/irc_catalog.cpp | 44 ++++--- src/storage/irc_clear_cache.cpp | 3 +- src/storage/irc_schema_entry.cpp | 12 +- src/storage/irc_schema_set.cpp | 3 +- src/storage/irc_table_entry.cpp | 32 +++-- src/storage/irc_table_set.cpp | 7 +- 32 files changed, 456 insertions(+), 392 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f23c448f..73d3203b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ set(EXTENSION_SOURCES src/iceberg_manifest.cpp src/manifest_reader.cpp src/catalog_api.cpp - src/catalog_utils.cpp + src/catalog_utils.cpp src/common/utils.cpp src/common/url_utils.cpp src/common/schema.cpp @@ -34,8 +34,7 @@ set(EXTENSION_SOURCES src/storage/irc_table_entry.cpp src/storage/irc_table_set.cpp src/storage/irc_transaction.cpp - src/storage/irc_transaction_manager.cpp -) + src/storage/irc_transaction_manager.cpp) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) @@ -47,13 +46,16 @@ find_package(AWSSDK REQUIRED COMPONENTS core sso sts) include_directories(${CURL_INCLUDE_DIRS}) # AWS SDK FROM vcpkg -target_include_directories(${EXTENSION_NAME} PUBLIC $) +target_include_directories(${EXTENSION_NAME} + PUBLIC $) target_link_libraries(${EXTENSION_NAME} PUBLIC ${AWSSDK_LINK_LIBRARIES}) -target_include_directories(${TARGET_NAME}_loadable_extension PRIVATE $) -target_link_libraries(${TARGET_NAME}_loadable_extension ${AWSSDK_LINK_LIBRARIES}) +target_include_directories(${TARGET_NAME}_loadable_extension + PRIVATE $) +target_link_libraries(${TARGET_NAME}_loadable_extension + ${AWSSDK_LINK_LIBRARIES}) # Link dependencies into extension -target_link_libraries(${EXTENSION_NAME} PUBLIC ${CURL_LIBRARIES}) +target_link_libraries(${EXTENSION_NAME} PUBLIC ${CURL_LIBRARIES}) target_link_libraries(${TARGET_NAME}_loadable_extension ${CURL_LIBRARIES}) install( diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 8962e561..36c7f61d 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -20,7 +20,8 @@ using namespace duckdb_yyjson; namespace duckdb { -static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, const string &secret_name) { +static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, + const string &secret_name) { struct curl_slist *extra_headers = NULL; auto url = catalog.GetBaseUrl(); url.AddPathComponent(catalog.prefix); @@ -29,19 +30,15 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const url.AddPathComponent("tables"); url.AddPathComponent(table); extra_headers = curl_slist_append(extra_headers, "X-Iceberg-Access-Delegation: vended-credentials"); - string api_result = APIUtils::GetRequest( - context, - url, - secret_name, - catalog.credentials.token, - extra_headers); + string api_result = APIUtils::GetRequest(context, url, secret_name, catalog.credentials.token, extra_headers); catalog.SetCachedValue(url.GetURL(), api_result); curl_slist_free_all(extra_headers); return api_result; } -static string GetTableMetadataCached(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, const string &secret_name) { +static string GetTableMetadataCached(ClientContext &context, IRCatalog &catalog, const string &schema, + const string &table, const string &secret_name) { auto url = catalog.GetBaseUrl(); url.AddPathComponent(catalog.prefix); url.AddPathComponent("namespaces"); @@ -66,7 +63,8 @@ static IRCAPIColumnDefinition ParseColumnDefinition(yyjson_val *column_def) { IRCAPIColumnDefinition result; result.name = IcebergUtils::TryGetStrFromObject(column_def, "name"); result.type_text = IcebergUtils::TryGetStrFromObject(column_def, "type"); - result.precision = (result.type_text == "decimal") ? IcebergUtils::TryGetNumFromObject(column_def, "type_precision") : -1; + result.precision = + (result.type_text == "decimal") ? IcebergUtils::TryGetNumFromObject(column_def, "type_precision") : -1; result.scale = (result.type_text == "decimal") ? IcebergUtils::TryGetNumFromObject(column_def, "type_scale") : -1; result.position = IcebergUtils::TryGetNumFromObject(column_def, "id") - 1; return result; @@ -74,13 +72,11 @@ static IRCAPIColumnDefinition ParseColumnDefinition(yyjson_val *column_def) { static void ParseConfigOptions(yyjson_val *config, case_insensitive_map_t &options) { //! Set of recognized config parameters and the duckdb secret option that matches it. - static const case_insensitive_map_t config_to_option = { - {"s3.access-key-id", "key_id"}, - {"s3.secret-access-key", "secret"}, - {"s3.session-token", "session_token"}, - {"s3.region", "region"}, - {"s3.endpoint", "endpoint"} - }; + static const case_insensitive_map_t config_to_option = {{"s3.access-key-id", "key_id"}, + {"s3.secret-access-key", "secret"}, + {"s3.session-token", "session_token"}, + {"s3.region", "region"}, + {"s3.endpoint", "endpoint"}}; auto config_size = yyjson_obj_size(config); if (!config || config_size == 0) { @@ -126,7 +122,8 @@ static void ParseConfigOptions(yyjson_val *config, case_insensitive_map_t endpoint_it->second = endpoint; } -IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, const string &secret_base_name) { +IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, + const string &table, const string &secret_base_name) { IRCAPITableCredentials result; string api_result = GetTableMetadataCached(context, catalog, schema, table, catalog.secret_name); std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); @@ -175,7 +172,8 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat if (result.storage_credentials.empty() && !config_options.empty()) { //! Only create a secret out of the 'config' if there are no 'storage-credentials' - result.config = make_uniq(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); + result.config = + make_uniq(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); auto &config = *result.config; config.options = config_options; config.name = secret_base_name; @@ -187,7 +185,8 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat return result; } -string IRCAPI::GetToken(ClientContext &context, const string &uri, const string &id, const string &secret, const string &endpoint, const string &scope) { +string IRCAPI::GetToken(ClientContext &context, const string &uri, const string &id, const string &secret, + const string &endpoint, const string &scope) { vector parameters; parameters.push_back(StringUtil::Format("%s=%s", "grant_type", "client_credentials")); parameters.push_back(StringUtil::Format("%s=%s", "client_id", id)); @@ -210,7 +209,8 @@ string IRCAPI::GetToken(ClientContext &context, const string &uri, const string } string token_type = yyjson_get_str(token_type_val); if (!StringUtil::CIEquals(token_type, "bearer")) { - throw NotImplementedException("token_type return value '%s' is not supported, only supports 'bearer' currently.", token_type); + throw NotImplementedException( + "token_type return value '%s' is not supported, only supports 'bearer' currently.", token_type); } string access_token = yyjson_get_str(access_token_val); return access_token; @@ -219,7 +219,7 @@ string IRCAPI::GetToken(ClientContext &context, const string &uri, const string static void populateTableMetadata(IRCAPITable &table, yyjson_val *metadata_root) { table.storage_location = IcebergUtils::TryGetStrFromObject(metadata_root, "metadata-location"); auto *metadata = yyjson_obj_get(metadata_root, "metadata"); - //table_result.table_id = IcebergUtils::TryGetStrFromObject(metadata, "table-uuid"); + // table_result.table_id = IcebergUtils::TryGetStrFromObject(metadata, "table-uuid"); uint64_t current_schema_id = IcebergUtils::TryGetNumFromObject(metadata, "current-schema-id"); auto *schemas = yyjson_obj_get(metadata, "schemas"); @@ -256,8 +256,8 @@ static IRCAPITable createTable(IRCatalog &catalog, const string &schema, const s return table_result; } -IRCAPITable IRCAPI::GetTable(ClientContext &context, - IRCatalog &catalog, const string &schema, const string &table_name, optional_ptr credentials) { +IRCAPITable IRCAPI::GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table_name, + optional_ptr credentials) { IRCAPITable table_result = createTable(catalog, schema, table_name); if (credentials) { string result = GetTableMetadata(context, catalog, schema, table_result.name, catalog.secret_name); @@ -293,7 +293,8 @@ vector IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog size_t idx, max; yyjson_val *table; yyjson_arr_foreach(tables, idx, max, table) { - auto table_result = GetTable(context, catalog, schema, IcebergUtils::TryGetStrFromObject(table, "name"), nullptr); + auto table_result = + GetTable(context, catalog, schema, IcebergUtils::TryGetStrFromObject(table, "name"), nullptr); result.push_back(table_result); } @@ -305,8 +306,7 @@ vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catal auto endpoint_builder = catalog.GetBaseUrl(); endpoint_builder.AddPathComponent(catalog.prefix); endpoint_builder.AddPathComponent("namespaces"); - string api_result = - APIUtils::GetRequest(context, endpoint_builder, catalog.secret_name, catalog.credentials.token); + string api_result = APIUtils::GetRequest(context, endpoint_builder, catalog.secret_name, catalog.credentials.token); std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); //! 'ListNamespacesResponse' @@ -324,26 +324,30 @@ vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catal return result; } -IRCAPISchema IRCAPI::CreateSchema(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials) { +IRCAPISchema IRCAPI::CreateSchema(ClientContext &context, IRCatalog &catalog, const string &internal, + const string &schema, IRCCredentials credentials) { throw NotImplementedException("IRCAPI::Create Schema not Implemented"); } -void IRCAPI::DropSchema(ClientContext &context, const string &internal, const string &schema, IRCCredentials credentials) { +void IRCAPI::DropSchema(ClientContext &context, const string &internal, const string &schema, + IRCCredentials credentials) { throw NotImplementedException("IRCAPI Drop Schema not Implemented"); } -void IRCAPI::DropTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, string &table_name, IRCCredentials credentials) { +void IRCAPI::DropTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, + string &table_name, IRCCredentials credentials) { throw NotImplementedException("IRCAPI Drop Table not Implemented"); } static std::string json_to_string(yyjson_mut_doc *doc, yyjson_write_flag flags = YYJSON_WRITE_PRETTY) { - char *json_chars = yyjson_mut_write(doc, flags, NULL); - std::string json_str(json_chars); - free(json_chars); - return json_str; + char *json_chars = yyjson_mut_write(doc, flags, NULL); + std::string json_str(json_chars); + free(json_chars); + return json_str; } -IRCAPITable IRCAPI::CreateTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials, CreateTableInfo *table_info) { +IRCAPITable IRCAPI::CreateTable(ClientContext &context, IRCatalog &catalog, const string &internal, + const string &schema, IRCCredentials credentials, CreateTableInfo *table_info) { throw NotImplementedException("IRCAPI Create Table not Implemented"); } diff --git a/src/common/api_utils.cpp b/src/common/api_utils.cpp index 5f7e742f..bf51938f 100644 --- a/src/common/api_utils.cpp +++ b/src/common/api_utils.cpp @@ -15,8 +15,10 @@ string APIUtils::GetAwsService(const string host) { return host.substr(0, host.find_first_of('.')); } -string APIUtils::GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &secret_name, const string &token, curl_slist *extra_headers) { - if (StringUtil::StartsWith(endpoint_builder.GetHost(), "glue." ) || StringUtil::StartsWith(endpoint_builder.GetHost(), "s3tables." )) { +string APIUtils::GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, + const string &secret_name, const string &token, curl_slist *extra_headers) { + if (StringUtil::StartsWith(endpoint_builder.GetHost(), "glue.") || + StringUtil::StartsWith(endpoint_builder.GetHost(), "s3tables.")) { auto str = GetRequestAws(context, endpoint_builder, secret_name); return str; } @@ -31,7 +33,7 @@ string APIUtils::GetRequest(ClientContext &context, const IRCEndpointBuilder &en curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); - if(extra_headers) { + if (extra_headers) { curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); } @@ -39,7 +41,8 @@ string APIUtils::GetRequest(ClientContext &context, const IRCEndpointBuilder &en res = curl_easy_perform(curl); curl_easy_cleanup(curl); - DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.Curl.HTTPRequest", "GET %s (curl code '%s')", url, curl_easy_strerror(res)); + DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.Curl.HTTPRequest", "GET %s (curl code '%s')", url, + curl_easy_strerror(res)); if (res != CURLcode::CURLE_OK) { string error = curl_easy_strerror(res); throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); @@ -84,9 +87,8 @@ string APIUtils::GetRequestAws(ClientContext &context, IRCEndpointBuilder endpoi auto encoded = uri.GetURLEncodedPath(); const Aws::Http::URI uri_const = Aws::Http::URI(uri); - auto create_http_req = Aws::Http::CreateHttpRequest(uri_const, - Aws::Http::HttpMethod::HTTP_GET, - Aws::Utils::Stream::DefaultResponseStreamFactoryMethod); + auto create_http_req = Aws::Http::CreateHttpRequest(uri_const, Aws::Http::HttpMethod::HTTP_GET, + Aws::Utils::Stream::DefaultResponseStreamFactoryMethod); std::shared_ptr req(create_http_req); @@ -96,37 +98,37 @@ string APIUtils::GetRequestAws(ClientContext &context, IRCEndpointBuilder endpoi std::shared_ptr provider; provider = std::make_shared( - kv_secret.secret_map["key_id"].GetValue(), - kv_secret.secret_map["secret"].GetValue(), - kv_secret.secret_map["session_token"].IsNull() ? "" : kv_secret.secret_map["session_token"].GetValue() - ); + kv_secret.secret_map["key_id"].GetValue(), kv_secret.secret_map["secret"].GetValue(), + kv_secret.secret_map["session_token"].IsNull() ? "" : kv_secret.secret_map["session_token"].GetValue()); auto signer = make_uniq(provider, service.c_str(), region.c_str()); signer->SignRequest(*req); std::shared_ptr res = MyHttpClient->MakeRequest(req); Aws::Http::HttpResponseCode resCode = res->GetResponseCode(); - DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.Aws.HTTPRequest", "GET %s (response %d) (signed with key_id '%s' for service '%s', in region '%s')", uri.GetURIString(), resCode, kv_secret.secret_map["key_id"].GetValue(), service.c_str(), region.c_str()); + DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.Aws.HTTPRequest", + "GET %s (response %d) (signed with key_id '%s' for service '%s', in region '%s')", + uri.GetURIString(), resCode, kv_secret.secret_map["key_id"].GetValue(), service.c_str(), + region.c_str()); if (resCode == Aws::Http::HttpResponseCode::OK) { Aws::StringStream resBody; resBody << res->GetResponseBody().rdbuf(); return resBody.str(); } else { Aws::StringStream resBody; - resBody << res->GetResponseBody().rdbuf(); - throw IOException("Failed to query %s, http error %d thrown. Message: %s", req->GetUri().GetURIString(true), res->GetResponseCode(), resBody.str()); + resBody << res->GetResponseBody().rdbuf(); + throw IOException("Failed to query %s, http error %d thrown. Message: %s", req->GetUri().GetURIString(true), + res->GetResponseCode(), resBody.str()); } } - size_t APIUtils::RequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { ((std::string *)userp)->append((char *)contents, size * nmemb); return size * nmemb; } - // Look through the the above locations and if one of the files exists, set that as the location curl should use. bool APIUtils::SelectCurlCertPath() { - for (string& caFile : certFileLocations) { + for (string &caFile : certFileLocations) { struct stat buf; if (stat(caFile.c_str(), &buf) == 0) { SELECTED_CURL_CERT_PATH = caFile; @@ -135,61 +137,56 @@ bool APIUtils::SelectCurlCertPath() { return false; } -bool APIUtils::SetCurlCAFileInfo(CURL* curl) { +bool APIUtils::SetCurlCAFileInfo(CURL *curl) { if (!SELECTED_CURL_CERT_PATH.empty()) { curl_easy_setopt(curl, CURLOPT_CAINFO, SELECTED_CURL_CERT_PATH.c_str()); - return true; + return true; } - return false; + return false; } -// Note: every curl object we use should set this, because without it some linux distro's may not find the CA certificate. -void APIUtils::InitializeCurlObject(CURL * curl, const string &token) { - if (!token.empty()) { +// Note: every curl object we use should set this, because without it some linux distro's may not find the CA +// certificate. +void APIUtils::InitializeCurlObject(CURL *curl, const string &token) { + if (!token.empty()) { curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str()); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER); } - SetCurlCAFileInfo(curl); + SetCurlCAFileInfo(curl); } string APIUtils::DeleteRequest(const string &url, const string &token, curl_slist *extra_headers) { - CURL *curl; - CURLcode res; - string readBuffer; - - curl = curl_easy_init(); - if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); - - if(extra_headers) { - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); - } - - InitializeCurlObject(curl, token); - res = curl_easy_perform(curl); - curl_easy_cleanup(curl); - - if (res != CURLcode::CURLE_OK) { - string error = curl_easy_strerror(res); - throw IOException("Curl DELETE Request to '%s' failed with error: '%s'", url, error); - } - - return readBuffer; - } - throw InternalException("Failed to initialize curl"); -} + CURL *curl; + CURLcode res; + string readBuffer; + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, RequestWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &readBuffer); + + if (extra_headers) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, extra_headers); + } + + InitializeCurlObject(curl, token); + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + + if (res != CURLcode::CURLE_OK) { + string error = curl_easy_strerror(res); + throw IOException("Curl DELETE Request to '%s' failed with error: '%s'", url, error); + } + + return readBuffer; + } + throw InternalException("Failed to initialize curl"); +} -string APIUtils::PostRequest( - ClientContext &context, - const string &url, - const string &post_data, - const string &content_type, - const string &token, - curl_slist *extra_headers) { +string APIUtils::PostRequest(ClientContext &context, const string &url, const string &post_data, + const string &content_type, const string &token, curl_slist *extra_headers) { string readBuffer; CURL *curl = curl_easy_init(); if (!curl) { @@ -226,7 +223,8 @@ string APIUtils::PostRequest( curl_slist_free_all(headers); curl_easy_cleanup(curl); - DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.Curl.HTTPRequest", "POST %s (curl code '%s')", url, curl_easy_strerror(res)); + DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.Curl.HTTPRequest", "POST %s (curl code '%s')", url, + curl_easy_strerror(res)); if (res != CURLcode::CURLE_OK) { string error = curl_easy_strerror(res); throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); @@ -234,4 +232,4 @@ string APIUtils::PostRequest( return readBuffer; } -} \ No newline at end of file +} // namespace duckdb \ No newline at end of file diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp index e5074bd7..f31bf026 100644 --- a/src/common/iceberg.cpp +++ b/src/common/iceberg.cpp @@ -8,7 +8,9 @@ namespace duckdb { template -static void ReadManifestEntries(ClientContext &context, const vector &manifests, bool allow_moved_paths, FileSystem &fs, const string &iceberg_path, vector &result) { +static void ReadManifestEntries(ClientContext &context, const vector &manifests, + bool allow_moved_paths, FileSystem &fs, const string &iceberg_path, + vector &result) { for (auto &manifest : manifests) { auto manifest_entry_full_path = allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, manifest.manifest_path, fs) @@ -18,7 +20,8 @@ static void ReadManifestEntries(ClientContext &context, const vector manifests; if (snapshot.iceberg_format_version == 1) { manifests = ScanAvroMetadata("IcebergManifestList", context, manifest_list_full_path); - ReadManifestEntries(context, manifests, options.allow_moved_paths, fs, iceberg_path, ret.entries); + ReadManifestEntries(context, manifests, options.allow_moved_paths, fs, iceberg_path, + ret.entries); } else if (snapshot.iceberg_format_version == 2) { manifests = ScanAvroMetadata("IcebergManifestList", context, manifest_list_full_path); - ReadManifestEntries(context, manifests, options.allow_moved_paths, fs, iceberg_path, ret.entries); + ReadManifestEntries(context, manifests, options.allow_moved_paths, fs, iceberg_path, + ret.entries); } else { throw InvalidInputException("iceberg_format_version %d not handled", snapshot.iceberg_format_version); } @@ -69,9 +74,10 @@ unique_ptr IcebergSnapshot::GetParseInfo(yyjson_doc &metadata return make_uniq(std::move(info)); } -unique_ptr IcebergSnapshot::GetParseInfo(const string &path, FileSystem &fs, const string &metadata_compression_codec) { +unique_ptr IcebergSnapshot::GetParseInfo(const string &path, FileSystem &fs, + const string &metadata_compression_codec) { auto metadata_json = ReadMetaData(path, fs, metadata_compression_codec); - auto* doc = yyjson_read(metadata_json.c_str(), metadata_json.size(), 0); + auto *doc = yyjson_read(metadata_json.c_str(), metadata_json.size(), 0); if (doc == nullptr) { throw InvalidInputException("Fails to parse iceberg metadata from %s", path); } @@ -95,7 +101,8 @@ IcebergSnapshot IcebergSnapshot::GetLatestSnapshot(const string &path, FileSyste return ParseSnapShot(latest_snapshot, info->iceberg_version, info->schema_id, info->schemas, options); } -IcebergSnapshot IcebergSnapshot::GetSnapshotById(const string &path, FileSystem &fs, idx_t snapshot_id, const IcebergOptions &options) { +IcebergSnapshot IcebergSnapshot::GetSnapshotById(const string &path, FileSystem &fs, idx_t snapshot_id, + const IcebergOptions &options) { auto info = GetParseInfo(path, fs, options.metadata_compression_codec); auto snapshot = FindSnapshotByIdInternal(info->snapshots, snapshot_id); @@ -106,7 +113,8 @@ IcebergSnapshot IcebergSnapshot::GetSnapshotById(const string &path, FileSystem return ParseSnapShot(snapshot, info->iceberg_version, info->schema_id, info->schemas, options); } -IcebergSnapshot IcebergSnapshot::GetSnapshotByTimestamp(const string &path, FileSystem &fs, timestamp_t timestamp, const IcebergOptions &options) { +IcebergSnapshot IcebergSnapshot::GetSnapshotByTimestamp(const string &path, FileSystem &fs, timestamp_t timestamp, + const IcebergOptions &options) { auto info = GetParseInfo(path, fs, options.metadata_compression_codec); auto snapshot = FindSnapshotByIdTimestampInternal(info->snapshots, timestamp); @@ -119,26 +127,28 @@ IcebergSnapshot IcebergSnapshot::GetSnapshotByTimestamp(const string &path, File // Function to generate a metadata file url from version and format string // default format is "v%s%s.metadata.json" -> v00###-xxxxxxxxx-.gz.metadata.json" -static string GenerateMetaDataUrl(FileSystem &fs, const string &meta_path, string &table_version, const IcebergOptions &options) { +static string GenerateMetaDataUrl(FileSystem &fs, const string &meta_path, string &table_version, + const IcebergOptions &options) { // TODO: Need to URL Encode table_version string compression_suffix = ""; string url; if (options.metadata_compression_codec == "gzip") { compression_suffix = ".gz"; } - for(auto try_format : StringUtil::Split(options.version_name_format, ',')) { + for (auto try_format : StringUtil::Split(options.version_name_format, ',')) { url = fs.JoinPath(meta_path, StringUtil::Format(try_format, table_version, compression_suffix)); - if(fs.FileExists(url)) { + if (fs.FileExists(url)) { return url; } } throw IOException( - "Iceberg metadata file not found for table version '%s' using '%s' compression and format(s): '%s'", table_version, options.metadata_compression_codec, options.version_name_format); + "Iceberg metadata file not found for table version '%s' using '%s' compression and format(s): '%s'", + table_version, options.metadata_compression_codec, options.version_name_format); } - -string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &path, FileSystem &fs, const IcebergOptions &options) { +string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &path, FileSystem &fs, + const IcebergOptions &options) { string version_hint; string meta_path = fs.JoinPath(path, "metadata"); @@ -148,7 +158,7 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa // We've been given a real metadata path. Nothing else to do. return path; } - if(StringUtil::EndsWith(table_version, ".text")||StringUtil::EndsWith(table_version, ".txt")) { + if (StringUtil::EndsWith(table_version, ".text") || StringUtil::EndsWith(table_version, ".txt")) { // We were given a hint filename version_hint = GetTableVersionFromHint(meta_path, fs, table_version); return GenerateMetaDataUrl(fs, meta_path, version_hint, options); @@ -165,7 +175,11 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa } if (!UnsafeVersionGuessingEnabled(context)) { // Make sure we're allowed to guess versions - throw IOException("Failed to read iceberg table. No version was provided and no version-hint could be found, globbing the filesystem to locate the latest version is disabled by default as this is considered unsafe and could result in reading uncommitted data. To enable this use 'SET %s = true;'", VERSION_GUESSING_CONFIG_VARIABLE); + throw IOException( + "Failed to read iceberg table. No version was provided and no version-hint could be found, globbing the " + "filesystem to locate the latest version is disabled by default as this is considered unsafe and could " + "result in reading uncommitted data. To enable this use 'SET %s = true;'", + VERSION_GUESSING_CONFIG_VARIABLE); } // We are allowed to guess to guess from file paths @@ -179,7 +193,8 @@ string IcebergSnapshot::ReadMetaData(const string &path, FileSystem &fs, const s return IcebergUtils::FileToString(path, fs); } -IcebergSnapshot IcebergSnapshot::ParseSnapShot(yyjson_val *snapshot, idx_t iceberg_format_version, idx_t schema_id, vector &schemas, const IcebergOptions &options) { +IcebergSnapshot IcebergSnapshot::ParseSnapShot(yyjson_val *snapshot, idx_t iceberg_format_version, idx_t schema_id, + vector &schemas, const IcebergOptions &options) { IcebergSnapshot ret; auto snapshot_tag = yyjson_get_type(snapshot); if (snapshot_tag != YYJSON_TYPE_OBJ) { @@ -203,7 +218,8 @@ IcebergSnapshot IcebergSnapshot::ParseSnapShot(yyjson_val *snapshot, idx_t icebe return ret; } -string IcebergSnapshot::GetTableVersionFromHint(const string &meta_path, FileSystem &fs, string version_file = DEFAULT_VERSION_HINT_FILE) { +string IcebergSnapshot::GetTableVersionFromHint(const string &meta_path, FileSystem &fs, + string version_file = DEFAULT_VERSION_HINT_FILE) { auto version_file_path = fs.JoinPath(meta_path, version_file); auto version_file_content = IcebergUtils::FileToString(version_file_path, fs); @@ -222,7 +238,6 @@ bool IcebergSnapshot::UnsafeVersionGuessingEnabled(ClientContext &context) { return !result.IsNull() && result.GetValue(); } - string IcebergSnapshot::GuessTableVersion(const string &meta_path, FileSystem &fs, const IcebergOptions &options) { string selected_metadata; string version_pattern = "*"; // TODO: Different "table_version" strings could customize this @@ -235,27 +250,26 @@ string IcebergSnapshot::GuessTableVersion(const string &meta_path, FileSystem &f compression_suffix = ".gz"; } - for(auto try_format : StringUtil::Split(version_format, ',')) { + for (auto try_format : StringUtil::Split(version_format, ',')) { auto glob_pattern = StringUtil::Format(try_format, version_pattern, compression_suffix); auto found_versions = fs.Glob(fs.JoinPath(meta_path, glob_pattern)); - if(found_versions.size() > 0) { + if (found_versions.size() > 0) { selected_metadata = PickTableVersion(found_versions, version_pattern, glob_pattern); - if(!selected_metadata.empty()) { // Found one + if (!selected_metadata.empty()) { // Found one return selected_metadata; } } } - throw IOException( - "Could not guess Iceberg table version using '%s' compression and format(s): '%s'", - metadata_compression_codec, version_format); + throw IOException("Could not guess Iceberg table version using '%s' compression and format(s): '%s'", + metadata_compression_codec, version_format); } string IcebergSnapshot::PickTableVersion(vector &found_metadata, string &version_pattern, string &glob) { // TODO: Different "table_version" strings could customize this // For now: just sort the versions and take the largest - if(!found_metadata.empty()) { + if (!found_metadata.empty()) { std::sort(found_metadata.begin(), found_metadata.end()); return found_metadata.back(); } else { diff --git a/src/common/url_utils.cpp b/src/common/url_utils.cpp index d67e4d4d..6c22a513 100644 --- a/src/common/url_utils.cpp +++ b/src/common/url_utils.cpp @@ -74,4 +74,4 @@ string IRCEndpointBuilder::GetURL() const { return ret; } -} +} // namespace duckdb diff --git a/src/common/utils.cpp b/src/common/utils.cpp index b67a13f0..0377185c 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -16,9 +16,9 @@ string IcebergUtils::FileToString(const string &path, FileSystem &fs) { // Function to decompress a gz file content string string IcebergUtils::GzFileToString(const string &path, FileSystem &fs) { - // Initialize zlib variables - string gzipped_string = FileToString(path, fs); - return GZipFileSystem::UncompressGZIPString(gzipped_string); + // Initialize zlib variables + string gzipped_string = FileToString(path, fs); + return GZipFileSystem::UncompressGZIPString(gzipped_string); } string IcebergUtils::GetFullPath(const string &iceberg_path, const string &relative_file_path, FileSystem &fs) { @@ -60,8 +60,7 @@ string IcebergUtils::TryGetStrFromObject(yyjson_val *obj, const string &field) { } template -static TYPE TemplatedTryGetYYJson(yyjson_val *obj, const string &field, TYPE default_val, - bool fail_on_missing = true) { +static TYPE TemplatedTryGetYYJson(yyjson_val *obj, const string &field, TYPE default_val, bool fail_on_missing = true) { auto val = yyjson_obj_get(obj, field.c_str()); if (val && yyjson_get_type(val) == TYPE_NUM) { return get_function(val); @@ -72,19 +71,16 @@ static TYPE TemplatedTryGetYYJson(yyjson_val *obj, const string &field, TYPE def } uint64_t IcebergUtils::TryGetNumFromObject(yyjson_val *obj, const string &field, bool fail_on_missing, - uint64_t default_val) { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); + uint64_t default_val) { + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } -bool IcebergUtils::TryGetBoolFromObject(yyjson_val *obj, const string &field, bool fail_on_missing, - bool default_val) { - return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); +bool IcebergUtils::TryGetBoolFromObject(yyjson_val *obj, const string &field, bool fail_on_missing, bool default_val) { + return TemplatedTryGetYYJson(obj, field, default_val, fail_on_missing); } string IcebergUtils::TryGetStrFromObject(yyjson_val *obj, const string &field, bool fail_on_missing, - const char *default_val) { + const char *default_val) { return TemplatedTryGetYYJson(obj, field, default_val, - fail_on_missing); + fail_on_missing); } } // namespace duckdb \ No newline at end of file diff --git a/src/iceberg_extension.cpp b/src/iceberg_extension.cpp index 2ccf3376..b20a8a93 100644 --- a/src/iceberg_extension.cpp +++ b/src/iceberg_extension.cpp @@ -30,12 +30,8 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context for (const auto &named_param : input.options) { auto lower_name = StringUtil::Lower(named_param.first); - if (lower_name == "key_id" || - lower_name == "secret" || - lower_name == "endpoint" || - lower_name == "aws_region" || - lower_name == "oauth2_scope" || - lower_name == "oauth2_server_uri") { + if (lower_name == "key_id" || lower_name == "secret" || lower_name == "endpoint" || + lower_name == "aws_region" || lower_name == "oauth2_scope" || lower_name == "oauth2_server_uri") { result->secret_map[lower_name] = named_param.second.ToString(); } else { throw InternalException("Unknown named parameter passed to CreateIRCSecretFunction: " + lower_name); @@ -43,14 +39,10 @@ static unique_ptr CreateCatalogSecretFunction(ClientContext &context } // Get token from catalog - result->secret_map["token"] = IRCAPI::GetToken( - context, - result->secret_map["oauth2_server_uri"].ToString(), - result->secret_map["key_id"].ToString(), - result->secret_map["secret"].ToString(), - result->secret_map["endpoint"].ToString(), - result->secret_map["oauth2_scope"].ToString() - ); + result->secret_map["token"] = + IRCAPI::GetToken(context, result->secret_map["oauth2_server_uri"].ToString(), + result->secret_map["key_id"].ToString(), result->secret_map["secret"].ToString(), + result->secret_map["endpoint"].ToString(), result->secret_map["oauth2_scope"].ToString()); //! Set redact keys result->redact_keys = {"token", "client_id", "client_secret"}; @@ -78,12 +70,13 @@ static bool SanityCheckGlueWarehouse(string warehouse) { if (bucket_sep_correct) { return true; } - throw IOException("Invalid Glue Catalog Format: '" + warehouse + "'. Expected ':s3tablescatalog/"); + throw IOException("Invalid Glue Catalog Format: '" + warehouse + + "'. Expected ':s3tablescatalog/"); } static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, - AttachedDatabase &db, const string &name, AttachInfo &info, - AccessMode access_mode) { + AttachedDatabase &db, const string &name, AttachInfo &info, + AccessMode access_mode) { IRCCredentials credentials; IRCEndpointBuilder endpoint_builder; @@ -125,7 +118,9 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in if (oauth2_server_uri.empty()) { //! If no oauth2_server_uri is provided, default to the (deprecated) REST API endpoint for it - DUCKDB_LOG_WARN(context, "iceberg", "'oauth2_server_uri' is not set, defaulting to deprecated '{endpoint}/v1/oauth/tokens' oauth2_server_uri"); + DUCKDB_LOG_WARN( + context, "iceberg", + "'oauth2_server_uri' is not set, defaulting to deprecated '{endpoint}/v1/oauth/tokens' oauth2_server_uri"); oauth2_server_uri = StringUtil::Format("%s/v1/oauth/tokens", endpoint); } auto catalog_type = ICEBERG_CATALOG_TYPE::INVALID; @@ -142,11 +137,12 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // if there is no secret, an error will be thrown auto secret_entry = IRCatalog::GetSecret(context, secret_name); - auto kv_secret = dynamic_cast(*secret_entry->secret); + auto kv_secret = dynamic_cast(*secret_entry->secret); auto region = kv_secret.TryGetValue("region"); if (region.IsNull()) { - throw IOException("Assumed catalog secret " + secret_entry->secret->GetName() + " for catalog " + name + " does not have a region"); + throw IOException("Assumed catalog secret " + secret_entry->secret->GetName() + " for catalog " + name + + " does not have a region"); } switch (catalog_type) { case ICEBERG_CATALOG_TYPE::AWS_S3TABLES: { @@ -189,8 +185,8 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in if (!secret_entry) { throw IOException("No secret found to use with catalog " + name); } - // secret found - read data - const auto &kv_secret = dynamic_cast(*secret_entry->secret); + // secret found - read data + const auto &kv_secret = dynamic_cast(*secret_entry->secret); Value key_val = kv_secret.TryGetValue("key_id"); Value secret_val = kv_secret.TryGetValue("secret"); CreateSecretInput create_secret_input; @@ -241,12 +237,10 @@ static void LoadInternal(DatabaseInstance &instance) { auto &config = DBConfig::GetConfig(instance); - config.AddExtensionOption( - "unsafe_enable_version_guessing", - "Enable globbing the filesystem (if possible) to find the latest version metadata. This could result in reading an uncommitted version.", - LogicalType::BOOLEAN, - Value::BOOLEAN(false) - ); + config.AddExtensionOption("unsafe_enable_version_guessing", + "Enable globbing the filesystem (if possible) to find the latest version metadata. This " + "could result in reading an uncommitted version.", + LogicalType::BOOLEAN, Value::BOOLEAN(false)); // Iceberg Table Functions for (auto &fun : IcebergFunctions::GetTableFunctions(instance)) { diff --git a/src/iceberg_functions/iceberg_metadata.cpp b/src/iceberg_functions/iceberg_metadata.cpp index 7d05bcc1..3b630133 100644 --- a/src/iceberg_functions/iceberg_metadata.cpp +++ b/src/iceberg_functions/iceberg_metadata.cpp @@ -55,7 +55,7 @@ static unique_ptr IcebergMetaDataBind(ClientContext &context, Tabl auto iceberg_path = input.inputs[0].ToString(); IcebergOptions options; - + for (auto &kv : input.named_parameters) { auto loption = StringUtil::Lower(kv.first); if (loption == "allow_moved_paths") { @@ -75,10 +75,11 @@ static unique_ptr IcebergMetaDataBind(ClientContext &context, Tabl IcebergSnapshot snapshot_to_scan; if (input.inputs.size() > 1) { if (input.inputs[1].type() == LogicalType::UBIGINT) { - snapshot_to_scan = IcebergSnapshot::GetSnapshotById(iceberg_meta_path, fs, input.inputs[1].GetValue(), options); - } else if (input.inputs[1].type() == LogicalType::TIMESTAMP) { snapshot_to_scan = - IcebergSnapshot::GetSnapshotByTimestamp(iceberg_meta_path, fs, input.inputs[1].GetValue(), options); + IcebergSnapshot::GetSnapshotById(iceberg_meta_path, fs, input.inputs[1].GetValue(), options); + } else if (input.inputs[1].type() == LogicalType::TIMESTAMP) { + snapshot_to_scan = IcebergSnapshot::GetSnapshotByTimestamp( + iceberg_meta_path, fs, input.inputs[1].GetValue(), options); } else { throw InvalidInputException("Unknown argument type in IcebergScanBindReplace."); } @@ -86,8 +87,7 @@ static unique_ptr IcebergMetaDataBind(ClientContext &context, Tabl snapshot_to_scan = IcebergSnapshot::GetLatestSnapshot(iceberg_meta_path, fs, options); } - ret->iceberg_table = - make_uniq(IcebergTable::Load(iceberg_path, snapshot_to_scan, context, options)); + ret->iceberg_table = make_uniq(IcebergTable::Load(iceberg_path, snapshot_to_scan, context, options)); auto manifest_types = IcebergManifest::Types(); return_types.insert(return_types.end(), manifest_types.begin(), manifest_types.end()); diff --git a/src/iceberg_functions/iceberg_multi_file_reader.cpp b/src/iceberg_functions/iceberg_multi_file_reader.cpp index 45a40825..91492df9 100644 --- a/src/iceberg_functions/iceberg_multi_file_reader.cpp +++ b/src/iceberg_functions/iceberg_multi_file_reader.cpp @@ -100,27 +100,29 @@ string IcebergMultiFileList::GetFile(idx_t file_id) { } auto &manifest = *current_data_manifest; auto manifest_entry_full_path = options.allow_moved_paths - ? IcebergUtils::GetFullPath(iceberg_path, manifest.manifest_path, fs) - : manifest.manifest_path; + ? IcebergUtils::GetFullPath(iceberg_path, manifest.manifest_path, fs) + : manifest.manifest_path; auto scan = make_uniq("IcebergManifest", context, manifest_entry_full_path); data_manifest_entry_reader->Initialize(std::move(scan)); } idx_t remaining = (file_id + 1) - data_files.size(); - data_manifest_entry_reader->ReadEntries(remaining, [&data_files, &entry_producer](DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input) { - return entry_producer(chunk, offset, count, input, data_files); - }); + data_manifest_entry_reader->ReadEntries( + remaining, [&data_files, &entry_producer](DataChunk &chunk, idx_t offset, idx_t count, + const ManifestReaderInput &input) { + return entry_producer(chunk, offset, count, input, data_files); + }); if (data_manifest_entry_reader->Finished()) { current_data_manifest++; continue; } } - #ifdef DEBUG +#ifdef DEBUG for (auto &entry : data_files) { D_ASSERT(entry.content == IcebergManifestEntryContentType::DATA); D_ASSERT(entry.status != IcebergManifestEntryStatusType::DELETED); } - #endif +#endif if (file_id >= data_files.size()) { return string(); @@ -170,25 +172,32 @@ void IcebergMultiFileList::InitializeFiles() { //! Set up the manifest + manifest entry readers if (snapshot.iceberg_format_version == 1) { - data_manifest_entry_reader = make_uniq(IcebergManifestEntryV1::PopulateNameMapping, IcebergManifestEntryV1::VerifySchema); - delete_manifest_entry_reader = make_uniq(IcebergManifestEntryV1::PopulateNameMapping, IcebergManifestEntryV1::VerifySchema); + data_manifest_entry_reader = make_uniq(IcebergManifestEntryV1::PopulateNameMapping, + IcebergManifestEntryV1::VerifySchema); + delete_manifest_entry_reader = make_uniq(IcebergManifestEntryV1::PopulateNameMapping, + IcebergManifestEntryV1::VerifySchema); delete_manifest_entry_reader->skip_deleted = true; data_manifest_entry_reader->skip_deleted = true; - manifest_reader = make_uniq(IcebergManifestV1::PopulateNameMapping, IcebergManifestV1::VerifySchema); + manifest_reader = + make_uniq(IcebergManifestV1::PopulateNameMapping, IcebergManifestV1::VerifySchema); manifest_producer = IcebergManifestV1::ProduceEntries; entry_producer = IcebergManifestEntryV1::ProduceEntries; } else if (snapshot.iceberg_format_version == 2) { - data_manifest_entry_reader = make_uniq(IcebergManifestEntryV2::PopulateNameMapping, IcebergManifestEntryV2::VerifySchema); - delete_manifest_entry_reader = make_uniq(IcebergManifestEntryV2::PopulateNameMapping, IcebergManifestEntryV2::VerifySchema); + data_manifest_entry_reader = make_uniq(IcebergManifestEntryV2::PopulateNameMapping, + IcebergManifestEntryV2::VerifySchema); + delete_manifest_entry_reader = make_uniq(IcebergManifestEntryV2::PopulateNameMapping, + IcebergManifestEntryV2::VerifySchema); delete_manifest_entry_reader->skip_deleted = true; data_manifest_entry_reader->skip_deleted = true; - manifest_reader = make_uniq(IcebergManifestV2::PopulateNameMapping, IcebergManifestV2::VerifySchema); + manifest_reader = + make_uniq(IcebergManifestV2::PopulateNameMapping, IcebergManifestV2::VerifySchema); manifest_producer = IcebergManifestV2::ProduceEntries; entry_producer = IcebergManifestEntryV2::ProduceEntries; } else { - throw InvalidInputException("Reading from Iceberg version %d is not supported yet", snapshot.iceberg_format_version); + throw InvalidInputException("Reading from Iceberg version %d is not supported yet", + snapshot.iceberg_format_version); } // Read the manifest list, we need all the manifests to determine if we've seen all deletes @@ -200,9 +209,11 @@ void IcebergMultiFileList::InitializeFiles() { vector all_manifests; while (!manifest_reader->Finished()) { - manifest_reader->ReadEntries(STANDARD_VECTOR_SIZE, [&all_manifests, manifest_producer](DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input) { - return manifest_producer(chunk, offset, count, input, all_manifests); - }); + manifest_reader->ReadEntries(STANDARD_VECTOR_SIZE, + [&all_manifests, manifest_producer](DataChunk &chunk, idx_t offset, idx_t count, + const ManifestReaderInput &input) { + return manifest_producer(chunk, offset, count, input, all_manifests); + }); } for (auto &manifest : all_manifests) { @@ -279,33 +290,32 @@ void IcebergMultiFileReader::CreateColumnMapping(const string &file_name, reader_data, bind_data, initial_file, global_state_p); auto &global_state = global_state_p->Cast(); - // Check if the file_row_number column is an "extra_column" which is not part of the projection + // Check if the file_row_number column is an "extra_column" which is not part of the projection if (!global_state.file_row_number_idx.IsValid()) { return; } auto file_row_number_idx = global_state.file_row_number_idx.GetIndex(); - if (file_row_number_idx >= global_column_ids.size()) { - // Build the name map - case_insensitive_map_t name_map; - for (idx_t col_idx = 0; col_idx < local_columns.size(); col_idx++) { - name_map[local_columns[col_idx].name] = col_idx; - } - - // Lookup the required column in the local map - auto entry = name_map.find("file_row_number"); - if (entry == name_map.end()) { - throw IOException("Failed to find the file_row_number column"); - } - - // Register the column to be scanned from this file - reader_data.column_ids.push_back(entry->second); - reader_data.column_indexes.emplace_back(entry->second); - reader_data.column_mapping.push_back(file_row_number_idx); - } - - // This may have changed: update it - reader_data.empty_columns = reader_data.column_ids.empty(); + if (file_row_number_idx >= global_column_ids.size()) { + // Build the name map + case_insensitive_map_t name_map; + for (idx_t col_idx = 0; col_idx < local_columns.size(); col_idx++) { + name_map[local_columns[col_idx].name] = col_idx; + } + + // Lookup the required column in the local map + auto entry = name_map.find("file_row_number"); + if (entry == name_map.end()) { + throw IOException("Failed to find the file_row_number column"); + } + + // Register the column to be scanned from this file + reader_data.column_ids.push_back(entry->second); + reader_data.column_indexes.emplace_back(entry->second); + reader_data.column_mapping.push_back(file_row_number_idx); + } + // This may have changed: update it + reader_data.empty_columns = reader_data.column_ids.empty(); } unique_ptr @@ -378,13 +388,14 @@ IcebergMultiFileReader::InitializeGlobalState(ClientContext &context, const Mult } void IcebergMultiFileReader::FinalizeBind(const MultiFileReaderOptions &file_options, - const MultiFileReaderBindData &options, const string &filename, - const vector &local_columns, - const vector &global_columns, - const vector &global_column_ids, MultiFileReaderData &reader_data, - ClientContext &context, optional_ptr global_state) { - MultiFileReader::FinalizeBind(file_options, options, filename, local_columns, global_columns, - global_column_ids, reader_data, context, global_state); + const MultiFileReaderBindData &options, const string &filename, + const vector &local_columns, + const vector &global_columns, + const vector &global_column_ids, + MultiFileReaderData &reader_data, ClientContext &context, + optional_ptr global_state) { + MultiFileReader::FinalizeBind(file_options, options, filename, local_columns, global_columns, global_column_ids, + reader_data, context, global_state); return; } @@ -490,15 +501,17 @@ void IcebergMultiFileList::ProcessDeletes() const { } auto &manifest = *current_delete_manifest; auto manifest_entry_full_path = options.allow_moved_paths - ? IcebergUtils::GetFullPath(iceberg_path, manifest.manifest_path, fs) - : manifest.manifest_path; + ? IcebergUtils::GetFullPath(iceberg_path, manifest.manifest_path, fs) + : manifest.manifest_path; auto scan = make_uniq("IcebergManifest", context, manifest_entry_full_path); delete_manifest_entry_reader->Initialize(std::move(scan)); } - delete_manifest_entry_reader->ReadEntries(STANDARD_VECTOR_SIZE, [&delete_files, &entry_producer](DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input) { - return entry_producer(chunk, offset, count, input, delete_files); - }); + delete_manifest_entry_reader->ReadEntries( + STANDARD_VECTOR_SIZE, [&delete_files, &entry_producer](DataChunk &chunk, idx_t offset, idx_t count, + const ManifestReaderInput &input) { + return entry_producer(chunk, offset, count, input, delete_files); + }); if (delete_manifest_entry_reader->Finished()) { current_delete_manifest++; @@ -506,12 +519,12 @@ void IcebergMultiFileList::ProcessDeletes() const { } } - #ifdef DEBUG +#ifdef DEBUG for (auto &entry : data_files) { D_ASSERT(entry.content == IcebergManifestEntryContentType::DATA); D_ASSERT(entry.status != IcebergManifestEntryStatusType::DELETED); } - #endif +#endif for (auto &entry : delete_files) { ScanDeleteFile(entry.file_path); diff --git a/src/iceberg_functions/iceberg_snapshots.cpp b/src/iceberg_functions/iceberg_snapshots.cpp index d754357e..fe897dd7 100644 --- a/src/iceberg_functions/iceberg_snapshots.cpp +++ b/src/iceberg_functions/iceberg_snapshots.cpp @@ -30,8 +30,7 @@ struct IcebergSnapshotGlobalTableFunctionState : public GlobalTableFunctionState FileSystem &fs = FileSystem::GetFileSystem(context); - auto iceberg_meta_path = - IcebergSnapshot::GetMetaDataPath(context, bind_data.filename, fs, bind_data.options); + auto iceberg_meta_path = IcebergSnapshot::GetMetaDataPath(context, bind_data.filename, fs, bind_data.options); global_state->metadata_file = IcebergSnapshot::ReadMetaData(iceberg_meta_path, fs, bind_data.options.metadata_compression_codec); global_state->metadata_doc = diff --git a/src/iceberg_manifest.cpp b/src/iceberg_manifest.cpp index 3320d0d3..208b02e1 100644 --- a/src/iceberg_manifest.cpp +++ b/src/iceberg_manifest.cpp @@ -6,11 +6,13 @@ namespace duckdb { -static void ManifestNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec) { +static void ManifestNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec) { name_to_vec[name] = ColumnIndex(column_id); } -idx_t IcebergManifestV1::ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, vector &result) { +idx_t IcebergManifestV1::ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, + vector &result) { auto &name_to_vec = input.name_to_vec; auto manifest_path = FlatVector::GetData(chunk.data[name_to_vec.at("manifest_path").GetPrimaryIndex()]); @@ -34,16 +36,18 @@ bool IcebergManifestV1::VerifySchema(const case_insensitive_map_t & return true; } -void IcebergManifestV1::PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec) { +void IcebergManifestV1::PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec) { ManifestNameMapping(column_id, type, name, name_to_vec); } idx_t IcebergManifestV2::ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, - vector &result) { + vector &result) { auto &name_to_vec = input.name_to_vec; auto manifest_path = FlatVector::GetData(chunk.data[name_to_vec.at("manifest_path").GetPrimaryIndex()]); auto content = FlatVector::GetData(chunk.data[name_to_vec.at("content").GetPrimaryIndex()]); - auto sequence_number = FlatVector::GetData(chunk.data[name_to_vec.at("sequence_number").GetPrimaryIndex()]); + auto sequence_number = + FlatVector::GetData(chunk.data[name_to_vec.at("sequence_number").GetPrimaryIndex()]); for (idx_t i = 0; i < count; i++) { idx_t index = i + offset; @@ -68,13 +72,15 @@ bool IcebergManifestV2::VerifySchema(const case_insensitive_map_t & return true; } -void IcebergManifestV2::PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec) { +void IcebergManifestV2::PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec) { ManifestNameMapping(column_id, type, name, name_to_vec); } //! Iceberg Manifest Entry scan routines -static void EntryNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec) { +static void EntryNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec) { auto lname = StringUtil::Lower(name); if (lname != "data_file") { name_to_vec[lname] = ColumnIndex(column_id); @@ -93,8 +99,8 @@ static void EntryNameMapping(idx_t column_id, const LogicalType &type, const str } } -idx_t IcebergManifestEntryV1::ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, - vector &result) { +idx_t IcebergManifestEntryV1::ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, + const ManifestReaderInput &input, vector &result) { auto &name_to_vec = input.name_to_vec; auto status = FlatVector::GetData(chunk.data[name_to_vec.at("status").GetPrimaryIndex()]); @@ -105,8 +111,10 @@ idx_t IcebergManifestEntryV1::ProduceEntries(DataChunk &chunk, idx_t offset, idx D_ASSERT(name_to_vec.at("record_count").GetPrimaryIndex()); auto file_path = FlatVector::GetData(*child_entries[file_path_idx.GetChildIndex(0).GetPrimaryIndex()]); - auto file_format = FlatVector::GetData(*child_entries[name_to_vec.at("file_format").GetChildIndex(0).GetPrimaryIndex()]); - auto record_count = FlatVector::GetData(*child_entries[name_to_vec.at("record_count").GetChildIndex(0).GetPrimaryIndex()]); + auto file_format = + FlatVector::GetData(*child_entries[name_to_vec.at("file_format").GetChildIndex(0).GetPrimaryIndex()]); + auto record_count = + FlatVector::GetData(*child_entries[name_to_vec.at("record_count").GetChildIndex(0).GetPrimaryIndex()]); idx_t produced = 0; for (idx_t i = 0; i < count; i++) { @@ -146,12 +154,13 @@ bool IcebergManifestEntryV1::VerifySchema(const case_insensitive_map_t &name_to_vec) { +void IcebergManifestEntryV1::PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec) { EntryNameMapping(column_id, type, name, name_to_vec); } -idx_t IcebergManifestEntryV2::ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, - vector &result) { +idx_t IcebergManifestEntryV2::ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, + const ManifestReaderInput &input, vector &result) { auto &name_to_vec = input.name_to_vec; auto status = FlatVector::GetData(chunk.data[name_to_vec.at("status").GetPrimaryIndex()]); @@ -162,10 +171,13 @@ idx_t IcebergManifestEntryV2::ProduceEntries(DataChunk &chunk, idx_t offset, idx D_ASSERT(name_to_vec.at("record_count").GetPrimaryIndex() == data_file_idx); D_ASSERT(name_to_vec.at("content").GetPrimaryIndex() == data_file_idx); - auto content = FlatVector::GetData(*child_entries[name_to_vec.at("content").GetChildIndex(0).GetPrimaryIndex()]); + auto content = + FlatVector::GetData(*child_entries[name_to_vec.at("content").GetChildIndex(0).GetPrimaryIndex()]); auto file_path = FlatVector::GetData(*child_entries[file_path_idx.GetChildIndex(0).GetPrimaryIndex()]); - auto file_format = FlatVector::GetData(*child_entries[name_to_vec.at("file_format").GetChildIndex(0).GetPrimaryIndex()]); - auto record_count = FlatVector::GetData(*child_entries[name_to_vec.at("record_count").GetChildIndex(0).GetPrimaryIndex()]); + auto file_format = + FlatVector::GetData(*child_entries[name_to_vec.at("file_format").GetChildIndex(0).GetPrimaryIndex()]); + auto record_count = + FlatVector::GetData(*child_entries[name_to_vec.at("record_count").GetChildIndex(0).GetPrimaryIndex()]); idx_t produced = 0; for (idx_t i = 0; i < count; i++) { @@ -199,7 +211,8 @@ bool IcebergManifestEntryV2::VerifySchema(const case_insensitive_map_t &name_to_vec) { +void IcebergManifestEntryV2::PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec) { EntryNameMapping(column_id, type, name, name_to_vec); } diff --git a/src/include/api_utils.hpp b/src/include/api_utils.hpp index bd5ac038..3861af58 100644 --- a/src/include/api_utils.hpp +++ b/src/include/api_utils.hpp @@ -24,17 +24,16 @@ static string SELECTED_CURL_CERT_PATH = ""; // place curl will look. But not every distro has this file in the same location, so we search a // number of common locations and use the first one we find. static string certFileLocations[] = { - // Arch, Debian-based, Gentoo - "/etc/ssl/certs/ca-certificates.crt", - // RedHat 7 based - "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", - // Redhat 6 based - "/etc/pki/tls/certs/ca-bundle.crt", - // OpenSUSE - "/etc/ssl/ca-bundle.pem", - // Alpine - "/etc/ssl/cert.pem" -}; + // Arch, Debian-based, Gentoo + "/etc/ssl/certs/ca-certificates.crt", + // RedHat 7 based + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + // Redhat 6 based + "/etc/pki/tls/certs/ca-bundle.crt", + // OpenSUSE + "/etc/ssl/ca-bundle.pem", + // Alpine + "/etc/ssl/cert.pem"}; class APIUtils { public: @@ -43,20 +42,16 @@ class APIUtils { static string GetRequestAws(ClientContext &context, IRCEndpointBuilder endpoint_builder, const string &secret_name); static string GetAwsRegion(const string host); static string GetAwsService(const string host); - static string GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &secret_name, const string &token = "", curl_slist *extra_headers = NULL); + static string GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, + const string &secret_name, const string &token = "", curl_slist *extra_headers = NULL); static string DeleteRequest(const string &url, const string &token = "", curl_slist *extra_headers = NULL); - static void InitializeCurlObject(CURL * curl, const string &token); - static bool SetCurlCAFileInfo(CURL* curl); + static void InitializeCurlObject(CURL *curl, const string &token); + static bool SetCurlCAFileInfo(CURL *curl); static bool SelectCurlCertPath(); static size_t RequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp); - static string PostRequest( - ClientContext &context, - const string &url, - const string &post_data, - const string &content_type = "x-www-form-urlencoded", - const string &token = "", - curl_slist *extra_headers = NULL); - + static string PostRequest(ClientContext &context, const string &url, const string &post_data, + const string &content_type = "x-www-form-urlencoded", const string &token = "", + curl_slist *extra_headers = NULL); }; } // namespace duckdb \ No newline at end of file diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 81a1df84..0e8001ff 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -46,19 +46,26 @@ class IRCAPI { public: static const string API_VERSION_1; - //! WARNING: not thread-safe. To be called once on extension initialization - static void InitializeCurl(); + //! WARNING: not thread-safe. To be called once on extension initialization + static void InitializeCurl(); - static IRCAPITableCredentials GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table, const string &secret_base_name); + static IRCAPITableCredentials GetTableCredentials(ClientContext &context, IRCatalog &catalog, const string &schema, + const string &table, const string &secret_base_name); static vector GetCatalogs(ClientContext &context, IRCatalog &catalog, IRCCredentials credentials); static vector GetTables(ClientContext &context, IRCatalog &catalog, const string &schema); - static IRCAPITable GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table_name, optional_ptr credentials = nullptr); + static IRCAPITable GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, + const string &table_name, optional_ptr credentials = nullptr); static vector GetSchemas(ClientContext &context, IRCatalog &catalog, IRCCredentials credentials); - static string GetToken(ClientContext &context, const string &uri, const string &id, const string &secret, const string &endpoint, const string &scope); - static IRCAPISchema CreateSchema(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials); - static void DropSchema(ClientContext &context, const string &internal, const string &schema, IRCCredentials credentials); - static IRCAPITable CreateTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, IRCCredentials credentials, CreateTableInfo *table_info); - static void DropTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, string &table_name, IRCCredentials credentials); + static string GetToken(ClientContext &context, const string &uri, const string &id, const string &secret, + const string &endpoint, const string &scope); + static IRCAPISchema CreateSchema(ClientContext &context, IRCatalog &catalog, const string &internal, + const string &schema, IRCCredentials credentials); + static void DropSchema(ClientContext &context, const string &internal, const string &schema, + IRCCredentials credentials); + static IRCAPITable CreateTable(ClientContext &context, IRCatalog &catalog, const string &internal, + const string &schema, IRCCredentials credentials, CreateTableInfo *table_info); + static void DropTable(ClientContext &context, IRCatalog &catalog, const string &internal, const string &schema, + string &table_name, IRCCredentials credentials); }; } // namespace duckdb diff --git a/src/include/catalog_utils.hpp b/src/include/catalog_utils.hpp index e6475654..f83ec2a8 100644 --- a/src/include/catalog_utils.hpp +++ b/src/include/catalog_utils.hpp @@ -28,10 +28,10 @@ class ICUtils { }; struct YyjsonDocDeleter { - void operator()(yyjson_doc* doc) { + void operator()(yyjson_doc *doc) { yyjson_doc_free(doc); } - void operator()(yyjson_mut_doc* doc) { + void operator()(yyjson_mut_doc *doc) { yyjson_mut_doc_free(doc); } }; diff --git a/src/include/credentials/credential_provider.hpp b/src/include/credentials/credential_provider.hpp index ad093d98..246412d9 100644 --- a/src/include/credentials/credential_provider.hpp +++ b/src/include/credentials/credential_provider.hpp @@ -7,10 +7,9 @@ namespace duckdb { -class DuckDBSecretCredentialProvider : public Aws::Auth::AWSCredentialsProviderChain -{ +class DuckDBSecretCredentialProvider : public Aws::Auth::AWSCredentialsProviderChain { public: - DuckDBSecretCredentialProvider(const string& key_id, const string &secret, const string &sesh_token) { + DuckDBSecretCredentialProvider(const string &key_id, const string &secret, const string &sesh_token) { credentials.SetAWSAccessKeyId(key_id); credentials.SetAWSSecretKey(secret); credentials.SetSessionToken(sesh_token); @@ -26,4 +25,4 @@ class DuckDBSecretCredentialProvider : public Aws::Auth::AWSCredentialsProviderC Aws::Auth::AWSCredentials credentials; }; -} \ No newline at end of file +} // namespace duckdb \ No newline at end of file diff --git a/src/include/iceberg_manifest.hpp b/src/include/iceberg_manifest.hpp index a60fcecf..3fba15a4 100644 --- a/src/include/iceberg_manifest.hpp +++ b/src/include/iceberg_manifest.hpp @@ -26,17 +26,21 @@ struct ManifestReaderInput; struct IcebergManifestV1 { static constexpr idx_t FORMAT_VERSION = 1; using entry_type = IcebergManifest; - static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, vector &entries); + static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, + vector &entries); static bool VerifySchema(const case_insensitive_map_t &name_to_vec); - static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec); + static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec); }; struct IcebergManifestV2 { static constexpr idx_t FORMAT_VERSION = 2; using entry_type = IcebergManifest; - static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, vector &entries); + static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, + vector &entries); static bool VerifySchema(const case_insensitive_map_t &name_to_vec); - static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec); + static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec); }; //! Iceberg Manifest Entry scan routines @@ -44,26 +48,32 @@ struct IcebergManifestV2 { struct IcebergManifestEntryV1 { static constexpr idx_t FORMAT_VERSION = 1; using entry_type = IcebergManifestEntry; - static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, vector &entries); + static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, + vector &entries); static bool VerifySchema(const case_insensitive_map_t &name_to_vec); - static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec); + static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec); }; struct IcebergManifestEntryV2 { static constexpr idx_t FORMAT_VERSION = 2; using entry_type = IcebergManifestEntry; - static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, vector &entries); + static idx_t ProduceEntries(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, + vector &entries); static bool VerifySchema(const case_insensitive_map_t &name_to_vec); - static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &name_to_vec); + static void PopulateNameMapping(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &name_to_vec); }; class AvroScan { public: AvroScan(const string &scan_name, ClientContext &context, const string &path); + public: bool GetNext(DataChunk &chunk); void InitializeChunk(DataChunk &chunk); bool Finished() const; + public: optional_ptr avro_scan; ClientContext &context; @@ -75,5 +85,4 @@ class AvroScan { bool finished = false; }; - } // namespace duckdb diff --git a/src/include/iceberg_metadata.hpp b/src/include/iceberg_metadata.hpp index 5960140c..990a648f 100644 --- a/src/include/iceberg_metadata.hpp +++ b/src/include/iceberg_metadata.hpp @@ -61,14 +61,18 @@ class IcebergSnapshot { uint64_t schema_id; vector schema; string metadata_compression_codec = "none"; + public: static IcebergSnapshot GetLatestSnapshot(const string &path, FileSystem &fs, const IcebergOptions &options); - static IcebergSnapshot GetSnapshotById(const string &path, FileSystem &fs, idx_t snapshot_id, const IcebergOptions &options); - static IcebergSnapshot GetSnapshotByTimestamp(const string &path, FileSystem &fs, timestamp_t timestamp, const IcebergOptions &options); + static IcebergSnapshot GetSnapshotById(const string &path, FileSystem &fs, idx_t snapshot_id, + const IcebergOptions &options); + static IcebergSnapshot GetSnapshotByTimestamp(const string &path, FileSystem &fs, timestamp_t timestamp, + const IcebergOptions &options); static IcebergSnapshot ParseSnapShot(yyjson_val *snapshot, idx_t iceberg_format_version, idx_t schema_id, vector &schemas, const IcebergOptions &options); - static string GetMetaDataPath(ClientContext &context, const string &path, FileSystem &fs, const IcebergOptions &options); + static string GetMetaDataPath(ClientContext &context, const string &path, FileSystem &fs, + const IcebergOptions &options); static string ReadMetaData(const string &path, FileSystem &fs, const string &metadata_compression_codec); static yyjson_val *GetSnapshots(const string &path, FileSystem &fs, string GetSnapshotByTimestamp); static unique_ptr GetParseInfo(yyjson_doc &metadata_json); @@ -85,14 +89,16 @@ class IcebergSnapshot { static yyjson_val *FindSnapshotByIdInternal(yyjson_val *snapshots, idx_t target_id); static yyjson_val *FindSnapshotByIdTimestampInternal(yyjson_val *snapshots, timestamp_t timestamp); static vector ParseSchema(vector &schemas, idx_t schema_id); - static unique_ptr GetParseInfo(const string &path, FileSystem &fs, const string &metadata_compression_codec); + static unique_ptr GetParseInfo(const string &path, FileSystem &fs, + const string &metadata_compression_codec); }; //! Represents the iceberg table at a specific IcebergSnapshot. Corresponds to a single Manifest List. struct IcebergTable { public: //! Loads all(!) metadata of into IcebergTable object - static IcebergTable Load(const string &iceberg_path, IcebergSnapshot &snapshot, ClientContext &context, const IcebergOptions &options); + static IcebergTable Load(const string &iceberg_path, IcebergSnapshot &snapshot, ClientContext &context, + const IcebergOptions &options); public: //! Returns all paths to be scanned for the IcebergManifestContentType diff --git a/src/include/iceberg_multi_file_reader.hpp b/src/include/iceberg_multi_file_reader.hpp index c2ed71b4..b2379097 100644 --- a/src/include/iceberg_multi_file_reader.hpp +++ b/src/include/iceberg_multi_file_reader.hpp @@ -154,12 +154,11 @@ struct IcebergMultiFileReader : public MultiFileReader { void BindOptions(MultiFileReaderOptions &options, MultiFileList &files, vector &return_types, vector &names, MultiFileReaderBindData &bind_data) override; - void CreateColumnMapping(const string &file_name, - const vector &local_columns, - const vector &global_columns, - const vector &global_column_ids, MultiFileReaderData &reader_data, - const MultiFileReaderBindData &bind_data, const string &initial_file, - optional_ptr global_state) override; + void CreateColumnMapping(const string &file_name, const vector &local_columns, + const vector &global_columns, + const vector &global_column_ids, MultiFileReaderData &reader_data, + const MultiFileReaderBindData &bind_data, const string &initial_file, + optional_ptr global_state) override; unique_ptr InitializeGlobalState(ClientContext &context, const MultiFileReaderOptions &file_options, @@ -167,12 +166,11 @@ struct IcebergMultiFileReader : public MultiFileReader { const vector &global_columns, const vector &global_column_ids) override; - void FinalizeBind(const MultiFileReaderOptions &file_options, - const MultiFileReaderBindData &options, const string &filename, - const vector &local_columns, - const vector &global_columns, - const vector &global_column_ids, MultiFileReaderData &reader_data, - ClientContext &context, optional_ptr global_state) override; + void FinalizeBind(const MultiFileReaderOptions &file_options, const MultiFileReaderBindData &options, + const string &filename, const vector &local_columns, + const vector &global_columns, + const vector &global_column_ids, MultiFileReaderData &reader_data, + ClientContext &context, optional_ptr global_state) override; //! Override the FinalizeChunk method void FinalizeChunk(ClientContext &context, const MultiFileReaderBindData &bind_data, diff --git a/src/include/iceberg_options.hpp b/src/include/iceberg_options.hpp index 194dee52..d0bad6f2 100644 --- a/src/include/iceberg_options.hpp +++ b/src/include/iceberg_options.hpp @@ -8,7 +8,7 @@ namespace duckdb { static string VERSION_GUESSING_CONFIG_VARIABLE = "unsafe_enable_version_guessing"; // When this is provided (and unsafe_enable_version_guessing is true) -// we first look for DEFAULT_VERSION_HINT_FILE, if it doesn't exist we +// we first look for DEFAULT_VERSION_HINT_FILE, if it doesn't exist we // then search for versions matching the DEFAULT_TABLE_VERSION_FORMAT // We take the lexographically "greatest" one as the latest version // Note that this will voliate ACID constraints in some situations. @@ -24,11 +24,7 @@ static string DEFAULT_VERSION_HINT_FILE = "version-hint.text"; // By default we will use the unknown version behavior mentioned above static string DEFAULT_TABLE_VERSION = UNKNOWN_TABLE_VERSION; -enum class SnapshotSource : uint8_t { - LATEST, - FROM_TIMESTAMP, - FROM_ID -}; +enum class SnapshotSource : uint8_t { LATEST, FROM_TIMESTAMP, FROM_ID }; struct IcebergOptions { bool allow_moved_paths = false; diff --git a/src/include/iceberg_types.hpp b/src/include/iceberg_types.hpp index e81e6d78..24d9e956 100644 --- a/src/include/iceberg_types.hpp +++ b/src/include/iceberg_types.hpp @@ -70,6 +70,7 @@ struct IcebergManifest { int64_t sequence_number; //! either data or deletes IcebergManifestContentType content; + public: void Print() { Printer::Print(" - Manifest = { content: " + IcebergManifestContentTypeToString(content) + @@ -98,6 +99,7 @@ struct IcebergManifestEntry { string file_path; string file_format; int64_t record_count; + public: void Print() { Printer::Print(" -> ManifestEntry = { type: " + IcebergManifestEntryStatusTypeToString(status) + diff --git a/src/include/iceberg_utils.hpp b/src/include/iceberg_utils.hpp index 8d289b75..cad9ffe9 100644 --- a/src/include/iceberg_utils.hpp +++ b/src/include/iceberg_utils.hpp @@ -34,11 +34,11 @@ class IcebergUtils { static bool TryGetBoolFromObject(yyjson_val *obj, const string &field); static uint64_t TryGetNumFromObject(yyjson_val *obj, const string &field, bool fail_on_missing, - uint64_t default_val = 0); + uint64_t default_val = 0); static bool TryGetBoolFromObject(yyjson_val *obj, const string &field, bool fail_on_missing, - bool default_val = false); + bool default_val = false); static string TryGetStrFromObject(yyjson_val *obj, const string &field, bool fail_on_missing, - const char *default_val = ""); + const char *default_val = ""); }; } // namespace duckdb \ No newline at end of file diff --git a/src/include/manifest_reader.hpp b/src/include/manifest_reader.hpp index 47d36bee..09f30fca 100644 --- a/src/include/manifest_reader.hpp +++ b/src/include/manifest_reader.hpp @@ -8,17 +8,23 @@ namespace duckdb { // Manifest Reader -typedef void (*manifest_reader_name_mapping)(idx_t column_id, const LogicalType &type, const string &name, case_insensitive_map_t &mapping); +typedef void (*manifest_reader_name_mapping)(idx_t column_id, const LogicalType &type, const string &name, + case_insensitive_map_t &mapping); typedef bool (*manifest_reader_schema_validation)(const case_insensitive_map_t &mapping); -typedef idx_t (*manifest_reader_manifest_producer)(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, vector &result); -typedef idx_t (*manifest_reader_manifest_entry_producer)(DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input, vector &result); +typedef idx_t (*manifest_reader_manifest_producer)(DataChunk &chunk, idx_t offset, idx_t count, + const ManifestReaderInput &input, vector &result); +typedef idx_t (*manifest_reader_manifest_entry_producer)(DataChunk &chunk, idx_t offset, idx_t count, + const ManifestReaderInput &input, + vector &result); -using manifest_reader_read = std::function; +using manifest_reader_read = + std::function; struct ManifestReaderInput { public: ManifestReaderInput(const case_insensitive_map_t &name_to_vec, bool skip_deleted = false); + public: const case_insensitive_map_t &name_to_vec; //! Whether the deleted entries should be skipped outright @@ -28,11 +34,14 @@ struct ManifestReaderInput { class ManifestReader { public: ManifestReader(manifest_reader_name_mapping name_mapping, manifest_reader_schema_validation schema_validator); + public: void Initialize(unique_ptr scan_p); + public: bool Finished() const; idx_t ReadEntries(idx_t count, manifest_reader_read callback); + private: unique_ptr scan; DataChunk chunk; @@ -42,11 +51,11 @@ class ManifestReader { manifest_reader_name_mapping name_mapping = nullptr; manifest_reader_schema_validation schema_validation = nullptr; + public: bool skip_deleted = false; }; - template vector ScanAvroMetadata(const string &scan_name, ClientContext &context, const string &path) { auto scan = make_uniq(scan_name, context, path); @@ -57,9 +66,11 @@ vector ScanAvroMetadata(const string &scan_name, Client vector ret; while (!manifest_reader.Finished()) { - manifest_reader.ReadEntries(STANDARD_VECTOR_SIZE, [&ret, manifest_producer](DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input) { - return manifest_producer(chunk, offset, count, input, ret); - }); + manifest_reader.ReadEntries( + STANDARD_VECTOR_SIZE, + [&ret, manifest_producer](DataChunk &chunk, idx_t offset, idx_t count, const ManifestReaderInput &input) { + return manifest_producer(chunk, offset, count, input, ret); + }); } return ret; } diff --git a/src/include/storage/irc_catalog.hpp b/src/include/storage/irc_catalog.hpp index 2701d8f9..c7314d4f 100644 --- a/src/include/storage/irc_catalog.hpp +++ b/src/include/storage/irc_catalog.hpp @@ -34,21 +34,22 @@ class ICRClearCacheFunction : public TableFunction { static void ClearCacheOnSetting(ClientContext &context, SetScope scope, Value ¶meter); }; -enum class ICEBERG_CATALOG_TYPE { AWS_S3TABLES, AWS_GLUE, OTHER, INVALID}; +enum class ICEBERG_CATALOG_TYPE { AWS_S3TABLES, AWS_GLUE, OTHER, INVALID }; class MetadataCacheValue { public: std::string data; std::chrono::system_clock::time_point expires_at; + public: - MetadataCacheValue(std::string data_, std::chrono::system_clock::time_point expires_at_) : - data(data_), expires_at(expires_at_) {}; + MetadataCacheValue(std::string data_, std::chrono::system_clock::time_point expires_at_) + : data(data_), expires_at(expires_at_) {}; }; class IRCatalog : public Catalog { public: - explicit IRCatalog(AttachedDatabase &db_p, AccessMode access_mode, - IRCCredentials credentials, string warehouse, string host, string secret_name, string version = "v1"); + explicit IRCatalog(AttachedDatabase &db_p, AccessMode access_mode, IRCCredentials credentials, string warehouse, + string host, string secret_name, string version = "v1"); ~IRCatalog(); string internal_name; @@ -120,14 +121,11 @@ class IRCatalog : public Catalog { IRCSchemaSet schemas; string default_schema; - // defaults and overrides provided by a catalog. unordered_map defaults; unordered_map overrides; - unordered_map> metadata_cache; - }; } // namespace duckdb diff --git a/src/include/storage/irc_table_set.hpp b/src/include/storage/irc_table_set.hpp index d99a2f8f..a150cd0f 100644 --- a/src/include/storage/irc_table_set.hpp +++ b/src/include/storage/irc_table_set.hpp @@ -9,7 +9,6 @@ struct CreateTableInfo; class ICResult; class IRCSchemaEntry; - class ICInSchemaSet : public IRCCatalogSet { public: ICInSchemaSet(IRCSchemaEntry &schema); @@ -20,14 +19,14 @@ class ICInSchemaSet : public IRCCatalogSet { IRCSchemaEntry &schema; }; - class ICTableSet : public ICInSchemaSet { public: explicit ICTableSet(IRCSchemaEntry &schema); public: optional_ptr CreateTable(ClientContext &context, BoundCreateTableInfo &info); - static unique_ptr GetTableInfo(ClientContext &context, IRCSchemaEntry &schema, const string &table_name); + static unique_ptr GetTableInfo(ClientContext &context, IRCSchemaEntry &schema, + const string &table_name); optional_ptr RefreshTable(ClientContext &context, const string &table_name); void AlterTable(ClientContext &context, AlterTableInfo &info); void DropTable(ClientContext &context, DropInfo &info); @@ -47,5 +46,4 @@ class ICTableSet : public ICInSchemaSet { unique_ptr _CreateCatalogEntry(ClientContext &context, IRCAPITable table); }; - } // namespace duckdb diff --git a/src/include/url_utils.hpp b/src/include/url_utils.hpp index 30fab578..8fb6d04f 100644 --- a/src/include/url_utils.hpp +++ b/src/include/url_utils.hpp @@ -36,6 +36,7 @@ class IRCEndpointBuilder { //! path components when querying. Like namespaces/tables etc. vector path_components; + private: //! host of the endpoint, like `glue` or `polaris` string host; diff --git a/src/manifest_reader.cpp b/src/manifest_reader.cpp index 5e7501c9..97342171 100644 --- a/src/manifest_reader.cpp +++ b/src/manifest_reader.cpp @@ -2,11 +2,14 @@ namespace duckdb { - -ManifestReaderInput::ManifestReaderInput(const case_insensitive_map_t &name_to_vec, bool skip_deleted) : name_to_vec(name_to_vec), skip_deleted(skip_deleted) { +ManifestReaderInput::ManifestReaderInput(const case_insensitive_map_t &name_to_vec, bool skip_deleted) + : name_to_vec(name_to_vec), skip_deleted(skip_deleted) { } -ManifestReader::ManifestReader(manifest_reader_name_mapping name_mapping, manifest_reader_schema_validation schema_validation) : name_mapping(name_mapping), schema_validation(schema_validation) {} +ManifestReader::ManifestReader(manifest_reader_name_mapping name_mapping, + manifest_reader_schema_validation schema_validation) + : name_mapping(name_mapping), schema_validation(schema_validation) { +} void ManifestReader::Initialize(unique_ptr scan_p) { const bool first_init = scan == nullptr; @@ -64,5 +67,4 @@ bool ManifestReader::Finished() const { return scan->Finished(); } - } // namespace duckdb diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index adb1477e..88c71923 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -15,10 +15,10 @@ using namespace duckdb_yyjson; namespace duckdb { -IRCatalog::IRCatalog(AttachedDatabase &db_p, AccessMode access_mode, - IRCCredentials credentials, string warehouse, string host, string secret_name, string version ) - : Catalog(db_p), access_mode(access_mode), credentials(std::move(credentials)), warehouse(warehouse), host(host), secret_name(secret_name), version(version), schemas(*this) { - +IRCatalog::IRCatalog(AttachedDatabase &db_p, AccessMode access_mode, IRCCredentials credentials, string warehouse, + string host, string secret_name, string version) + : Catalog(db_p), access_mode(access_mode), credentials(std::move(credentials)), warehouse(warehouse), host(host), + secret_name(secret_name), version(version), schemas(*this) { } IRCatalog::~IRCatalog() = default; @@ -39,7 +39,8 @@ void IRCatalog::GetConfig(ClientContext &context) { auto *overrides_json = yyjson_obj_get(root, "overrides"); auto *defaults_json = yyjson_obj_get(root, "defaults"); // save overrides and defaults. - // See https://iceberg.apache.org/docs/latest/configuration/#catalog-properties for sometimes used catalog properties + // See https://iceberg.apache.org/docs/latest/configuration/#catalog-properties for sometimes used catalog + // properties if (defaults_json && yyjson_obj_size(defaults_json) > 0) { yyjson_val *key, *val; yyjson_obj_iter iter = yyjson_obj_iter_with(defaults_json); @@ -70,7 +71,8 @@ void IRCatalog::GetConfig(ClientContext &context) { } } if (prefix.empty()) { - DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.HttpReqeust", "No prefix found for catalog with warehouse value %s", warehouse); + DUCKDB_LOG_DEBUG(context, "iceberg.Catalog.HttpReqeust", "No prefix found for catalog with warehouse value %s", + warehouse); } // TODO: store optional endpoints param as well. We can enforce per catalog the endpoints that // are allowed to be hit @@ -97,14 +99,14 @@ void IRCatalog::ScanSchemas(ClientContext &context, std::function IRCatalog::GetSchema(CatalogTransaction transaction, const string &schema_name, - OnEntryNotFound if_not_found, QueryErrorContext error_context) { + OnEntryNotFound if_not_found, QueryErrorContext error_context) { if (schema_name == DEFAULT_SCHEMA) { if (default_schema.empty()) { if (if_not_found == OnEntryNotFound::RETURN_NULL) { return nullptr; } throw InvalidInputException("Attempting to fetch the default schema - but no database was " - "provided in the connection string"); + "provided in the connection string"); } return GetSchema(transaction, default_schema, if_not_found, error_context); } @@ -127,7 +129,7 @@ string IRCatalog::GetDBPath() { DatabaseSize IRCatalog::GetDatabaseSize(ClientContext &context) { if (default_schema.empty()) { throw InvalidInputException("Attempting to fetch the database size - but no database was provided " - "in the connection string"); + "in the connection string"); } DatabaseSize size; return size; @@ -143,7 +145,8 @@ IRCEndpointBuilder IRCatalog::GetBaseUrl() const { base_url.AddPathComponent("iceberg"); base_url.AddPathComponent(version); break; - } default: + } + default: break; } return base_url; @@ -194,26 +197,25 @@ unique_ptr IRCatalog::BindCreateIndex(Binder &binder, CreateSta throw NotImplementedException("ICCatalog BindCreateIndex"); } - bool IRCatalog::HasCachedValue(string url) const { auto value = metadata_cache.find(url); - if (value != metadata_cache.end()) { - auto now = std::chrono::system_clock::now(); - if (now < value->second->expires_at) { + if (value != metadata_cache.end()) { + auto now = std::chrono::system_clock::now(); + if (now < value->second->expires_at) { return true; } - } + } return false; } string IRCatalog::GetCachedValue(string url) const { auto value = metadata_cache.find(url); - if (value != metadata_cache.end()) { - auto now = std::chrono::system_clock::now(); - if (now < value->second->expires_at) { - return value->second->data; - } - } + if (value != metadata_cache.end()) { + auto now = std::chrono::system_clock::now(); + if (now < value->second->expires_at) { + return value->second->data; + } + } throw InternalException("Cached value does not exist"); } diff --git a/src/storage/irc_clear_cache.cpp b/src/storage/irc_clear_cache.cpp index b44386c9..d13275d8 100644 --- a/src/storage/irc_clear_cache.cpp +++ b/src/storage/irc_clear_cache.cpp @@ -45,6 +45,7 @@ void ICRClearCacheFunction::ClearCacheOnSetting(ClientContext &context, SetScope ClearIRCCaches(context); } -ICRClearCacheFunction::ICRClearCacheFunction() : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { +ICRClearCacheFunction::ICRClearCacheFunction() + : TableFunction("pc_clear_cache", {}, ClearCacheFunction, ClearCacheBind) { } } // namespace duckdb diff --git a/src/storage/irc_schema_entry.cpp b/src/storage/irc_schema_entry.cpp index 1b1c3e30..9b11e900 100644 --- a/src/storage/irc_schema_entry.cpp +++ b/src/storage/irc_schema_entry.cpp @@ -50,7 +50,7 @@ void ICUnqualifyColumnRef(ParsedExpression &expr) { } optional_ptr IRCSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info, - TableCatalogEntry &table) { + TableCatalogEntry &table) { throw NotImplementedException("Create Index"); } @@ -71,17 +71,17 @@ optional_ptr IRCSchemaEntry::CreateSequence(CatalogTransaction tra } optional_ptr IRCSchemaEntry::CreateTableFunction(CatalogTransaction transaction, - CreateTableFunctionInfo &info) { + CreateTableFunctionInfo &info) { throw BinderException("PC databases do not support creating table functions"); } optional_ptr IRCSchemaEntry::CreateCopyFunction(CatalogTransaction transaction, - CreateCopyFunctionInfo &info) { + CreateCopyFunctionInfo &info) { throw BinderException("PC databases do not support creating copy functions"); } optional_ptr IRCSchemaEntry::CreatePragmaFunction(CatalogTransaction transaction, - CreatePragmaFunctionInfo &info) { + CreatePragmaFunctionInfo &info) { throw BinderException("PC databases do not support creating pragma functions"); } @@ -105,7 +105,7 @@ static bool CatalogTypeIsSupported(CatalogType type) { } void IRCSchemaEntry::Scan(ClientContext &context, CatalogType type, - const std::function &callback) { + const std::function &callback) { if (!CatalogTypeIsSupported(type)) { return; } @@ -116,7 +116,7 @@ void IRCSchemaEntry::Scan(CatalogType type, const std::function IRCSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type, - const string &name) { + const string &name) { if (!CatalogTypeIsSupported(type)) { return nullptr; } diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index 08a673c1..5eef5fac 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -34,7 +34,8 @@ void IRCSchemaSet::FillEntry(ClientContext &context, unique_ptr &e optional_ptr IRCSchemaSet::CreateSchema(ClientContext &context, CreateSchemaInfo &info) { auto &ic_catalog = catalog.Cast(); - auto schema = IRCAPI::CreateSchema(context, ic_catalog, ic_catalog.internal_name, info.schema, ic_catalog.credentials); + auto schema = + IRCAPI::CreateSchema(context, ic_catalog, ic_catalog.internal_name, info.schema, ic_catalog.credentials); auto schema_entry = make_uniq(catalog, info); schema_entry->schema_data = make_uniq(schema); return CreateEntry(std::move(schema_entry)); diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index 91cebaa2..1654a38f 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -37,7 +37,8 @@ void ICTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &, LogicalPr TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr &bind_data) { auto &db = DatabaseInstance::GetDatabase(context); auto &iceberg_scan_function_set = ExtensionUtil::GetTableFunction(db, "iceberg_scan"); - auto iceberg_scan_function = iceberg_scan_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); + auto iceberg_scan_function = + iceberg_scan_function_set.functions.GetFunctionByArguments(context, {LogicalType::VARCHAR}); auto &ic_catalog = catalog.Cast(); D_ASSERT(table_data); @@ -48,10 +49,12 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrtable_id, table_data->schema_name, table_data->name); - auto table_credentials = IRCAPI::GetTableCredentials(context, ic_catalog, table_data->schema_name, table_data->name, secret_base_name); + auto secret_base_name = + StringUtil::Format("__internal_ic_%s__%s__%s", table_data->table_id, table_data->schema_name, table_data->name); + auto table_credentials = + IRCAPI::GetTableCredentials(context, ic_catalog, table_data->schema_name, table_data->name, secret_base_name); CreateSecretInfo info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); // First check if table credentials are set (possible the IC catalog does not return credentials) @@ -61,7 +64,8 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptrstorage_location.size()); - std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), lc_storage_location.begin(), ::tolower); + std::transform(table_data->storage_location.begin(), table_data->storage_location.end(), + lc_storage_location.begin(), ::tolower); size_t metadata_pos = lc_storage_location.find("metadata"); if (metadata_pos != std::string::npos) { info.scope = {lc_storage_location.substr(0, metadata_pos)}; @@ -84,13 +88,13 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr inputs = {table_data->storage_location}; - TableFunctionBindInput bind_input(inputs, param_map, return_types, names, nullptr, nullptr, - iceberg_scan_function, empty_ref); + TableFunctionBindInput bind_input(inputs, param_map, return_types, names, nullptr, nullptr, iceberg_scan_function, + empty_ref); auto result = iceberg_scan_function.bind(context, bind_input, return_types, names); bind_data = std::move(result); diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index 6ca929fa..38573b56 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -39,11 +39,11 @@ unique_ptr ICTableSet::_CreateCatalogEntry(ClientContext &context, } void ICTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { - auto* derived = static_cast(entry.get()); + auto *derived = static_cast(entry.get()); if (!derived->table_data->storage_location.empty()) { return; } - + auto &ic_catalog = catalog.Cast(); auto table = IRCAPI::GetTable(context, ic_catalog, schema.name, entry->name, ic_catalog.credentials); entry = _CreateCatalogEntry(context, table); @@ -80,7 +80,8 @@ unique_ptr ICTableSet::GetTableInfo(ClientContext &context, IRCSche optional_ptr ICTableSet::CreateTable(ClientContext &context, BoundCreateTableInfo &info) { auto &ic_catalog = catalog.Cast(); auto *table_info = dynamic_cast(info.base.get()); - auto table = IRCAPI::CreateTable(context, ic_catalog, ic_catalog.internal_name, schema.name, ic_catalog.credentials, table_info); + auto table = IRCAPI::CreateTable(context, ic_catalog, ic_catalog.internal_name, schema.name, ic_catalog.credentials, + table_info); auto entry = _CreateCatalogEntry(context, table); return CreateEntry(std::move(entry)); } From 54204d5d62b7ad46b7a435abcf089c06a0730b16 Mon Sep 17 00:00:00 2001 From: Florian Date: Fri, 28 Mar 2025 11:30:47 +0100 Subject: [PATCH 17/66] Replace IOExceptions --- src/common/api_utils.cpp | 15 +++++++------- src/common/iceberg.cpp | 20 +++++++++---------- src/common/schema.cpp | 12 +++++------ src/common/utils.cpp | 10 +++++----- src/iceberg_extension.cpp | 20 ++++++++++--------- .../iceberg_multi_file_reader.cpp | 2 +- src/include/iceberg_types.hpp | 6 +++--- src/storage/irc_catalog.cpp | 4 ++-- 8 files changed, 46 insertions(+), 43 deletions(-) diff --git a/src/common/api_utils.cpp b/src/common/api_utils.cpp index 41042217..c7168ebc 100644 --- a/src/common/api_utils.cpp +++ b/src/common/api_utils.cpp @@ -1,7 +1,8 @@ #include "api_utils.hpp" -#include -#include "storage/irc_catalog.hpp" #include "credentials/credential_provider.hpp" +#include "duckdb/common/exception/http_exception.hpp" +#include "storage/irc_catalog.hpp" +#include namespace duckdb { @@ -45,7 +46,7 @@ string APIUtils::GetRequest(ClientContext &context, const IRCEndpointBuilder &en curl_easy_strerror(res)); if (res != CURLcode::CURLE_OK) { string error = curl_easy_strerror(res); - throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); + throw HTTPException("Curl Request to '%s' failed with error: '%s'", url, error); } return readBuffer; @@ -116,8 +117,8 @@ string APIUtils::GetRequestAws(ClientContext &context, IRCEndpointBuilder endpoi } else { Aws::StringStream resBody; resBody << res->GetResponseBody().rdbuf(); - throw IOException("Failed to query %s, http error %d thrown. Message: %s", req->GetUri().GetURIString(true), - res->GetResponseCode(), resBody.str()); + throw HTTPException("Failed to query %s, http error %d thrown. Message: %s", req->GetUri().GetURIString(true), + res->GetResponseCode(), resBody.str()); } } @@ -177,7 +178,7 @@ string APIUtils::DeleteRequest(const string &url, const string &token, curl_slis if (res != CURLcode::CURLE_OK) { string error = curl_easy_strerror(res); - throw IOException("Curl DELETE Request to '%s' failed with error: '%s'", url, error); + throw HTTPException("Curl DELETE Request to '%s' failed with error: '%s'", url, error); } return readBuffer; @@ -227,7 +228,7 @@ string APIUtils::PostRequest(ClientContext &context, const string &url, const st curl_easy_strerror(res)); if (res != CURLcode::CURLE_OK) { string error = curl_easy_strerror(res); - throw IOException("Curl Request to '%s' failed with error: '%s'", url, error); + throw HTTPException("Curl Request to '%s' failed with error: '%s'", url, error); } return readBuffer; } diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp index f1d3aecf..6a62e8db 100644 --- a/src/common/iceberg.cpp +++ b/src/common/iceberg.cpp @@ -64,7 +64,7 @@ unique_ptr IcebergSnapshot::GetParseInfo(yyjson_doc &metadata } else { auto schema = yyjson_obj_get(root, "schema"); if (!schema) { - throw IOException("Neither a valid schema or schemas field was found"); + throw InvalidInputException("Neither a valid schema or schemas field was found"); } auto found_schema_id = IcebergUtils::TryGetNumFromObject(schema, "schema-id"); info.schemas.push_back(schema); @@ -103,7 +103,7 @@ IcebergSnapshot IcebergSnapshot::GetSnapshotById(const string &path, FileSystem auto snapshot = FindSnapshotByIdInternal(info->snapshots, snapshot_id); if (!snapshot) { - throw IOException("Could not find snapshot with id " + to_string(snapshot_id)); + throw InvalidInputException("Could not find snapshot with id " + to_string(snapshot_id)); } return ParseSnapShot(snapshot, info->iceberg_version, info->schema_id, info->schemas, options); @@ -115,7 +115,7 @@ IcebergSnapshot IcebergSnapshot::GetSnapshotByTimestamp(const string &path, File auto snapshot = FindSnapshotByIdTimestampInternal(info->snapshots, timestamp); if (!snapshot) { - throw IOException("Could not find latest snapshots for timestamp " + Timestamp::ToString(timestamp)); + throw InvalidInputException("Could not find latest snapshots for timestamp " + Timestamp::ToString(timestamp)); } return ParseSnapShot(snapshot, info->iceberg_version, info->schema_id, info->schemas, options); @@ -138,7 +138,7 @@ static string GenerateMetaDataUrl(FileSystem &fs, const string &meta_path, strin } } - throw IOException( + throw InvalidInputException( "Iceberg metadata file not found for table version '%s' using '%s' compression and format(s): '%s'", table_version, options.metadata_compression_codec, options.version_name_format); } @@ -171,7 +171,7 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa } if (!UnsafeVersionGuessingEnabled(context)) { // Make sure we're allowed to guess versions - throw IOException( + throw InvalidInputException( "Failed to read iceberg table. No version was provided and no version-hint could be found, globbing the " "filesystem to locate the latest version is disabled by default as this is considered unsafe and could " "result in reading uncommitted data. To enable this use 'SET %s = true;'", @@ -195,7 +195,7 @@ IcebergSnapshot IcebergSnapshot::ParseSnapShot(yyjson_val *snapshot, idx_t icebe if (snapshot) { auto snapshot_tag = yyjson_get_type(snapshot); if (snapshot_tag != YYJSON_TYPE_OBJ) { - throw IOException("Invalid snapshot field found parsing iceberg metadata.json"); + throw InvalidInputException("Invalid snapshot field found parsing iceberg metadata.json"); } ret.metadata_compression_codec = options.metadata_compression_codec; if (iceberg_format_version == 1) { @@ -226,9 +226,9 @@ string IcebergSnapshot::GetTableVersionFromHint(const string &meta_path, FileSys try { return version_file_content; } catch (std::invalid_argument &e) { - throw IOException("Iceberg version hint file contains invalid value"); + throw InvalidInputException("Iceberg version hint file contains invalid value"); } catch (std::out_of_range &e) { - throw IOException("Iceberg version hint file contains invalid value"); + throw InvalidInputException("Iceberg version hint file contains invalid value"); } } @@ -262,8 +262,8 @@ string IcebergSnapshot::GuessTableVersion(const string &meta_path, FileSystem &f } } - throw IOException("Could not guess Iceberg table version using '%s' compression and format(s): '%s'", - metadata_compression_codec, version_format); + throw InvalidInputException("Could not guess Iceberg table version using '%s' compression and format(s): '%s'", + metadata_compression_codec, version_format); } string IcebergSnapshot::PickTableVersion(vector &found_metadata, string &version_pattern, string &glob) { diff --git a/src/common/schema.cpp b/src/common/schema.cpp index fe800adf..cb789099 100644 --- a/src/common/schema.cpp +++ b/src/common/schema.cpp @@ -63,13 +63,13 @@ static LogicalType ParseComplexType(yyjson_val *type) { if (type_str == "map") { return ParseMap(type); } - throw IOException("Invalid field found while parsing field: type"); + throw InvalidInputException("Invalid field found while parsing field: type"); } static LogicalType ParseType(yyjson_val *type) { auto val = yyjson_obj_get(type, "type"); if (!val) { - throw IOException("Invalid field found while parsing field: type"); + throw InvalidInputException("Invalid field found while parsing field: type"); } return ParseTypeValue(val); } @@ -79,7 +79,7 @@ static LogicalType ParseTypeValue(yyjson_val *val) { return ParseComplexType(val); } if (yyjson_get_type(val) != YYJSON_TYPE_STR) { - throw IOException("Invalid field found while parsing field: type"); + throw InvalidInputException("Invalid field found while parsing field: type"); } string type_str = yyjson_get_str(val); @@ -136,7 +136,7 @@ static LogicalType ParseTypeValue(yyjson_val *val) { auto scale = std::stoi(digits[1]); return LogicalType::DECIMAL(width, scale); } - throw IOException("Encountered an unrecognized type in JSON schema: \"%s\"", type_str); + throw InvalidInputException("Encountered an unrecognized type in JSON schema: \"%s\"", type_str); } IcebergColumnDefinition IcebergColumnDefinition::ParseFromJson(yyjson_val *val) { @@ -155,7 +155,7 @@ static vector ParseSchemaFromJson(yyjson_val *schema_js // Assert that the top level 'type' is a struct auto type_str = IcebergUtils::TryGetStrFromObject(schema_json, "type"); if (type_str != "struct") { - throw IOException("Schema in JSON Metadata is invalid"); + throw InvalidInputException("Schema in JSON Metadata is invalid"); } D_ASSERT(yyjson_get_type(schema_json) == YYJSON_TYPE_OBJ); D_ASSERT(IcebergUtils::TryGetStrFromObject(schema_json, "type") == "struct"); @@ -180,7 +180,7 @@ vector IcebergSnapshot::ParseSchema(vector:s3tablescatalog/"); + throw InvalidInputException("Invalid Glue Catalog Format: '" + warehouse + + "'. Expected ':s3tablescatalog/"); } static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, @@ -120,8 +121,8 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in auto region = kv_secret.TryGetValue("region"); if (region.IsNull()) { - throw IOException("Assumed catalog secret " + secret_entry->secret->GetName() + " for catalog " + name + - " does not have a region"); + throw InvalidInputException("Assumed catalog secret " + secret_entry->secret->GetName() + " for catalog " + + name + " does not have a region"); } switch (catalog_type) { case ICEBERG_CATALOG_TYPE::AWS_S3TABLES: { @@ -137,7 +138,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in SanityCheckGlueWarehouse(warehouse); break; default: - throw IOException("Unsupported AWS catalog type"); + throw NotImplementedException("Unsupported AWS catalog type"); } auto catalog_host = service + "." + region.ToString() + ".amazonaws.com"; @@ -149,10 +150,11 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // Check no endpoint type has been passed. if (!endpoint_type.empty()) { - throw IOException("Unrecognized endpoint point: %s. Expected either S3_TABLES or GLUE", endpoint_type); + throw InvalidInputException("Unrecognized endpoint point: %s. Expected either S3_TABLES or GLUE", + endpoint_type); } if (endpoint_type.empty() && endpoint.empty()) { - throw IOException("No endpoint type or endpoint provided"); + throw InvalidInputException("No endpoint type or endpoint provided"); } catalog_type = ICEBERG_CATALOG_TYPE::OTHER; @@ -162,7 +164,7 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // if no secret is referenced, this throw auto secret_entry = IRCatalog::GetSecret(context, secret_name); if (!secret_entry) { - throw IOException("No secret found to use with catalog " + name); + throw InvalidConfigurationException("No secret found to use with catalog " + name); } // secret found - read data const auto &kv_secret = dynamic_cast(*secret_entry->secret); diff --git a/src/iceberg_functions/iceberg_multi_file_reader.cpp b/src/iceberg_functions/iceberg_multi_file_reader.cpp index 3f6a0712..84b1e7b3 100644 --- a/src/iceberg_functions/iceberg_multi_file_reader.cpp +++ b/src/iceberg_functions/iceberg_multi_file_reader.cpp @@ -313,7 +313,7 @@ void IcebergMultiFileReader::CreateColumnMapping(const string &file_name, // Lookup the required column in the local map auto entry = name_map.find("file_row_number"); if (entry == name_map.end()) { - throw IOException("Failed to find the file_row_number column"); + throw InvalidInputException("Failed to find the file_row_number column"); } // Register the column to be scanned from this file diff --git a/src/include/iceberg_types.hpp b/src/include/iceberg_types.hpp index 24d9e956..777d9312 100644 --- a/src/include/iceberg_types.hpp +++ b/src/include/iceberg_types.hpp @@ -27,7 +27,7 @@ static string IcebergManifestContentTypeToString(IcebergManifestContentType type case IcebergManifestContentType::DELETE: return "DELETE"; default: - throw IOException("Invalid Manifest Content Type"); + throw InvalidInputException("Invalid Manifest Content Type"); } } @@ -42,7 +42,7 @@ static string IcebergManifestEntryStatusTypeToString(IcebergManifestEntryStatusT case IcebergManifestEntryStatusType::DELETED: return "DELETED"; default: - throw IOException("Invalid matifest entry type"); + throw InvalidInputException("Invalid matifest entry type"); } } @@ -57,7 +57,7 @@ static string IcebergManifestEntryContentTypeToString(IcebergManifestEntryConten case IcebergManifestEntryContentType::EQUALITY_DELETES: return "EQUALITY_DELETES"; default: - throw IOException("Invalid Manifest Entry Content Type"); + throw InvalidInputException("Invalid Manifest Entry Content Type"); } } diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index b831789b..9a360570 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -168,14 +168,14 @@ unique_ptr IRCatalog::GetSecret(ClientContext &context, const strin if (!secret_entry) { auto secret_match = context.db->GetSecretManager().LookupSecret(transaction, "s3://", "s3"); if (!secret_match.HasMatch()) { - throw IOException("Failed to find a secret and no explicit secret was passed!"); + throw InvalidConfigurationException("Failed to find a secret and no explicit secret was passed!"); } secret_entry = std::move(secret_match.secret_entry); } if (secret_entry) { return secret_entry; } - throw IOException("Could not find valid Iceberg secret"); + throw InvalidConfigurationException("Could not find valid Iceberg secret"); } unique_ptr IRCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, From ccf9507b27c216e0122e12387259b2854c340590 Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 31 Mar 2025 17:03:32 +0200 Subject: [PATCH 18/66] accidentally lost this change it seems?? --- src/iceberg_functions/iceberg_multi_file_reader.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/iceberg_functions/iceberg_multi_file_reader.cpp b/src/iceberg_functions/iceberg_multi_file_reader.cpp index 27b2c4fb..3f6a0712 100644 --- a/src/iceberg_functions/iceberg_multi_file_reader.cpp +++ b/src/iceberg_functions/iceberg_multi_file_reader.cpp @@ -200,6 +200,13 @@ void IcebergMultiFileList::InitializeFiles() { snapshot.iceberg_format_version); } + if (snapshot.snapshot_id == DConstants::INVALID_INDEX) { + // we are in an empty table + current_data_manifest = data_manifests.begin(); + current_delete_manifest = delete_manifests.begin(); + return; + } + // Read the manifest list, we need all the manifests to determine if we've seen all deletes auto manifest_list_full_path = options.allow_moved_paths ? IcebergUtils::GetFullPath(iceberg_path, snapshot.manifest_list, fs) From 0932428db725f719a0a46c96466b59cacbda847a Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 10:09:51 +0200 Subject: [PATCH 19/66] formatting on cmakelists (with extension-ci-tools checked out at the right commit) --- CMakeLists.txt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f23c448f..73d3203b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ set(EXTENSION_SOURCES src/iceberg_manifest.cpp src/manifest_reader.cpp src/catalog_api.cpp - src/catalog_utils.cpp + src/catalog_utils.cpp src/common/utils.cpp src/common/url_utils.cpp src/common/schema.cpp @@ -34,8 +34,7 @@ set(EXTENSION_SOURCES src/storage/irc_table_entry.cpp src/storage/irc_table_set.cpp src/storage/irc_transaction.cpp - src/storage/irc_transaction_manager.cpp -) + src/storage/irc_transaction_manager.cpp) add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES}) @@ -47,13 +46,16 @@ find_package(AWSSDK REQUIRED COMPONENTS core sso sts) include_directories(${CURL_INCLUDE_DIRS}) # AWS SDK FROM vcpkg -target_include_directories(${EXTENSION_NAME} PUBLIC $) +target_include_directories(${EXTENSION_NAME} + PUBLIC $) target_link_libraries(${EXTENSION_NAME} PUBLIC ${AWSSDK_LINK_LIBRARIES}) -target_include_directories(${TARGET_NAME}_loadable_extension PRIVATE $) -target_link_libraries(${TARGET_NAME}_loadable_extension ${AWSSDK_LINK_LIBRARIES}) +target_include_directories(${TARGET_NAME}_loadable_extension + PRIVATE $) +target_link_libraries(${TARGET_NAME}_loadable_extension + ${AWSSDK_LINK_LIBRARIES}) # Link dependencies into extension -target_link_libraries(${EXTENSION_NAME} PUBLIC ${CURL_LIBRARIES}) +target_link_libraries(${EXTENSION_NAME} PUBLIC ${CURL_LIBRARIES}) target_link_libraries(${TARGET_NAME}_loadable_extension ${CURL_LIBRARIES}) install( From 41eec365647d3169f675cfe856115cdef57b7b80 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 10:16:14 +0200 Subject: [PATCH 20/66] fix Format Check CI --- .github/workflows/CodeQuality.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CodeQuality.yml b/.github/workflows/CodeQuality.yml index db65f565..8bf78201 100644 --- a/.github/workflows/CodeQuality.yml +++ b/.github/workflows/CodeQuality.yml @@ -44,6 +44,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: 'true' - name: Install shell: bash @@ -65,4 +66,4 @@ jobs: shell: bash run: | make generate-files - git diff --exit-code \ No newline at end of file + git diff --exit-code From beb486cd197fb22f527abaed05770d5dc2ecc45c Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 10:19:31 +0200 Subject: [PATCH 21/66] remove generated check --- .github/workflows/CodeQuality.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/CodeQuality.yml b/.github/workflows/CodeQuality.yml index 8bf78201..8918d414 100644 --- a/.github/workflows/CodeQuality.yml +++ b/.github/workflows/CodeQuality.yml @@ -62,8 +62,3 @@ jobs: black --version make format-check-silent - - name: Generated Check - shell: bash - run: | - make generate-files - git diff --exit-code From 8d117a39f2ce564cf9c14c1144f51ee4edc27429 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 10:56:22 +0200 Subject: [PATCH 22/66] install ninja-build before running the extension-ci-tools workflow --- .github/workflows/MainDistributionPipeline.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index e4ca91fb..b7e308a1 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -12,8 +12,16 @@ concurrency: cancel-in-progress: true jobs: + duckdb-stable-build: name: Build extension binaries + steps: + - name: Install Ninja (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq ninja-build + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1 with: extension_name: iceberg From e13aa5d62a5a1a2325f99fde9063fe4000089fa0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 12:01:59 +0200 Subject: [PATCH 23/66] downgrade exception types --- src/catalog_api.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 36c7f61d..373705ef 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -100,7 +100,8 @@ static void ParseConfigOptions(yyjson_val *config, case_insensitive_map_t } else if (value == "false") { path_style = false; } else { - throw InternalException("Unexpected value ('%s') for 's3.path-style-access' in 'config' property", value); + throw InvalidInputException("Unexpected value ('%s') for 's3.path-style-access' in 'config' property", + value); } options["use_ssl"] = Value(!path_style); if (path_style) { @@ -149,13 +150,13 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat yyjson_arr_foreach(storage_credentials, index, max, storage_credential) { auto *sc_prefix = yyjson_obj_get(storage_credential, "prefix"); if (!sc_prefix) { - throw InternalException("required property 'prefix' is missing from the StorageCredential schema"); + throw InvalidInputException("required property 'prefix' is missing from the StorageCredential schema"); } CreateSecretInfo create_secret_info(OnCreateConflict::REPLACE_ON_CONFLICT, SecretPersistType::TEMPORARY); auto prefix_string = yyjson_get_str(sc_prefix); if (!prefix_string) { - throw InternalException("property 'prefix' of StorageCredential is NULL"); + throw InvalidInputException("property 'prefix' of StorageCredential is NULL"); } create_secret_info.scope.push_back(string(prefix_string)); create_secret_info.name = StringUtil::Format("%s_%d_%s", secret_base_name, index, prefix_string); @@ -241,7 +242,7 @@ static void populateTableMetadata(IRCAPITable &table, yyjson_val *metadata_root) } if (!found) { - throw InternalException("Current schema not found"); + throw InvalidInputException("Current schema not found"); } } From cb5185b8a304653ba042d0651969b32baa6be380 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 12:05:43 +0200 Subject: [PATCH 24/66] address feedback --- src/catalog_utils.cpp | 4 ++-- src/storage/irc_table_entry.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/catalog_utils.cpp b/src/catalog_utils.cpp index 50e3dffe..166de528 100644 --- a/src/catalog_utils.cpp +++ b/src/catalog_utils.cpp @@ -50,7 +50,7 @@ string ICUtils::LogicalToIcebergType(const LogicalType &input) { break; } - throw std::runtime_error("Unsupported type: " + input.ToString()); + throw InvalidInputException("Unsupported type: %s", input.ToString()); } string ICUtils::TypeToString(const LogicalType &input) { @@ -240,7 +240,7 @@ yyjson_doc *ICUtils::api_result_to_doc(const string &api_result) { auto *error = yyjson_obj_get(root, "error"); if (error != NULL) { string err_msg = IcebergUtils::TryGetStrFromObject(error, "message"); - throw std::runtime_error(err_msg); + throw InvalidInputException(err_msg); } return doc; } diff --git a/src/storage/irc_table_entry.cpp b/src/storage/irc_table_entry.cpp index 1654a38f..d1439044 100644 --- a/src/storage/irc_table_entry.cpp +++ b/src/storage/irc_table_entry.cpp @@ -70,7 +70,7 @@ TableFunction ICTableEntry::GetScanFunction(ClientContext &context, unique_ptr Date: Tue, 1 Apr 2025 13:33:49 +0200 Subject: [PATCH 25/66] add a step to verify that ninja is on the path --- .github/workflows/LocalTesting.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 7aabeb11..a63224b7 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -37,6 +37,10 @@ jobs: fetch-depth: 0 submodules: 'true' + - name: Verify ninja + run: | + ninja --version + - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: From 0930bf50835e21963c07f829dfbcbb1f5b05845c Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 14:21:11 +0200 Subject: [PATCH 26/66] clean up veeeery long install line, add cmake=3.22 and cmake-data=3.22 --- .github/workflows/LocalTesting.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index a63224b7..db179834 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -27,7 +27,31 @@ jobs: sudo apt-get install -y -qq software-properties-common sudo add-apt-repository ppa:git-core/ppa sudo apt-get update -y -qq - sudo apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client + sudo apt-get install -y -qq \ + ninja-build \ + make gcc-multilib \ + g++-multilib \ + libssl-dev \ + wget \ + openjdk-8-jdk \ + zip \ + maven \ + unixodbc-dev \ + libc6-dev-i386 \ + lib32readline6-dev \ + libssl-dev \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + gettext \ + unzip \ + build-essential \ + 'cmake=3.22.1*' \ + 'cmake-data=3.22.1*' \ + checkinstall \ + libffi-dev \ + curl \ + libz-dev \ + openssh-client sudo apt-get install -y -qq tar pkg-config sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose From e4a3617697ee8576bdfb7fe5eb380843fa8957ce Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 14:22:35 +0200 Subject: [PATCH 27/66] any 3.* version then? --- .github/workflows/LocalTesting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index db179834..f4b15449 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -45,8 +45,8 @@ jobs: gettext \ unzip \ build-essential \ - 'cmake=3.22.1*' \ - 'cmake-data=3.22.1*' \ + 'cmake=3.*' \ + 'cmake-data=3.*' \ checkinstall \ libffi-dev \ curl \ From 0581fcb5e5a93b1203105fc03987f1a847ad5919 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 14:26:12 +0200 Subject: [PATCH 28/66] check cmake version --- .github/workflows/LocalTesting.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index f4b15449..2dad9ccc 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -61,9 +61,10 @@ jobs: fetch-depth: 0 submodules: 'true' - - name: Verify ninja + - name: Check installed versions run: | ninja --version + cmake --version - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 From 8fe54be61d7c79ce555d34426cd85c419900b998 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 14:28:04 +0200 Subject: [PATCH 29/66] move the cmake install to a separate line --- .github/workflows/LocalTesting.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 2dad9ccc..a84c2029 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -45,13 +45,12 @@ jobs: gettext \ unzip \ build-essential \ - 'cmake=3.*' \ - 'cmake-data=3.*' \ checkinstall \ libffi-dev \ curl \ libz-dev \ openssh-client + sudo apt-get install --allow-downgrades -y -qq 'cmake=3.*' 'cmake-data=3.*' sudo apt-get install -y -qq tar pkg-config sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose From 4a47dff537c14f66271e58b19faf759c37bc7a3e Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 1 Apr 2025 15:09:13 +0200 Subject: [PATCH 30/66] move the install of cmake 3.* to a separate step, to enforce that it runs AFTER build-essential is installed, which somehow isn't guaranteed by them being on separate lines... --- .github/workflows/LocalTesting.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index a84c2029..4bb021b3 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -50,11 +50,15 @@ jobs: curl \ libz-dev \ openssh-client - sudo apt-get install --allow-downgrades -y -qq 'cmake=3.*' 'cmake-data=3.*' sudo apt-get install -y -qq tar pkg-config sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose + - name: Install CMake 3.x + run: | + sudo apt-get remove -y cmake cmake-data + sudo apt-get install --allow-downgrades -y -qq 'cmake=3.*' 'cmake-data=3.*' + - uses: actions/checkout@v4 with: fetch-depth: 0 From e44759d34457e1bb357a39c9ff0100c02537536c Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 2 Apr 2025 14:02:27 +0200 Subject: [PATCH 31/66] initial commit. want to test on AWS --- .github/workflows/PolarisTesting.yml | 125 ++++++++++++++++++++ scripts/polaris/create_data.py | 43 +++++++ scripts/polaris/get_polaris_client_creds.py | 21 ++++ scripts/polaris/get_polaris_root_creds.py | 24 ++++ scripts/polaris/setup_polaris_catalog.py | 103 ++++++++++++++++ scripts/polaris/setup_polaris_catalog.sh | 61 ++++++++++ scripts/requirements-polaris.txt | 2 + test/sql/local/irc/test_polaris.test | 14 ++- 8 files changed, 387 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/PolarisTesting.yml create mode 100644 scripts/polaris/create_data.py create mode 100644 scripts/polaris/get_polaris_client_creds.py create mode 100644 scripts/polaris/get_polaris_root_creds.py create mode 100644 scripts/polaris/setup_polaris_catalog.py create mode 100755 scripts/polaris/setup_polaris_catalog.sh create mode 100644 scripts/requirements-polaris.txt diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml new file mode 100644 index 00000000..f20a8308 --- /dev/null +++ b/.github/workflows/PolarisTesting.yml @@ -0,0 +1,125 @@ +name: Local functional tests +on: [push, pull_request,repository_dispatch] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true +defaults: + run: + shell: bash + +env: + BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} + +jobs: + rest: + name: Test against local Rest Catalog + runs-on: ubuntu-latest + env: + VCPKG_TARGET_TRIPLET: 'x64-linux' + GEN: ninja + VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake + PIP_BREAK_SYSTEM_PACKAGES: 1 + + steps: + - name: Install required ubuntu packages + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq software-properties-common + sudo add-apt-repository ppa:git-core/ppa + sudo apt-get update -y -qq + sudo apt-get install -y -qq \ + ninja-build \ + make gcc-multilib \ + g++-multilib \ + libssl-dev \ + wget \ + openjdk-8-jdk \ + zip \ + maven \ + unixodbc-dev \ + libc6-dev-i386 \ + lib32readline6-dev \ + libssl-dev \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + gettext \ + unzip \ + build-essential \ + checkinstall \ + libffi-dev \ + curl \ + libz-dev \ + openssh-client + sudo apt-get install -y -qq tar pkg-config + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + + - name: Install CMake 3.x + run: | + sudo apt-get remove -y cmake cmake-data + sudo apt-get install --allow-downgrades -y -qq 'cmake=3.*' 'cmake-data=3.*' + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: 'true' + + - name: Setup vcpkg + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: 5e5d0e1cd7785623065e77eff011afdeec1a3574 + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + continue-on-error: true + + - name: Build extension + env: + GEN: ninja + STATIC_LIBCPP: 1 + run: | + make release + + - name: Install java 21 + run: | + sudo apt install -y -qq openjdk-21-jre-headless + sudo apt install -y -qq openjdk-21-jdk-headless + + - name: Install python venv + run: | + sudo apt-get install -y -qq python3-venv + + - name: Setup Jenv + run: | + git clone https://github.com/jenv/jenv.git ~/.jenv + echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bash_profile + source ~/.bash_profile + eval "$(jenv init -)" + jenv enable-plugin export + exec $SHELL -l + jenv add /usr/lib/jvm/java-21-openjdk-amd64 + + + - name: Setup Polaris + run: | + make setup_polaris + python3 -m venv . + source ./bin/activate + python3 -m pip install -r ../scripts/requirements-polaris.txt + python3 scripts/polaris/get_polaris_root_creds.py + # needed for setup_polaris_catalog.sh + export POLARIS_ROOT_ID=$(cat polaris_root_id.txt) + export POLARIS_ROOT_SECRET=$(cat polaris_root_password.txt) + ./scripts/polaris/setup_polaris_catalog.sh > user_credentials.json + export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) + export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) + ./scripts/polaris/create_data.py + + - name: Test with rest catalog + env: + POLARIS_SERVER_AVAILABLE: 1 + run: | + export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) + export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) + make test_polaris + diff --git a/scripts/polaris/create_data.py b/scripts/polaris/create_data.py new file mode 100644 index 00000000..5d6515d7 --- /dev/null +++ b/scripts/polaris/create_data.py @@ -0,0 +1,43 @@ +import os +from pyspark.sql import SparkSession + +client_id = os.getenv('POLARIS_CLIENT_ID', '') +client_secret = os.getenv('POLARIS_SECRET_ID', '') + +spark = (SparkSession.builder + .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19") + .config('spark.sql.iceberg.vectorization.enabled', 'false') + # Configure the 'polaris' catalog as an Iceberg rest catalog + .config("spark.sql.catalog.polaris.type", "rest") + .config("spark.sql.catalog.polaris", "org.apache.iceberg.spark.SparkCatalog") + # Specify the rest catalog endpoint + .config("spark.sql.catalog.polaris.uri", "http://polaris:8181/api/catalog") + # Enable token refresh + .config("spark.sql.catalog.polaris.token-refresh-enabled", "true") + # specify the client_id:client_secret pair + .config("spark.sql.catalog.polaris.credential", f"{client_id}:{client_secret}") + # Set the warehouse to the name of the catalog we created + .config("spark.sql.catalog.polaris.warehouse", catalog_name) + # Scope set to PRINCIPAL_ROLE:ALL + .config("spark.sql.catalog.polaris.scope", 'PRINCIPAL_ROLE:ALL') + # Enable access credential delegation + .config("spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation", 'vended-credentials') + .config("spark.sql.catalog.polaris.io-impl", "org.apache.iceberg.io.ResolvingFileIO") + .config("spark.sql.catalog.polaris.s3.region", "us-west-2") + .config("spark.history.fs.logDirectory", "/home/iceberg/spark-events")).getOrCreate() + + +spark.sql("USE polaris") +spark.sql("SHOW NAMESPACES").show() + +spark.sql("CREATE NAMESPACE IF NOT EXISTS COLLADO_TEST") +spark.sql("USE NAMESPACE COLLADO_TEST") +spark.sql("""CREATE TABLE IF NOT EXISTS TEST_TABLE ( + id bigint NOT NULL COMMENT 'unique id', + data string) +USING iceberg; +""") +spark.sql("INSERT INTO TEST_TABLE VALUES (1, 'some data'), (2, 'more data'), (3, 'yet more data')") +spark.sql("SELECT * FROM TEST_TABLE").show() + diff --git a/scripts/polaris/get_polaris_client_creds.py b/scripts/polaris/get_polaris_client_creds.py new file mode 100644 index 00000000..bfc1bfc5 --- /dev/null +++ b/scripts/polaris/get_polaris_client_creds.py @@ -0,0 +1,21 @@ +import re +import os +import json + +# Read and parse the JSON file +with open("user_credentials.json", "r") as json_file: + config = json.load(json_file) + + client_id = config.get("clientId") + client_secret = config.get("clientSecret") + + if client_id and client_secret: + # Write client_id and client_secret to separate files + with open("polaris_client_id.txt", "w") as id_file: + id_file.write(client_id) + + with open("polaris_client_secret.txt", "w") as secret_file: + secret_file.write(client_secret) + else: + print("clientId or clientSecret not found in config.json") + diff --git a/scripts/polaris/get_polaris_root_creds.py b/scripts/polaris/get_polaris_root_creds.py new file mode 100644 index 00000000..d3d5f6ca --- /dev/null +++ b/scripts/polaris/get_polaris_root_creds.py @@ -0,0 +1,24 @@ +import re +import os + +# Read the log file (hopefully it isn't too big) +with open("polaris_catalog/polaris-server.log", "r") as file: + log_content = file.read() + +# Regular expression to capture the credentials +match = re.search(r"realm: POLARIS root principal credentials: (\w+):(\w+)", log_content) + +if match: + root_user = match.group(1) + root_password = match.group(2) + if root_user and root_password: + # Write client_id and client_secret to separate files + with open("polaris_root_id.txt", "w") as id_file: + id_file.write(root_user) + + with open("polaris_root_password.txt", "w") as secret_file: + secret_file.write(root_password) + +else: + print("Credentials not found in the log file.") + exit(1) diff --git a/scripts/polaris/setup_polaris_catalog.py b/scripts/polaris/setup_polaris_catalog.py new file mode 100644 index 00000000..8e35061d --- /dev/null +++ b/scripts/polaris/setup_polaris_catalog.py @@ -0,0 +1,103 @@ +import os +from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI +from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API +from polaris.catalog.api_client import ApiClient as CatalogApiClient +from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration + +# some of this is from https://github.com/apache/polaris/blob/e32ef89bb97642f2ac9a4db82252a4fcf7aa0039/getting-started/spark/notebooks/SparkPolaris.ipynb +polaris_credential = 'root:s3cr3t' # pragma: allowlist secret + +client_id = os.getenv('POLARIS_ROOT_ID', '') +client_secret = os.getenv('POLARIS_ROOT_SECRET', '') + +if client_id == '' or client_secret == '': + Print("could not find polaris root id or polaris root secret") +client = CatalogApiClient(CatalogApiClientConfiguration(username=client_id, + password=client_secret, + host='http://polaris:8181/api/catalog')) + +oauth_api = IcebergOAuth2API(client) +token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', + client_id=client_id, + client_secret=client_secret, + grant_type='client_credentials', + _headers={'realm': 'default-realm'}) + +# create a catalog +from polaris.management import * + +client = ApiClient(Configuration(access_token=token.access_token, + host='http://polaris:8181/api/management/v1')) +root_client = PolarisDefaultApi(client) + +storage_conf = FileStorageConfigInfo(storage_type="FILE", allowed_locations=["file:///tmp"]) +catalog_name = 'polaris_demo' +catalog = Catalog(name=catalog_name, type='INTERNAL', properties={"default-base-location": "file:///tmp/polaris/"}, + storage_config_info=storage_conf) +catalog.storage_config_info = storage_conf +root_client.create_catalog(create_catalog_request=CreateCatalogRequest(catalog=catalog)) +resp = root_client.get_catalog(catalog_name=catalog.name) + + +# UTILITY FUNCTIONS +# Creates a principal with the given name +def create_principal(api, principal_name): + principal = Principal(name=principal_name, type="SERVICE") + try: + principal_result = api.create_principal(CreatePrincipalRequest(principal=principal)) + return principal_result + except ApiException as e: + if e.status == 409: + return api.rotate_credentials(principal_name=principal_name) + else: + raise e + +# Create a catalog role with the given name +def create_catalog_role(api, catalog, role_name): + catalog_role = CatalogRole(name=role_name) + try: + api.create_catalog_role(catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role)) + return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) + except ApiException as e: + return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) + else: + raise e + +# Create a principal role with the given name +def create_principal_role(api, role_name): + principal_role = PrincipalRole(name=role_name) + try: + api.create_principal_role(CreatePrincipalRoleRequest(principal_role=principal_role)) + return api.get_principal_role(principal_role_name=role_name) + except ApiException as e: + return api.get_principal_role(principal_role_name=role_name) + + +# Create a bunch of new roles + +# Create the engineer_principal +engineer_principal = create_principal(root_client, "collado") + +# Create the principal role +engineer_role = create_principal_role(root_client, "engineer") + +# Create the catalog role +manager_catalog_role = create_catalog_role(root_client, catalog, "manage_catalog") + +# Grant the catalog role to the principal role +# All principals in the principal role have the catalog role's privileges +root_client.assign_catalog_role_to_principal_role(principal_role_name=engineer_role.name, + catalog_name=catalog.name, + grant_catalog_role_request=GrantCatalogRoleRequest(catalog_role=manager_catalog_role)) + +# Assign privileges to the catalog role +# Here, we grant CATALOG_MANAGE_CONTENT +root_client.add_grant_to_catalog_role(catalog.name, manager_catalog_role.name, + AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name, + type='catalog', + privilege=CatalogPrivilege.CATALOG_MANAGE_CONTENT))) + +# Assign the principal role to the principal +root_client.assign_principal_role(engineer_principal.principal.name, grant_principal_role_request=GrantPrincipalRoleRequest(principal_role=engineer_role)) + + diff --git a/scripts/polaris/setup_polaris_catalog.sh b/scripts/polaris/setup_polaris_catalog.sh new file mode 100755 index 00000000..834e2d38 --- /dev/null +++ b/scripts/polaris/setup_polaris_catalog.sh @@ -0,0 +1,61 @@ +# TODO: use the python module to execute these steps. +# Seems like the python module is not yet publicly available/installable, so unsure what to do about this. + +./polaris_catalog/polaris \ + --client-id ${POLARIS_ROOT_ID} \ + --client-secret ${POLARIS_ROOT_SECRET} \ + catalogs \ + create \ + --storage-type FILE \ + --default-base-location file://${PWD}/storage_files \ + quickstart_catalog + + +./polaris_catalog/polaris \ + --client-id ${POLARIS_ROOT_ID} \ + --client-secret ${POLARIS_ROOT_SECRET} \ + principals \ + create \ + quickstart_user + +./polaris_catalog/polaris \ + --client-id ${POLARIS_ROOT_ID} \ + --client-secret ${POLARIS_ROOT_SECRET} \ + principal-roles \ + create \ + quickstart_user_role + +./polaris_catalog/polaris \ + --client-id ${POLARIS_ROOT_ID} \ + --client-secret ${POLARIS_ROOT_SECRET} \ + catalog-roles \ + create \ + --catalog quickstart_catalog \ + quickstart_catalog_role + +./polaris_catalog/polaris \ + --client-id ${POLARIS_ROOT_ID} \ + --client-secret ${POLARIS_ROOT_SECRET} \ + principal-roles \ + grant \ + --principal quickstart_user \ + quickstart_user_role + +./polaris_catalog/polaris \ + --client-id ${POLARIS_ROOT_ID} \ + --client-secret ${POLARIS_ROOT_SECRET} \ + catalog-roles \ + grant \ + --catalog quickstart_catalog \ + --principal-role quickstart_user_role \ + quickstart_catalog_role + +./polaris_catalog/polaris \ + --client-id ${POLARIS_ROOT_ID} \ + --client-secret ${POLARIS_ROOT_SECRET} \ + privileges \ + catalog \ + grant \ + --catalog quickstart_catalog \ + --catalog-role quickstart_catalog_role \ + CATALOG_MANAGE_CONTENT diff --git a/scripts/requirements-polaris.txt b/scripts/requirements-polaris.txt new file mode 100644 index 00000000..af385d6b --- /dev/null +++ b/scripts/requirements-polaris.txt @@ -0,0 +1,2 @@ +pydantic==2.11.1 +pyspark==3.5.0 \ No newline at end of file diff --git a/test/sql/local/irc/test_polaris.test b/test/sql/local/irc/test_polaris.test index fffeee0f..af3d324c 100644 --- a/test/sql/local/irc/test_polaris.test +++ b/test/sql/local/irc/test_polaris.test @@ -2,6 +2,10 @@ # description: test integration with iceberg catalog read # group: [irc] +require-env POLARIS_CLIENT_ID + +require-env POLARIS_CLIENT_SECRET + require avro require parquet @@ -12,17 +16,15 @@ require iceberg require aws -mode skip - statement ok create secret polaris_secret ( - type s3, - KEY_ID '', - SECRET '' + TYPE S3, + KEY_ID '${POLARIS_CLIENT_ID}', + SECRET '${POLARIS_CLIENT_SECRET}' ); statement ok -attach 'polaris_demo' as my_datalake ( +attach 'quickstart_catalog' as my_datalake ( type ICEBERG, ENDPOINT 'http://0.0.0.0:8181/api/catalog' ); From d78f435933b7270b7f3c02796304a83a920d108f Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 2 Apr 2025 14:15:20 +0200 Subject: [PATCH 32/66] add Makefile --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 1a0dc7ad..d737e724 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,15 @@ data: data_clean start-rest-catalog data_large: data data_clean python3 scripts/data_generators/generate_data.py +# setup polaris server. See PolarisTesting.yml to see instructions for a specific machine. +setup_polaris: + mkdir polaris_catalog + git clone https://github.com/apache/polaris.git polaris_catalog + cd polairs_catalog + jenv local 21 + ./gradlew --stop + nohup ./gradlew run > polaris-server.log 2> polaris-error.log & + data_clean: rm -rf data/generated From 72bda43f8d0c42405a1282b8b4d4dab4277da828 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 2 Apr 2025 16:07:12 +0200 Subject: [PATCH 33/66] mods to get some things working --- .github/workflows/PolarisTesting.yml | 3 +- Makefile | 11 +++--- scripts/polaris/create_data.py | 41 ++++++++++++--------- scripts/polaris/get_polaris_client_creds.py | 28 ++++++++------ scripts/polaris/setup_polaris_catalog.sh | 14 +++---- 5 files changed, 54 insertions(+), 43 deletions(-) diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index f20a8308..c19fb8d7 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -105,7 +105,8 @@ jobs: make setup_polaris python3 -m venv . source ./bin/activate - python3 -m pip install -r ../scripts/requirements-polaris.txt + python3 -m pip install poetry + python3 -m pip install spark==3.5.0 python3 scripts/polaris/get_polaris_root_creds.py # needed for setup_polaris_catalog.sh export POLARIS_ROOT_ID=$(cat polaris_root_id.txt) diff --git a/Makefile b/Makefile index d737e724..00254492 100644 --- a/Makefile +++ b/Makefile @@ -25,12 +25,11 @@ data_large: data data_clean # setup polaris server. See PolarisTesting.yml to see instructions for a specific machine. setup_polaris: - mkdir polaris_catalog - git clone https://github.com/apache/polaris.git polaris_catalog - cd polairs_catalog - jenv local 21 - ./gradlew --stop - nohup ./gradlew run > polaris-server.log 2> polaris-error.log & + mkdir polaris_catalog + git clone https://github.com/apache/polaris.git polaris_catalog + cd polaris_catalog && jenv local 21 + cd polaris_catalog && ./gradlew --stop + cd polaris_catalog && nohup ./gradlew run > polaris-server.log 2> polaris-error.log & data_clean: rm -rf data/generated diff --git a/scripts/polaris/create_data.py b/scripts/polaris/create_data.py index 5d6515d7..d76faeea 100644 --- a/scripts/polaris/create_data.py +++ b/scripts/polaris/create_data.py @@ -2,42 +2,47 @@ from pyspark.sql import SparkSession client_id = os.getenv('POLARIS_CLIENT_ID', '') -client_secret = os.getenv('POLARIS_SECRET_ID', '') +client_secret = os.getenv('POLARIS_CLIENT_SECRET', '') + +if client_id == '' or client_secret == '': + print("no client_id or client_secret") + exit(1) spark = (SparkSession.builder .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19") .config('spark.sql.iceberg.vectorization.enabled', 'false') # Configure the 'polaris' catalog as an Iceberg rest catalog - .config("spark.sql.catalog.polaris.type", "rest") - .config("spark.sql.catalog.polaris", "org.apache.iceberg.spark.SparkCatalog") + .config("spark.sql.catalog.quickstart_catalog.type", "rest") + .config("spark.sql.catalog.quickstart_catalog", "org.apache.iceberg.spark.SparkCatalog") # Specify the rest catalog endpoint - .config("spark.sql.catalog.polaris.uri", "http://polaris:8181/api/catalog") + .config("spark.sql.catalog.quickstart_catalog.uri", "http://localhost:8181/api/catalog") # Enable token refresh - .config("spark.sql.catalog.polaris.token-refresh-enabled", "true") + .config("spark.sql.catalog.quickstart_catalog.token-refresh-enabled", "true") # specify the client_id:client_secret pair - .config("spark.sql.catalog.polaris.credential", f"{client_id}:{client_secret}") + .config("spark.sql.catalog.quickstart_catalog.credential", f"{client_id}:{client_secret}") # Set the warehouse to the name of the catalog we created - .config("spark.sql.catalog.polaris.warehouse", catalog_name) + .config("spark.sql.catalog.quickstart_catalog.warehouse", "quickstart_catalog") # Scope set to PRINCIPAL_ROLE:ALL - .config("spark.sql.catalog.polaris.scope", 'PRINCIPAL_ROLE:ALL') + .config("spark.sql.catalog.quickstart_catalog.scope", 'PRINCIPAL_ROLE:ALL') # Enable access credential delegation - .config("spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation", 'vended-credentials') - .config("spark.sql.catalog.polaris.io-impl", "org.apache.iceberg.io.ResolvingFileIO") - .config("spark.sql.catalog.polaris.s3.region", "us-west-2") + .config("spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation", 'vended-credentials') + .config("spark.sql.catalog.quickstart_catalog.io-impl", "org.apache.iceberg.io.ResolvingFileIO") + .config("spark.sql.catalog.quickstart_catalog.s3.region", "us-west-2") .config("spark.history.fs.logDirectory", "/home/iceberg/spark-events")).getOrCreate() -spark.sql("USE polaris") +spark.sql("USE quickstart_catalog") spark.sql("SHOW NAMESPACES").show() spark.sql("CREATE NAMESPACE IF NOT EXISTS COLLADO_TEST") spark.sql("USE NAMESPACE COLLADO_TEST") -spark.sql("""CREATE TABLE IF NOT EXISTS TEST_TABLE ( - id bigint NOT NULL COMMENT 'unique id', - data string) -USING iceberg; +spark.sql(""" +CREATE TABLE IF NOT EXISTS quickstart_table ( + id BIGINT, data STRING +) +USING ICEBERG """) -spark.sql("INSERT INTO TEST_TABLE VALUES (1, 'some data'), (2, 'more data'), (3, 'yet more data')") -spark.sql("SELECT * FROM TEST_TABLE").show() +spark.sql("INSERT INTO quickstart_table VALUES (1, 'some data'), (2, 'more data'), (3, 'yet more data')") +spark.sql("SELECT * FROM quickstart_table").show() diff --git a/scripts/polaris/get_polaris_client_creds.py b/scripts/polaris/get_polaris_client_creds.py index bfc1bfc5..ef3b0e2d 100644 --- a/scripts/polaris/get_polaris_client_creds.py +++ b/scripts/polaris/get_polaris_client_creds.py @@ -1,21 +1,27 @@ import re import os -import json -# Read and parse the JSON file -with open("user_credentials.json", "r") as json_file: - config = json.load(json_file) +log_content = "" +# Read the log file (hopefully it isn't too big) +with open("polaris_catalog/user_credentials.json", "r") as file: + log_content = file.read() - client_id = config.get("clientId") - client_secret = config.get("clientSecret") +# Regular expression to capture the credentials +match = re.search(r"{\"clientId\": \"(\w+)\", \"clientSecret\": \"(\w+)\"}", log_content) - if client_id and client_secret: +if match: + clientId = match.group(1) + clientSecret = match.group(2) + if clientId and clientSecret: # Write client_id and client_secret to separate files with open("polaris_client_id.txt", "w") as id_file: - id_file.write(client_id) + print(f"clientId {clientId}")` + id_file.write(clientId) with open("polaris_client_secret.txt", "w") as secret_file: - secret_file.write(client_secret) - else: - print("clientId or clientSecret not found in config.json") + print(f"clientSecret {clientSecret}") + secret_file.write(clientSecret) +else: + print("Credentials not found in the log file.") + exit(1) diff --git a/scripts/polaris/setup_polaris_catalog.sh b/scripts/polaris/setup_polaris_catalog.sh index 834e2d38..aa1b1874 100755 --- a/scripts/polaris/setup_polaris_catalog.sh +++ b/scripts/polaris/setup_polaris_catalog.sh @@ -1,7 +1,7 @@ # TODO: use the python module to execute these steps. # Seems like the python module is not yet publicly available/installable, so unsure what to do about this. -./polaris_catalog/polaris \ +./polaris \ --client-id ${POLARIS_ROOT_ID} \ --client-secret ${POLARIS_ROOT_SECRET} \ catalogs \ @@ -11,21 +11,21 @@ quickstart_catalog -./polaris_catalog/polaris \ +./polaris \ --client-id ${POLARIS_ROOT_ID} \ --client-secret ${POLARIS_ROOT_SECRET} \ principals \ create \ quickstart_user -./polaris_catalog/polaris \ +./polaris \ --client-id ${POLARIS_ROOT_ID} \ --client-secret ${POLARIS_ROOT_SECRET} \ principal-roles \ create \ quickstart_user_role -./polaris_catalog/polaris \ +./polaris \ --client-id ${POLARIS_ROOT_ID} \ --client-secret ${POLARIS_ROOT_SECRET} \ catalog-roles \ @@ -33,7 +33,7 @@ --catalog quickstart_catalog \ quickstart_catalog_role -./polaris_catalog/polaris \ +./polaris \ --client-id ${POLARIS_ROOT_ID} \ --client-secret ${POLARIS_ROOT_SECRET} \ principal-roles \ @@ -41,7 +41,7 @@ --principal quickstart_user \ quickstart_user_role -./polaris_catalog/polaris \ +./polaris \ --client-id ${POLARIS_ROOT_ID} \ --client-secret ${POLARIS_ROOT_SECRET} \ catalog-roles \ @@ -50,7 +50,7 @@ --principal-role quickstart_user_role \ quickstart_catalog_role -./polaris_catalog/polaris \ +./polaris \ --client-id ${POLARIS_ROOT_ID} \ --client-secret ${POLARIS_ROOT_SECRET} \ privileges \ From 68b47d1de9db95291fbbd84dc89bb56d059a85b8 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 2 Apr 2025 16:15:36 +0200 Subject: [PATCH 34/66] copy all the options from the referenced storage secret to the scoped secret we create from the returned 'config' of the LoadTableResult --- src/catalog_api.cpp | 5 +++-- test/sql/local/iceberg_on_tpch.test | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 373705ef..115a74a4 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -137,8 +137,9 @@ IRCAPITableCredentials IRCAPI::GetTableCredentials(ClientContext &context, IRCat auto *config_val = yyjson_obj_get(root, "config"); if (config_val && catalog_credentials) { auto kv_secret = dynamic_cast(*catalog_credentials->secret); - auto region = kv_secret.TryGetValue("region").ToString(); - config_options["region"] = region; + for (auto &option : kv_secret.secret_map) { + config_options.emplace(option); + } } ParseConfigOptions(config_val, config_options); diff --git a/test/sql/local/iceberg_on_tpch.test b/test/sql/local/iceberg_on_tpch.test index db725256..8b9b8a71 100644 --- a/test/sql/local/iceberg_on_tpch.test +++ b/test/sql/local/iceberg_on_tpch.test @@ -22,7 +22,7 @@ CREATE SECRET ( ENDPOINT '127.0.0.1:9000', URL_STYLE 'path', USE_SSL 0 - ); +); statement ok ATTACH '' AS my_datalake ( From f2a5e42a3c8252ffa8270366b740745263b412b1 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 2 Apr 2025 16:30:06 +0200 Subject: [PATCH 35/66] test polaris in its own workflow (for now) --- .github/workflows/PolarisTesting.yml | 10 +++++++--- scripts/polaris/get_polaris_client_creds.py | 2 +- test/sql/local/irc/test_polaris.test | 12 +++++++++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index c19fb8d7..f4746801 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -103,18 +103,22 @@ jobs: - name: Setup Polaris run: | make setup_polaris + # let polaris initialize + sleep 30 python3 -m venv . source ./bin/activate python3 -m pip install poetry - python3 -m pip install spark==3.5.0 + python3 -m pip install pyspark==3.5.0 python3 scripts/polaris/get_polaris_root_creds.py # needed for setup_polaris_catalog.sh export POLARIS_ROOT_ID=$(cat polaris_root_id.txt) export POLARIS_ROOT_SECRET=$(cat polaris_root_password.txt) - ./scripts/polaris/setup_polaris_catalog.sh > user_credentials.json + cd polaris_catalog && ../scripts/polaris/setup_polaris_catalog.sh > user_credentials.json + cd .. + python3 scripts/polaris/get_polaris_client_creds.py export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) - ./scripts/polaris/create_data.py + python3 scripts/polaris/create_data.py - name: Test with rest catalog env: diff --git a/scripts/polaris/get_polaris_client_creds.py b/scripts/polaris/get_polaris_client_creds.py index ef3b0e2d..c940156f 100644 --- a/scripts/polaris/get_polaris_client_creds.py +++ b/scripts/polaris/get_polaris_client_creds.py @@ -15,7 +15,7 @@ if clientId and clientSecret: # Write client_id and client_secret to separate files with open("polaris_client_id.txt", "w") as id_file: - print(f"clientId {clientId}")` + print(f"clientId {clientId}") id_file.write(clientId) with open("polaris_client_secret.txt", "w") as secret_file: diff --git a/test/sql/local/irc/test_polaris.test b/test/sql/local/irc/test_polaris.test index af3d324c..a4bc18ac 100644 --- a/test/sql/local/irc/test_polaris.test +++ b/test/sql/local/irc/test_polaris.test @@ -32,9 +32,15 @@ attach 'quickstart_catalog' as my_datalake ( statement ok show all tables; -query II -select * from namespace.catalog.table; +query I +select count(*) from (show all tables); ---- 1 -2 + +query II +select * from my_datalake.COLLADO_TEST.quickstart_table; +---- +1 some data +2 more data +3 yet more data From fb80c4006aace193e570de218ce87f475bd4b6a4 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 2 Apr 2025 16:35:30 +0200 Subject: [PATCH 36/66] put polaris CI in existing local test --- .github/workflows/LocalTesting.yml | 46 ++++++++++++++++++++++++++++ test/sql/local/irc/test_polaris.test | 2 ++ 2 files changed, 48 insertions(+) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 4bb021b3..8ff9b4cc 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -94,4 +94,50 @@ jobs: ICEBERG_SERVER_AVAILABLE: 1 DUCKDB_ICEBERG_HAVE_GENERATED_DATA: 1 run: | + make test_release + + - name: Set up for Polaris + run: | + # install java + sudo apt install -y -qq openjdk-21-jre-headless + sudo apt install -y -qq openjdk-21-jdk-headless + # install python virtual environment (is this needed?) + sudo apt-get install -y -qq python3-venv + + - name: Setup Jenv + run: | + git clone https://github.com/jenv/jenv.git ~/.jenv + echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bash_profile + source ~/.bash_profile + eval "$(jenv init -)" + jenv enable-plugin export + exec $SHELL -l + jenv add /usr/lib/jvm/java-21-openjdk-amd64 + + - name: Setup Polaris + run: | + make setup_polaris + # let polaris initialize + sleep 30 + python3 -m venv . + source ./bin/activate + python3 -m pip install poetry + python3 -m pip install pyspark==3.5.0 + python3 scripts/polaris/get_polaris_root_creds.py + # needed for setup_polaris_catalog.sh + export POLARIS_ROOT_ID=$(cat polaris_root_id.txt) + export POLARIS_ROOT_SECRET=$(cat polaris_root_password.txt) + cd polaris_catalog && ../scripts/polaris/setup_polaris_catalog.sh > user_credentials.json + cd .. + python3 scripts/polaris/get_polaris_client_creds.py + export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) + export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) + python3 scripts/polaris/create_data.py + + - name: Test with rest catalog + env: + POLARIS_SERVER_AVAILABLE: 1 + run: | + export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) + export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) make test_release \ No newline at end of file diff --git a/test/sql/local/irc/test_polaris.test b/test/sql/local/irc/test_polaris.test index a4bc18ac..8c778ff9 100644 --- a/test/sql/local/irc/test_polaris.test +++ b/test/sql/local/irc/test_polaris.test @@ -6,6 +6,8 @@ require-env POLARIS_CLIENT_ID require-env POLARIS_CLIENT_SECRET +require-env POLARIS_SERVER_AVAILABLE + require avro require parquet From 854d43ac2c739e7af6251422dd8bd48cb3d17227 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 2 Apr 2025 23:08:36 +0200 Subject: [PATCH 37/66] fix compilation for linux arm64 --- CMakeLists.txt | 3 +++ src/storage/irc_table_set.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 73d3203b..8b9e8e13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,9 @@ find_package(CURL REQUIRED) find_package(AWSSDK REQUIRED COMPONENTS core sso sts) include_directories(${CURL_INCLUDE_DIRS}) +# Reset the TARGET_NAME, the AWS find_package build could bleed into our build - overriding `TARGET_NAME` +set(TARGET_NAME iceberg) + # AWS SDK FROM vcpkg target_include_directories(${EXTENSION_NAME} PUBLIC $) diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index 38573b56..71bac284 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -35,7 +35,7 @@ unique_ptr ICTableSet::_CreateCatalogEntry(ClientContext &context, auto table_entry = make_uniq(catalog, schema, info); table_entry->table_data = make_uniq(table); - return table_entry; + return std::move(table_entry); } void ICTableSet::FillEntry(ClientContext &context, unique_ptr &entry) { From 3d3997ad2678b672b5d0cb887562c1181f3cae57 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 2 Apr 2025 23:50:03 +0200 Subject: [PATCH 38/66] add environment variable to force minimum cmake requirement to 3.5, to fix compilation error on cmake4.0.0 --- .github/workflows/CloudTesting.yml | 3 +++ .github/workflows/HighPriorityIssues.yml | 1 + .github/workflows/LocalTesting.yml | 1 + 3 files changed, 5 insertions(+) diff --git a/.github/workflows/CloudTesting.yml b/.github/workflows/CloudTesting.yml index ccdefef0..81a24820 100644 --- a/.github/workflows/CloudTesting.yml +++ b/.github/workflows/CloudTesting.yml @@ -7,6 +7,9 @@ defaults: run: shell: bash +env: + CMAKE_POLICY_VERSION_MINIMUM: 3.5 + jobs: rest: name: Test against remote AWS account diff --git a/.github/workflows/HighPriorityIssues.yml b/.github/workflows/HighPriorityIssues.yml index 6adc6d87..919b6ac3 100644 --- a/.github/workflows/HighPriorityIssues.yml +++ b/.github/workflows/HighPriorityIssues.yml @@ -10,6 +10,7 @@ env: # hence only one of the numbers will be filled in the TITLE_PREFIX TITLE_PREFIX: "[duckdb_iceberg/#${{ github.event.issue.number }}]" PUBLIC_ISSUE_TITLE: ${{ github.event.issue.title }} + CMAKE_POLICY_VERSION_MINIMUM: 3.5 jobs: create_or_label_issue: diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 4bb021b3..c2f79fb7 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -9,6 +9,7 @@ defaults: env: BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} + CMAKE_POLICY_VERSION_MINIMUM: 3.5 jobs: rest: From 4dac4c40d47c7e78da3a0a7c74b968dab7a199ea Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 2 Apr 2025 16:39:08 +0200 Subject: [PATCH 39/66] remove steps --- .github/workflows/MainDistributionPipeline.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index b7e308a1..e4ca91fb 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -12,16 +12,8 @@ concurrency: cancel-in-progress: true jobs: - duckdb-stable-build: name: Build extension binaries - steps: - - name: Install Ninja (Linux) - if: runner.os == 'Linux' - run: | - sudo apt-get update -y -qq - sudo apt-get install -y -qq ninja-build - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1 with: extension_name: iceberg From 333e210d93e4ade0e1a96810050579ef1a6bffcd Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 2 Apr 2025 09:51:48 +0200 Subject: [PATCH 40/66] Bump cmake_minimum_required to range (tracking duckdb/duckdb) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b9e8e13..a1eae37b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 3.5...3.29) # Set extension name here set(TARGET_NAME iceberg) From 2f04f321f55bc4c12ae7314c20d10b5158f7be25 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Wed, 2 Apr 2025 12:27:30 +0200 Subject: [PATCH 41/66] Move avro to subfolder --- vcpkg_ports/{ => avro-c}/avro.patch | 0 vcpkg_ports/{ => avro-c}/duckdb.patch | 0 vcpkg_ports/{ => avro-c}/portfile.cmake | 0 vcpkg_ports/{ => avro-c}/vcpkg.json | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename vcpkg_ports/{ => avro-c}/avro.patch (100%) rename vcpkg_ports/{ => avro-c}/duckdb.patch (100%) rename vcpkg_ports/{ => avro-c}/portfile.cmake (100%) rename vcpkg_ports/{ => avro-c}/vcpkg.json (100%) diff --git a/vcpkg_ports/avro.patch b/vcpkg_ports/avro-c/avro.patch similarity index 100% rename from vcpkg_ports/avro.patch rename to vcpkg_ports/avro-c/avro.patch diff --git a/vcpkg_ports/duckdb.patch b/vcpkg_ports/avro-c/duckdb.patch similarity index 100% rename from vcpkg_ports/duckdb.patch rename to vcpkg_ports/avro-c/duckdb.patch diff --git a/vcpkg_ports/portfile.cmake b/vcpkg_ports/avro-c/portfile.cmake similarity index 100% rename from vcpkg_ports/portfile.cmake rename to vcpkg_ports/avro-c/portfile.cmake diff --git a/vcpkg_ports/vcpkg.json b/vcpkg_ports/avro-c/vcpkg.json similarity index 100% rename from vcpkg_ports/vcpkg.json rename to vcpkg_ports/avro-c/vcpkg.json From cb6c40a881cadfb6d2a908f2aab7dd2e6d7d2b1f Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 3 Apr 2025 10:17:03 +0200 Subject: [PATCH 42/66] Add vcpkg-cmake --- vcpkg_ports/vcpkg-cmake/portfile.cmake | 14 + .../vcpkg-cmake/vcpkg-port-config.cmake | 3 + vcpkg_ports/vcpkg-cmake/vcpkg.json | 6 + .../vcpkg-cmake/vcpkg_cmake_build.cmake | 91 +++++ .../vcpkg-cmake/vcpkg_cmake_configure.cmake | 353 ++++++++++++++++++ .../vcpkg-cmake/vcpkg_cmake_install.cmake | 21 ++ 6 files changed, 488 insertions(+) create mode 100644 vcpkg_ports/vcpkg-cmake/portfile.cmake create mode 100644 vcpkg_ports/vcpkg-cmake/vcpkg-port-config.cmake create mode 100644 vcpkg_ports/vcpkg-cmake/vcpkg.json create mode 100644 vcpkg_ports/vcpkg-cmake/vcpkg_cmake_build.cmake create mode 100644 vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake create mode 100644 vcpkg_ports/vcpkg-cmake/vcpkg_cmake_install.cmake diff --git a/vcpkg_ports/vcpkg-cmake/portfile.cmake b/vcpkg_ports/vcpkg-cmake/portfile.cmake new file mode 100644 index 00000000..0b7dd502 --- /dev/null +++ b/vcpkg_ports/vcpkg-cmake/portfile.cmake @@ -0,0 +1,14 @@ +if(VCPKG_CROSSCOMPILING) + # make FATAL_ERROR in CI when issue #16773 fixed + message(WARNING "vcpkg-cmake is a host-only port; please mark it as a host port in your dependencies.") +endif() + +file(INSTALL + "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_configure.cmake" + "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_build.cmake" + "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_install.cmake" + "${CMAKE_CURRENT_LIST_DIR}/vcpkg-port-config.cmake" + DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") + +file(INSTALL "${VCPKG_ROOT_DIR}/LICENSE.txt" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) +set(VCPKG_POLICY_CMAKE_HELPER_PORT enabled) diff --git a/vcpkg_ports/vcpkg-cmake/vcpkg-port-config.cmake b/vcpkg_ports/vcpkg-cmake/vcpkg-port-config.cmake new file mode 100644 index 00000000..f2a973d4 --- /dev/null +++ b/vcpkg_ports/vcpkg-cmake/vcpkg-port-config.cmake @@ -0,0 +1,3 @@ +include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_configure.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_build.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_install.cmake") diff --git a/vcpkg_ports/vcpkg-cmake/vcpkg.json b/vcpkg_ports/vcpkg-cmake/vcpkg.json new file mode 100644 index 00000000..fa484eaf --- /dev/null +++ b/vcpkg_ports/vcpkg-cmake/vcpkg.json @@ -0,0 +1,6 @@ +{ + "name": "vcpkg-cmake", + "version-date": "2024-04-23", + "documentation": "https://learn.microsoft.com/vcpkg/maintainers/functions/vcpkg_cmake_configure", + "license": "MIT" +} diff --git a/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_build.cmake b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_build.cmake new file mode 100644 index 00000000..47933b3f --- /dev/null +++ b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_build.cmake @@ -0,0 +1,91 @@ +include_guard(GLOBAL) + +function(vcpkg_cmake_build) + cmake_parse_arguments(PARSE_ARGV 0 "arg" "DISABLE_PARALLEL;ADD_BIN_TO_PATH" "TARGET;LOGFILE_BASE" "") + + if(DEFINED arg_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "vcpkg_cmake_build was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") + endif() + if(NOT DEFINED arg_LOGFILE_BASE) + set(arg_LOGFILE_BASE "build") + endif() + vcpkg_list(SET build_param) + vcpkg_list(SET parallel_param) + vcpkg_list(SET no_parallel_param) + + if("${Z_VCPKG_CMAKE_GENERATOR}" STREQUAL "Ninja") + vcpkg_list(SET build_param "-v") # verbose output + vcpkg_list(SET parallel_param "-j${VCPKG_CONCURRENCY}") + vcpkg_list(SET no_parallel_param "-j1") + elseif("${Z_VCPKG_CMAKE_GENERATOR}" MATCHES "^Visual Studio") + vcpkg_list(SET build_param + "/p:VCPkgLocalAppDataDisabled=true" + "/p:UseIntelMKL=No" + ) + vcpkg_list(SET parallel_param "/m") + elseif("${Z_VCPKG_CMAKE_GENERATOR}" STREQUAL "NMake Makefiles") + # No options are currently added for nmake builds + elseif(Z_VCPKG_CMAKE_GENERATOR STREQUAL "Unix Makefiles") + vcpkg_list(SET build_param "VERBOSE=1") + vcpkg_list(SET parallel_param "-j${VCPKG_CONCURRENCY}") + vcpkg_list(SET no_parallel_param "") + elseif(Z_VCPKG_CMAKE_GENERATOR STREQUAL "Xcode") + vcpkg_list(SET parallel_param -jobs "${VCPKG_CONCURRENCY}") + vcpkg_list(SET no_parallel_param -jobs 1) + else() + message(WARNING "Unrecognized GENERATOR setting from vcpkg_cmake_configure().") + endif() + + vcpkg_list(SET target_param) + if(arg_TARGET) + vcpkg_list(SET target_param "--target" "${arg_TARGET}") + endif() + + foreach(build_type IN ITEMS debug release) + if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "${build_type}") + if("${build_type}" STREQUAL "debug") + set(short_build_type "dbg") + set(config "Debug") + else() + set(short_build_type "rel") + set(config "Release") + endif() + + message(STATUS "Building ${TARGET_TRIPLET}-${short_build_type}") + + if(arg_ADD_BIN_TO_PATH) + vcpkg_backup_env_variables(VARS PATH) + if("${build_type}" STREQUAL "debug") + vcpkg_add_to_path(PREPEND "${CURRENT_INSTALLED_DIR}/debug/bin") + else() + vcpkg_add_to_path(PREPEND "${CURRENT_INSTALLED_DIR}/bin") + endif() + endif() + + if(arg_DISABLE_PARALLEL) + vcpkg_execute_build_process( + COMMAND + "${CMAKE_COMMAND}" --build . --config "${config}" ${target_param} + -- ${build_param} ${no_parallel_param} + WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${short_build_type}" + LOGNAME "${arg_LOGFILE_BASE}-${TARGET_TRIPLET}-${short_build_type}" + ) + else() + vcpkg_execute_build_process( + COMMAND + "${CMAKE_COMMAND}" --build . --config "${config}" ${target_param} + -- ${build_param} ${parallel_param} + NO_PARALLEL_COMMAND + "${CMAKE_COMMAND}" --build . --config "${config}" ${target_param} + -- ${build_param} ${no_parallel_param} + WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${short_build_type}" + LOGNAME "${arg_LOGFILE_BASE}-${TARGET_TRIPLET}-${short_build_type}" + ) + endif() + + if(arg_ADD_BIN_TO_PATH) + vcpkg_restore_env_variables(VARS PATH) + endif() + endif() + endforeach() +endfunction() diff --git a/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake new file mode 100644 index 00000000..acd510f5 --- /dev/null +++ b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake @@ -0,0 +1,353 @@ +include_guard(GLOBAL) + +macro(z_vcpkg_cmake_configure_both_set_or_unset var1 var2) + if(DEFINED ${var1} AND NOT DEFINED ${var2}) + message(FATAL_ERROR "If ${var1} is set, then ${var2} must be set.") + elseif(NOT DEFINED ${var1} AND DEFINED ${var2}) + message(FATAL_ERROR "If ${var2} is set, then ${var1} must be set.") + endif() +endmacro() + +function(vcpkg_cmake_configure) + cmake_parse_arguments(PARSE_ARGV 0 "arg" + "PREFER_NINJA;DISABLE_PARALLEL_CONFIGURE;WINDOWS_USE_MSBUILD;NO_CHARSET_FLAG;Z_CMAKE_GET_VARS_USAGE" + "SOURCE_PATH;GENERATOR;LOGFILE_BASE" + "OPTIONS;OPTIONS_DEBUG;OPTIONS_RELEASE;MAYBE_UNUSED_VARIABLES" + ) + + if(NOT arg_Z_CMAKE_GET_VARS_USAGE AND DEFINED CACHE{Z_VCPKG_CMAKE_GENERATOR}) + message(WARNING "${CMAKE_CURRENT_FUNCTION} already called; this function should only be called once.") + endif() + if(arg_PREFER_NINJA) + message(WARNING "PREFER_NINJA has been deprecated in ${CMAKE_CURRENT_FUNCTION}. Please remove it from the portfile!") + endif() + + if(DEFINED arg_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "${CMAKE_CURRENT_FUNCTION} was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") + endif() + + if(NOT DEFINED arg_SOURCE_PATH) + message(FATAL_ERROR "SOURCE_PATH must be set") + endif() + if(NOT DEFINED arg_LOGFILE_BASE) + set(arg_LOGFILE_BASE "config-${TARGET_TRIPLET}") + endif() + + set(invalid_maybe_unused_vars "${arg_MAYBE_UNUSED_VARIABLES}") + list(FILTER invalid_maybe_unused_vars INCLUDE REGEX "^-D") + if(NOT invalid_maybe_unused_vars STREQUAL "") + list(JOIN invalid_maybe_unused_vars " " bad_items) + message(${Z_VCPKG_BACKCOMPAT_MESSAGE_LEVEL} + "Option MAYBE_UNUSED_VARIABLES must be used with variables names. " + "The following items are invalid: ${bad_items}") + endif() + + set(manually_specified_variables "") + + if(arg_Z_CMAKE_GET_VARS_USAGE) + set(configuring_message "Getting CMake variables for ${TARGET_TRIPLET}") + else() + set(configuring_message "Configuring ${TARGET_TRIPLET}") + + foreach(option IN LISTS arg_OPTIONS arg_OPTIONS_RELEASE arg_OPTIONS_DEBUG) + if("${option}" MATCHES "^-D([^:=]*)[:=]") + vcpkg_list(APPEND manually_specified_variables "${CMAKE_MATCH_1}") + endif() + endforeach() + vcpkg_list(REMOVE_DUPLICATES manually_specified_variables) + foreach(maybe_unused_var IN LISTS arg_MAYBE_UNUSED_VARIABLES) + vcpkg_list(REMOVE_ITEM manually_specified_variables "${maybe_unused_var}") + endforeach() + debug_message("manually specified variables: ${manually_specified_variables}") + endif() + + if(CMAKE_HOST_WIN32) + if(DEFINED ENV{PROCESSOR_ARCHITEW6432}) + set(host_architecture "$ENV{PROCESSOR_ARCHITEW6432}") + else() + set(host_architecture "$ENV{PROCESSOR_ARCHITECTURE}") + endif() + endif() + + set(ninja_host ON) # Ninja availability + if(host_architecture STREQUAL "x86" OR DEFINED ENV{VCPKG_FORCE_SYSTEM_BINARIES}) + # Prebuilt ninja binaries are only provided for x64 hosts + find_program(NINJA NAMES ninja ninja-build) + if(NOT NINJA) + set(ninja_host OFF) + set(arg_DISABLE_PARALLEL_CONFIGURE ON) + set(arg_WINDOWS_USE_MSBUILD ON) + endif() + endif() + + set(generator "") + set(architecture_options "") + if(arg_WINDOWS_USE_MSBUILD AND VCPKG_HOST_IS_WINDOWS AND VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + z_vcpkg_get_visual_studio_generator(OUT_GENERATOR generator OUT_ARCH arch) + vcpkg_list(APPEND architecture_options "-A${arch}") + if(DEFINED VCPKG_PLATFORM_TOOLSET) + vcpkg_list(APPEND arg_OPTIONS "-T${VCPKG_PLATFORM_TOOLSET}") + endif() + if(NOT generator) + message(FATAL_ERROR "Unable to determine appropriate Visual Studio generator for triplet ${TARGET_TRIPLET}: + ENV{VisualStudioVersion} : $ENV{VisualStudioVersion} + VCPKG_TARGET_ARCHITECTURE: ${VCPKG_TARGET_ARCHITECTURE}") + endif() + elseif(DEFINED arg_GENERATOR) + set(generator "${arg_GENERATOR}") + elseif(ninja_host) + set(generator "Ninja") + elseif(NOT VCPKG_HOST_IS_WINDOWS) + set(generator "Unix Makefiles") + endif() + + if(NOT generator) + if(NOT VCPKG_CMAKE_SYSTEM_NAME) + set(VCPKG_CMAKE_SYSTEM_NAME "Windows") + endif() + message(FATAL_ERROR "Unable to determine appropriate generator for: " + "${VCPKG_CMAKE_SYSTEM_NAME}-${VCPKG_TARGET_ARCHITECTURE}-${VCPKG_PLATFORM_TOOLSET}") + endif() + + set(parallel_log_args "") + set(log_args "") + + if(generator STREQUAL "Ninja") + vcpkg_find_acquire_program(NINJA) + vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_MAKE_PROGRAM=${NINJA}") + # If we use Ninja, it must be on PATH for CMake's ExternalProject, + # cf. https://gitlab.kitware.com/cmake/cmake/-/issues/23355. + get_filename_component(ninja_path "${NINJA}" DIRECTORY) + vcpkg_add_to_path("${ninja_path}") + set(parallel_log_args + "../build.ninja" ALIAS "rel-ninja.log" + "../../${TARGET_TRIPLET}-dbg/build.ninja" ALIAS "dbg-ninja.log" + ) + set(log_args "build.ninja") + endif() + + set(build_dir_release "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel") + set(build_dir_debug "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg") + file(REMOVE_RECURSE + "${build_dir_release}" + "${build_dir_debug}") + file(MAKE_DIRECTORY "${build_dir_release}") + if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + file(MAKE_DIRECTORY "${build_dir_debug}") + endif() + + if(DEFINED VCPKG_CMAKE_SYSTEM_NAME) + vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_SYSTEM_NAME=${VCPKG_CMAKE_SYSTEM_NAME}") + if(VCPKG_TARGET_IS_UWP AND NOT DEFINED VCPKG_CMAKE_SYSTEM_VERSION) + set(VCPKG_CMAKE_SYSTEM_VERSION 10.0) + elseif(VCPKG_TARGET_IS_ANDROID AND NOT DEFINED VCPKG_CMAKE_SYSTEM_VERSION) + set(VCPKG_CMAKE_SYSTEM_VERSION 21) + endif() + endif() + + if(DEFINED VCPKG_CMAKE_SYSTEM_VERSION) + vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_SYSTEM_VERSION=${VCPKG_CMAKE_SYSTEM_VERSION}") + endif() + + if(DEFINED VCPKG_XBOX_CONSOLE_TARGET) + vcpkg_list(APPEND arg_OPTIONS "-DXBOX_CONSOLE_TARGET=${VCPKG_XBOX_CONSOLE_TARGET}") + endif() + + if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") + vcpkg_list(APPEND arg_OPTIONS "-DBUILD_SHARED_LIBS=ON") + elseif(VCPKG_LIBRARY_LINKAGE STREQUAL "static") + vcpkg_list(APPEND arg_OPTIONS "-DBUILD_SHARED_LIBS=OFF") + else() + message(FATAL_ERROR + "Invalid setting for VCPKG_LIBRARY_LINKAGE: \"${VCPKG_LIBRARY_LINKAGE}\". " + "It must be \"static\" or \"dynamic\"") + endif() + + z_vcpkg_cmake_configure_both_set_or_unset(VCPKG_CXX_FLAGS_DEBUG VCPKG_C_FLAGS_DEBUG) + z_vcpkg_cmake_configure_both_set_or_unset(VCPKG_CXX_FLAGS_RELEASE VCPKG_C_FLAGS_RELEASE) + z_vcpkg_cmake_configure_both_set_or_unset(VCPKG_CXX_FLAGS VCPKG_C_FLAGS) + + set(VCPKG_SET_CHARSET_FLAG ON) + if(arg_NO_CHARSET_FLAG) + set(VCPKG_SET_CHARSET_FLAG OFF) + endif() + + if(NOT DEFINED VCPKG_CHAINLOAD_TOOLCHAIN_FILE) + z_vcpkg_select_default_vcpkg_chainload_toolchain() + endif() + + list(JOIN VCPKG_TARGET_ARCHITECTURE "\;" target_architecture_string) + vcpkg_list(APPEND arg_OPTIONS + "-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}" + "-DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET}" + "-DVCPKG_SET_CHARSET_FLAG=${VCPKG_SET_CHARSET_FLAG}" + "-DVCPKG_PLATFORM_TOOLSET=${VCPKG_PLATFORM_TOOLSET}" + "-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON" + "-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON" + "-DCMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY=ON" + "-DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=TRUE" + "-DCMAKE_VERBOSE_MAKEFILE=ON" + "-DVCPKG_APPLOCAL_DEPS=OFF" + "-DCMAKE_TOOLCHAIN_FILE=${SCRIPTS}/buildsystems/vcpkg.cmake" + "-DCMAKE_ERROR_ON_ABSOLUTE_INSTALL_DESTINATION=ON" + "-DVCPKG_CXX_FLAGS=${VCPKG_CXX_FLAGS}" + "-DVCPKG_CXX_FLAGS_RELEASE=${VCPKG_CXX_FLAGS_RELEASE}" + "-DVCPKG_CXX_FLAGS_DEBUG=${VCPKG_CXX_FLAGS_DEBUG}" + "-DVCPKG_C_FLAGS=${VCPKG_C_FLAGS}" + "-DVCPKG_C_FLAGS_RELEASE=${VCPKG_C_FLAGS_RELEASE}" + "-DVCPKG_C_FLAGS_DEBUG=${VCPKG_C_FLAGS_DEBUG}" + "-DVCPKG_CRT_LINKAGE=${VCPKG_CRT_LINKAGE}" + "-DVCPKG_LINKER_FLAGS=${VCPKG_LINKER_FLAGS}" + "-DVCPKG_LINKER_FLAGS_RELEASE=${VCPKG_LINKER_FLAGS_RELEASE}" + "-DVCPKG_LINKER_FLAGS_DEBUG=${VCPKG_LINKER_FLAGS_DEBUG}" + "-DVCPKG_TARGET_ARCHITECTURE=${target_architecture_string}" + "-DCMAKE_INSTALL_LIBDIR:STRING=lib" + "-DCMAKE_INSTALL_BINDIR:STRING=bin" + "-D_VCPKG_ROOT_DIR=${VCPKG_ROOT_DIR}" + "-D_VCPKG_INSTALLED_DIR=${_VCPKG_INSTALLED_DIR}" + "-DVCPKG_MANIFEST_INSTALL=OFF" + ) + + # Sets configuration variables for macOS builds + foreach(config_var IN ITEMS INSTALL_NAME_DIR OSX_DEPLOYMENT_TARGET OSX_SYSROOT OSX_ARCHITECTURES) + if(DEFINED VCPKG_${config_var}) + vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_${config_var}=${VCPKG_${config_var}}") + endif() + endforeach() + + vcpkg_list(PREPEND arg_OPTIONS "-DFETCHCONTENT_FULLY_DISCONNECTED=ON") + + # Allow overrides / additional configuration variables from triplets + if(DEFINED VCPKG_CMAKE_CONFIGURE_OPTIONS) + vcpkg_list(APPEND arg_OPTIONS ${VCPKG_CMAKE_CONFIGURE_OPTIONS}) + endif() + if(DEFINED VCPKG_CMAKE_CONFIGURE_OPTIONS_RELEASE) + vcpkg_list(APPEND arg_OPTIONS_RELEASE ${VCPKG_CMAKE_CONFIGURE_OPTIONS_RELEASE}) + endif() + if(DEFINED VCPKG_CMAKE_CONFIGURE_OPTIONS_DEBUG) + vcpkg_list(APPEND arg_OPTIONS_DEBUG ${VCPKG_CMAKE_CONFIGURE_OPTIONS_DEBUG}) + endif() + + vcpkg_list(SET rel_command + "${CMAKE_COMMAND}" "${arg_SOURCE_PATH}" + -G "${generator}" + ${architecture_options} + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX=${CURRENT_PACKAGES_DIR}" + ${arg_OPTIONS} ${arg_OPTIONS_RELEASE}) + vcpkg_list(SET dbg_command + "${CMAKE_COMMAND}" "${arg_SOURCE_PATH}" + -G "${generator}" + ${architecture_options} + "-DCMAKE_BUILD_TYPE=Debug" + "-DCMAKE_INSTALL_PREFIX=${CURRENT_PACKAGES_DIR}/debug" + ${arg_OPTIONS} ${arg_OPTIONS_DEBUG}) + + if(NOT arg_DISABLE_PARALLEL_CONFIGURE) + vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_DISABLE_SOURCE_CHANGES=ON") + + vcpkg_find_acquire_program(NINJA) + + #parallelize the configure step + set(ninja_configure_contents + "rule CreateProcess\n command = \$process\n\n" + ) + + if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "release") + z_vcpkg_configure_cmake_build_cmakecache(ninja_configure_contents ".." "rel") + endif() + if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "debug") + z_vcpkg_configure_cmake_build_cmakecache(ninja_configure_contents "../../${TARGET_TRIPLET}-dbg" "dbg") + endif() + + file(MAKE_DIRECTORY "${build_dir_release}/vcpkg-parallel-configure") + file(WRITE + "${build_dir_release}/vcpkg-parallel-configure/build.ninja" + "${ninja_configure_contents}") + + message(STATUS "${configuring_message}") + vcpkg_execute_required_process( + COMMAND "${NINJA}" -v + WORKING_DIRECTORY "${build_dir_release}/vcpkg-parallel-configure" + LOGNAME "${arg_LOGFILE_BASE}" + SAVE_LOG_FILES + "../../${TARGET_TRIPLET}-dbg/CMakeCache.txt" ALIAS "dbg-CMakeCache.txt.log" + "../CMakeCache.txt" ALIAS "rel-CMakeCache.txt.log" + "../../${TARGET_TRIPLET}-dbg/CMakeFiles/CMakeConfigureLog.yaml" ALIAS "dbg-CMakeConfigureLog.yaml.log" + "../CMakeFiles/CMakeConfigureLog.yaml" ALIAS "rel-CMakeConfigureLog.yaml.log" + ${parallel_log_args} + ) + + vcpkg_list(APPEND config_logs + "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-out.log" + "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-err.log") + else() + if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "debug") + message(STATUS "${configuring_message}-dbg") + vcpkg_execute_required_process( + COMMAND ${dbg_command} + WORKING_DIRECTORY "${build_dir_debug}" + LOGNAME "${arg_LOGFILE_BASE}-dbg" + SAVE_LOG_FILES + "CMakeCache.txt" + "CMakeFiles/CMakeConfigureLog.yaml" + ${log_args} + ) + vcpkg_list(APPEND config_logs + "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-dbg-out.log" + "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-dbg-err.log") + endif() + + if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "release") + message(STATUS "${configuring_message}-rel") + vcpkg_execute_required_process( + COMMAND ${rel_command} + WORKING_DIRECTORY "${build_dir_release}" + LOGNAME "${arg_LOGFILE_BASE}-rel" + SAVE_LOG_FILES + "CMakeCache.txt" + "CMakeFiles/CMakeConfigureLog.yaml" + ${log_args} + ) + vcpkg_list(APPEND config_logs + "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-rel-out.log" + "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-rel-err.log") + endif() + endif() + + set(all_unused_variables) + foreach(config_log IN LISTS config_logs) + if(NOT EXISTS "${config_log}") + continue() + endif() + file(READ "${config_log}" log_contents) + debug_message("Reading configure log ${config_log}...") + if(NOT log_contents MATCHES "Manually-specified variables were not used by the project:\n\n(( [^\n]*\n)*)") + continue() + endif() + string(STRIP "${CMAKE_MATCH_1}" unused_variables) # remove leading ` ` and trailing `\n` + string(REPLACE "\n " ";" unused_variables "${unused_variables}") + debug_message("unused variables: ${unused_variables}") + foreach(unused_variable IN LISTS unused_variables) + if(unused_variable IN_LIST manually_specified_variables) + debug_message("manually specified unused variable: ${unused_variable}") + vcpkg_list(APPEND all_unused_variables "${unused_variable}") + else() + debug_message("unused variable (not manually specified): ${unused_variable}") + endif() + endforeach() + endforeach() + + if(DEFINED all_unused_variables) + vcpkg_list(REMOVE_DUPLICATES all_unused_variables) + vcpkg_list(JOIN all_unused_variables "\n " all_unused_variables) + message(WARNING "The following variables are not used in CMakeLists.txt: + ${all_unused_variables} +Please recheck them and remove the unnecessary options from the `vcpkg_cmake_configure` call. +If these options should still be passed for whatever reason, please use the `MAYBE_UNUSED_VARIABLES` argument.") + endif() + + if(NOT arg_Z_CMAKE_GET_VARS_USAGE) + set(Z_VCPKG_CMAKE_GENERATOR "${generator}" CACHE INTERNAL "The generator which was used to configure CMake.") + endif() +endfunction() diff --git a/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_install.cmake b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_install.cmake new file mode 100644 index 00000000..2bd8b4ea --- /dev/null +++ b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_install.cmake @@ -0,0 +1,21 @@ +include_guard(GLOBAL) + +function(vcpkg_cmake_install) + cmake_parse_arguments(PARSE_ARGV 0 "arg" "DISABLE_PARALLEL;ADD_BIN_TO_PATH" "" "") + if(DEFINED arg_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "vcpkg_cmake_install was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") + endif() + + set(args) + foreach(arg IN ITEMS DISABLE_PARALLEL ADD_BIN_TO_PATH) + if(arg_${arg}) + list(APPEND args "${arg}") + endif() + endforeach() + + vcpkg_cmake_build( + ${args} + LOGFILE_BASE install + TARGET install + ) +endfunction() From aaed76e49167a9cde0bcb0faba57271cfbaf9df0 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 3 Apr 2025 10:18:03 +0200 Subject: [PATCH 43/66] Add CMAKE_POLICY_VERSION_MINIMUM=3.5 This reverts commit caa8ec72d94c33979902bd56631c8bb15bd74f7c. --- vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake index acd510f5..39ea39c2 100644 --- a/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake +++ b/vcpkg_ports/vcpkg-cmake/vcpkg_cmake_configure.cmake @@ -44,6 +44,8 @@ function(vcpkg_cmake_configure) set(manually_specified_variables "") + vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_POLICY_VERSION_MINIMUM=3.5") + if(arg_Z_CMAKE_GET_VARS_USAGE) set(configuring_message "Getting CMake variables for ${TARGET_TRIPLET}") else() From 0e1974ae7a340892e88bbbf4c76abc67b9297aa6 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 3 Apr 2025 10:38:58 +0200 Subject: [PATCH 44/66] Revert previous CMAKE_POLICY_VERSION_MINIMUM env variable --- .github/workflows/CloudTesting.yml | 5 +---- .github/workflows/HighPriorityIssues.yml | 1 - .github/workflows/LocalTesting.yml | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CloudTesting.yml b/.github/workflows/CloudTesting.yml index 81a24820..833595fe 100644 --- a/.github/workflows/CloudTesting.yml +++ b/.github/workflows/CloudTesting.yml @@ -7,9 +7,6 @@ defaults: run: shell: bash -env: - CMAKE_POLICY_VERSION_MINIMUM: 3.5 - jobs: rest: name: Test against remote AWS account @@ -60,4 +57,4 @@ jobs: AWS_DEFAULT_REGION: ${{secrets.S3_ICEBERG_TEST_USER_REGION}} ICEBERG_AWS_REMOTE_AVAILABLE: 1 run: | - make test_release \ No newline at end of file + make test_release diff --git a/.github/workflows/HighPriorityIssues.yml b/.github/workflows/HighPriorityIssues.yml index 919b6ac3..6adc6d87 100644 --- a/.github/workflows/HighPriorityIssues.yml +++ b/.github/workflows/HighPriorityIssues.yml @@ -10,7 +10,6 @@ env: # hence only one of the numbers will be filled in the TITLE_PREFIX TITLE_PREFIX: "[duckdb_iceberg/#${{ github.event.issue.number }}]" PUBLIC_ISSUE_TITLE: ${{ github.event.issue.title }} - CMAKE_POLICY_VERSION_MINIMUM: 3.5 jobs: create_or_label_issue: diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index c2f79fb7..03fcd627 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -9,7 +9,6 @@ defaults: env: BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} - CMAKE_POLICY_VERSION_MINIMUM: 3.5 jobs: rest: @@ -95,4 +94,4 @@ jobs: ICEBERG_SERVER_AVAILABLE: 1 DUCKDB_ICEBERG_HAVE_GENERATED_DATA: 1 run: | - make test_release \ No newline at end of file + make test_release From ddccfbf240ec44480ee757904ba7150a45513180 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 3 Apr 2025 11:05:49 +0200 Subject: [PATCH 45/66] Use InvalidConfigurationExceptions instead --- src/common/iceberg.cpp | 22 ++++++++++--------- src/common/schema.cpp | 12 +++++----- src/common/utils.cpp | 10 ++++----- src/iceberg_extension.cpp | 18 +++++++-------- .../iceberg_multi_file_reader.cpp | 2 +- src/include/iceberg_types.hpp | 6 ++--- src/storage/irc_catalog.cpp | 4 ++-- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/src/common/iceberg.cpp b/src/common/iceberg.cpp index 6a62e8db..fd69bf9b 100644 --- a/src/common/iceberg.cpp +++ b/src/common/iceberg.cpp @@ -64,7 +64,7 @@ unique_ptr IcebergSnapshot::GetParseInfo(yyjson_doc &metadata } else { auto schema = yyjson_obj_get(root, "schema"); if (!schema) { - throw InvalidInputException("Neither a valid schema or schemas field was found"); + throw InvalidConfigurationException("Neither a valid schema or schemas field was found"); } auto found_schema_id = IcebergUtils::TryGetNumFromObject(schema, "schema-id"); info.schemas.push_back(schema); @@ -103,7 +103,7 @@ IcebergSnapshot IcebergSnapshot::GetSnapshotById(const string &path, FileSystem auto snapshot = FindSnapshotByIdInternal(info->snapshots, snapshot_id); if (!snapshot) { - throw InvalidInputException("Could not find snapshot with id " + to_string(snapshot_id)); + throw InvalidConfigurationException("Could not find snapshot with id " + to_string(snapshot_id)); } return ParseSnapShot(snapshot, info->iceberg_version, info->schema_id, info->schemas, options); @@ -115,7 +115,8 @@ IcebergSnapshot IcebergSnapshot::GetSnapshotByTimestamp(const string &path, File auto snapshot = FindSnapshotByIdTimestampInternal(info->snapshots, timestamp); if (!snapshot) { - throw InvalidInputException("Could not find latest snapshots for timestamp " + Timestamp::ToString(timestamp)); + throw InvalidConfigurationException("Could not find latest snapshots for timestamp " + + Timestamp::ToString(timestamp)); } return ParseSnapShot(snapshot, info->iceberg_version, info->schema_id, info->schemas, options); @@ -138,7 +139,7 @@ static string GenerateMetaDataUrl(FileSystem &fs, const string &meta_path, strin } } - throw InvalidInputException( + throw InvalidConfigurationException( "Iceberg metadata file not found for table version '%s' using '%s' compression and format(s): '%s'", table_version, options.metadata_compression_codec, options.version_name_format); } @@ -171,7 +172,7 @@ string IcebergSnapshot::GetMetaDataPath(ClientContext &context, const string &pa } if (!UnsafeVersionGuessingEnabled(context)) { // Make sure we're allowed to guess versions - throw InvalidInputException( + throw InvalidConfigurationException( "Failed to read iceberg table. No version was provided and no version-hint could be found, globbing the " "filesystem to locate the latest version is disabled by default as this is considered unsafe and could " "result in reading uncommitted data. To enable this use 'SET %s = true;'", @@ -195,7 +196,7 @@ IcebergSnapshot IcebergSnapshot::ParseSnapShot(yyjson_val *snapshot, idx_t icebe if (snapshot) { auto snapshot_tag = yyjson_get_type(snapshot); if (snapshot_tag != YYJSON_TYPE_OBJ) { - throw InvalidInputException("Invalid snapshot field found parsing iceberg metadata.json"); + throw InvalidConfigurationException("Invalid snapshot field found parsing iceberg metadata.json"); } ret.metadata_compression_codec = options.metadata_compression_codec; if (iceberg_format_version == 1) { @@ -226,9 +227,9 @@ string IcebergSnapshot::GetTableVersionFromHint(const string &meta_path, FileSys try { return version_file_content; } catch (std::invalid_argument &e) { - throw InvalidInputException("Iceberg version hint file contains invalid value"); + throw InvalidConfigurationException("Iceberg version hint file contains invalid value"); } catch (std::out_of_range &e) { - throw InvalidInputException("Iceberg version hint file contains invalid value"); + throw InvalidConfigurationException("Iceberg version hint file contains invalid value"); } } @@ -262,8 +263,9 @@ string IcebergSnapshot::GuessTableVersion(const string &meta_path, FileSystem &f } } - throw InvalidInputException("Could not guess Iceberg table version using '%s' compression and format(s): '%s'", - metadata_compression_codec, version_format); + throw InvalidConfigurationException( + "Could not guess Iceberg table version using '%s' compression and format(s): '%s'", metadata_compression_codec, + version_format); } string IcebergSnapshot::PickTableVersion(vector &found_metadata, string &version_pattern, string &glob) { diff --git a/src/common/schema.cpp b/src/common/schema.cpp index cb789099..1c2d715f 100644 --- a/src/common/schema.cpp +++ b/src/common/schema.cpp @@ -63,13 +63,13 @@ static LogicalType ParseComplexType(yyjson_val *type) { if (type_str == "map") { return ParseMap(type); } - throw InvalidInputException("Invalid field found while parsing field: type"); + throw InvalidConfigurationException("Invalid field found while parsing field: type"); } static LogicalType ParseType(yyjson_val *type) { auto val = yyjson_obj_get(type, "type"); if (!val) { - throw InvalidInputException("Invalid field found while parsing field: type"); + throw InvalidConfigurationException("Invalid field found while parsing field: type"); } return ParseTypeValue(val); } @@ -79,7 +79,7 @@ static LogicalType ParseTypeValue(yyjson_val *val) { return ParseComplexType(val); } if (yyjson_get_type(val) != YYJSON_TYPE_STR) { - throw InvalidInputException("Invalid field found while parsing field: type"); + throw InvalidConfigurationException("Invalid field found while parsing field: type"); } string type_str = yyjson_get_str(val); @@ -136,7 +136,7 @@ static LogicalType ParseTypeValue(yyjson_val *val) { auto scale = std::stoi(digits[1]); return LogicalType::DECIMAL(width, scale); } - throw InvalidInputException("Encountered an unrecognized type in JSON schema: \"%s\"", type_str); + throw InvalidConfigurationException("Encountered an unrecognized type in JSON schema: \"%s\"", type_str); } IcebergColumnDefinition IcebergColumnDefinition::ParseFromJson(yyjson_val *val) { @@ -155,7 +155,7 @@ static vector ParseSchemaFromJson(yyjson_val *schema_js // Assert that the top level 'type' is a struct auto type_str = IcebergUtils::TryGetStrFromObject(schema_json, "type"); if (type_str != "struct") { - throw InvalidInputException("Schema in JSON Metadata is invalid"); + throw InvalidConfigurationException("Schema in JSON Metadata is invalid"); } D_ASSERT(yyjson_get_type(schema_json) == YYJSON_TYPE_OBJ); D_ASSERT(IcebergUtils::TryGetStrFromObject(schema_json, "type") == "struct"); @@ -180,7 +180,7 @@ vector IcebergSnapshot::ParseSchema(vector:s3tablescatalog/"); + throw InvalidConfigurationException("Invalid Glue Catalog Format: '" + warehouse + + "'. Expected ':s3tablescatalog/"); } static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_info, ClientContext &context, @@ -121,8 +121,8 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in auto region = kv_secret.TryGetValue("region"); if (region.IsNull()) { - throw InvalidInputException("Assumed catalog secret " + secret_entry->secret->GetName() + " for catalog " + - name + " does not have a region"); + throw InvalidConfigurationException("Assumed catalog secret " + secret_entry->secret->GetName() + + " for catalog " + name + " does not have a region"); } switch (catalog_type) { case ICEBERG_CATALOG_TYPE::AWS_S3TABLES: { @@ -150,11 +150,11 @@ static unique_ptr IcebergCatalogAttach(StorageExtensionInfo *storage_in // Check no endpoint type has been passed. if (!endpoint_type.empty()) { - throw InvalidInputException("Unrecognized endpoint point: %s. Expected either S3_TABLES or GLUE", - endpoint_type); + throw InvalidConfigurationException("Unrecognized endpoint point: %s. Expected either S3_TABLES or GLUE", + endpoint_type); } if (endpoint_type.empty() && endpoint.empty()) { - throw InvalidInputException("No endpoint type or endpoint provided"); + throw InvalidConfigurationException("No endpoint type or endpoint provided"); } catalog_type = ICEBERG_CATALOG_TYPE::OTHER; diff --git a/src/iceberg_functions/iceberg_multi_file_reader.cpp b/src/iceberg_functions/iceberg_multi_file_reader.cpp index 84b1e7b3..605c794b 100644 --- a/src/iceberg_functions/iceberg_multi_file_reader.cpp +++ b/src/iceberg_functions/iceberg_multi_file_reader.cpp @@ -313,7 +313,7 @@ void IcebergMultiFileReader::CreateColumnMapping(const string &file_name, // Lookup the required column in the local map auto entry = name_map.find("file_row_number"); if (entry == name_map.end()) { - throw InvalidInputException("Failed to find the file_row_number column"); + throw InvalidConfigurationException("Failed to find the file_row_number column"); } // Register the column to be scanned from this file diff --git a/src/include/iceberg_types.hpp b/src/include/iceberg_types.hpp index 777d9312..1be81408 100644 --- a/src/include/iceberg_types.hpp +++ b/src/include/iceberg_types.hpp @@ -27,7 +27,7 @@ static string IcebergManifestContentTypeToString(IcebergManifestContentType type case IcebergManifestContentType::DELETE: return "DELETE"; default: - throw InvalidInputException("Invalid Manifest Content Type"); + throw InvalidConfigurationException("Invalid Manifest Content Type"); } } @@ -42,7 +42,7 @@ static string IcebergManifestEntryStatusTypeToString(IcebergManifestEntryStatusT case IcebergManifestEntryStatusType::DELETED: return "DELETED"; default: - throw InvalidInputException("Invalid matifest entry type"); + throw InvalidConfigurationException("Invalid matifest entry type"); } } @@ -57,7 +57,7 @@ static string IcebergManifestEntryContentTypeToString(IcebergManifestEntryConten case IcebergManifestEntryContentType::EQUALITY_DELETES: return "EQUALITY_DELETES"; default: - throw InvalidInputException("Invalid Manifest Entry Content Type"); + throw InvalidConfigurationException("Invalid Manifest Entry Content Type"); } } diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index 9a360570..c5003125 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -168,14 +168,14 @@ unique_ptr IRCatalog::GetSecret(ClientContext &context, const strin if (!secret_entry) { auto secret_match = context.db->GetSecretManager().LookupSecret(transaction, "s3://", "s3"); if (!secret_match.HasMatch()) { - throw InvalidConfigurationException("Failed to find a secret and no explicit secret was passed!"); + throw InvalidInputException("Failed to find a secret and no explicit secret was passed!"); } secret_entry = std::move(secret_match.secret_entry); } if (secret_entry) { return secret_entry; } - throw InvalidConfigurationException("Could not find valid Iceberg secret"); + throw InvalidInputException("Could not find valid Iceberg secret"); } unique_ptr IRCatalog::PlanInsert(ClientContext &context, LogicalInsert &op, From 4f93ec25ae5ce370b521e6e415519b4389bda8b4 Mon Sep 17 00:00:00 2001 From: Carlo Piovesan Date: Thu, 3 Apr 2025 10:39:30 +0200 Subject: [PATCH 46/66] Add also vcpkg-cmake --- vcpkg.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vcpkg.json b/vcpkg.json index 10371314..d5d7e32d 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,5 +1,6 @@ { "dependencies": [ + "vcpkg-cmake", "avro-c", "curl", "openssl", @@ -24,4 +25,4 @@ "version": "3.0.8" } ] -} \ No newline at end of file +} From 7baa22a2672d786b2318f070c50306ed50349b96 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 12:04:12 +0200 Subject: [PATCH 47/66] thijs comments --- .github/workflows/LocalTesting.yml | 18 ++++++++++++++++-- Makefile | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index cf875957..d9ba741c 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -115,11 +115,25 @@ jobs: exec $SHELL -l jenv add /usr/lib/jvm/java-21-openjdk-amd64 - - name: Setup Polaris + - name: Wait for polaris initializatino run: | make setup_polaris # let polaris initialize - sleep 30 + max_attempts=50 + attempt=1 + while ! (curl -sf http://localhost:8182/healthcheck || curl -sf http://localhost:8182/q/health); do + if [ $attempt -gt $max_attempts ]; then + echo "Polaris failed to initialize after $max_attempts attempts" + exit 1 + fi + echo "Waiting for Polaris to initialize (attempt $attempt/$max_attempts)..." + sleep 5 + attempt=$((attempt + 1)) + done + echo "Polaris is healthy" + + - name: Generate Polaris Data + run: | python3 -m venv . source ./bin/activate python3 -m pip install poetry diff --git a/Makefile b/Makefile index 00254492..fae5231a 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ setup_polaris: mkdir polaris_catalog git clone https://github.com/apache/polaris.git polaris_catalog cd polaris_catalog && jenv local 21 + cd polaris_catalog && ./gradlew clean :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true --no-build-cache cd polaris_catalog && ./gradlew --stop cd polaris_catalog && nohup ./gradlew run > polaris-server.log 2> polaris-error.log & From a1f7aa2c360719eb2fe91f663481de7c024c0bb2 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 12:04:39 +0200 Subject: [PATCH 48/66] typo --- .github/workflows/LocalTesting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index d9ba741c..42109504 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -115,7 +115,7 @@ jobs: exec $SHELL -l jenv add /usr/lib/jvm/java-21-openjdk-amd64 - - name: Wait for polaris initializatino + - name: Wait for polaris initialization run: | make setup_polaris # let polaris initialize From 0287c74c4fd2a6562f9ab036135db9c3be302258 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 12:11:17 +0200 Subject: [PATCH 49/66] dont forget to source bashrc --- .github/workflows/LocalTesting.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 42109504..503e584b 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -108,15 +108,15 @@ jobs: - name: Setup Jenv run: | git clone https://github.com/jenv/jenv.git ~/.jenv - echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bash_profile + echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bashrc + echo 'eval "$(jenv init -)"' >> ~/.bashrc source ~/.bash_profile - eval "$(jenv init -)" jenv enable-plugin export - exec $SHELL -l jenv add /usr/lib/jvm/java-21-openjdk-amd64 - name: Wait for polaris initialization run: | + source ~/.bashrc make setup_polaris # let polaris initialize max_attempts=50 From 598203781b04f2ea8a6a5a2fffead26b6ff9bc9e Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 12:28:55 +0200 Subject: [PATCH 50/66] better naming for workflow jobs --- .github/workflows/LocalTesting.yml | 1 - .github/workflows/PolarisTesting.yml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 503e584b..aa59b37b 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -9,7 +9,6 @@ defaults: env: BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} - CMAKE_POLICY_VERSION_MINIMUM: 3.5 jobs: rest: diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index f4746801..4b56d6ca 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -1,4 +1,4 @@ -name: Local functional tests +name: Polaris tests on: [push, pull_request,repository_dispatch] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} @@ -12,7 +12,7 @@ env: jobs: rest: - name: Test against local Rest Catalog + name: Test against Polaris Catalog runs-on: ubuntu-latest env: VCPKG_TARGET_TRIPLET: 'x64-linux' From 2b830c750f5e6c6d27310c61baf7a7f3bd486a3d Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 12:29:50 +0200 Subject: [PATCH 51/66] better workflow for polaris --- .github/workflows/PolarisTesting.yml | 35 ++++++++++++++++++---------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index 4b56d6ca..491dda85 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -80,31 +80,43 @@ jobs: run: | make release - - name: Install java 21 + - name: Set up for Polaris run: | + # install java sudo apt install -y -qq openjdk-21-jre-headless sudo apt install -y -qq openjdk-21-jdk-headless - - - name: Install python venv - run: | + # install python virtual environment (is this needed?) sudo apt-get install -y -qq python3-venv - name: Setup Jenv run: | git clone https://github.com/jenv/jenv.git ~/.jenv - echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bash_profile + echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bashrc + echo 'eval "$(jenv init -)"' >> ~/.bashrc source ~/.bash_profile - eval "$(jenv init -)" jenv enable-plugin export - exec $SHELL -l jenv add /usr/lib/jvm/java-21-openjdk-amd64 - - - name: Setup Polaris + - name: Wait for polaris initialization run: | + source ~/.bashrc make setup_polaris # let polaris initialize - sleep 30 + max_attempts=50 + attempt=1 + while ! (curl -sf http://localhost:8182/healthcheck || curl -sf http://localhost:8182/q/health); do + if [ $attempt -gt $max_attempts ]; then + echo "Polaris failed to initialize after $max_attempts attempts" + exit 1 + fi + echo "Waiting for Polaris to initialize (attempt $attempt/$max_attempts)..." + sleep 5 + attempt=$((attempt + 1)) + done + echo "Polaris is healthy" + + - name: Generate Polaris Data + run: | python3 -m venv . source ./bin/activate python3 -m pip install poetry @@ -126,5 +138,4 @@ jobs: run: | export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) - make test_polaris - + make test_release \ No newline at end of file From fd65be1db69c057f703e1301809828096777b6f2 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 13:21:59 +0200 Subject: [PATCH 52/66] source bashrc not bash_profile --- .github/workflows/LocalTesting.yml | 3 ++- .github/workflows/PolarisTesting.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index aa59b37b..e56cd50b 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -9,6 +9,7 @@ defaults: env: BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} + CMAKE_POLICY_VERSION_MINIMUM: 3.5 jobs: rest: @@ -109,7 +110,7 @@ jobs: git clone https://github.com/jenv/jenv.git ~/.jenv echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bashrc echo 'eval "$(jenv init -)"' >> ~/.bashrc - source ~/.bash_profile + source ~/.bashrc jenv enable-plugin export jenv add /usr/lib/jvm/java-21-openjdk-amd64 diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index 491dda85..cb9fc8ac 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -9,6 +9,7 @@ defaults: env: BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }} + CMAKE_POLICY_VERSION_MINIMUM: 3.5 jobs: rest: @@ -93,7 +94,7 @@ jobs: git clone https://github.com/jenv/jenv.git ~/.jenv echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bashrc echo 'eval "$(jenv init -)"' >> ~/.bashrc - source ~/.bash_profile + source ~/.bashrc jenv enable-plugin export jenv add /usr/lib/jvm/java-21-openjdk-amd64 From 6a2b9234dcb01d823445834d616f668eea4dff57 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 13:37:09 +0200 Subject: [PATCH 53/66] add tmate check --- .github/workflows/LocalTesting.yml | 5 ++++- .github/workflows/PolarisTesting.yml | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 9e44233a..82627b5a 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -103,7 +103,10 @@ jobs: sudo apt install -y -qq openjdk-21-jdk-headless # install python virtual environment (is this needed?) sudo apt-get install -y -qq python3-venv - + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Setup Jenv run: | git clone https://github.com/jenv/jenv.git ~/.jenv diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index cb9fc8ac..05d99af9 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -89,6 +89,9 @@ jobs: # install python virtual environment (is this needed?) sudo apt-get install -y -qq python3-venv + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Setup Jenv run: | git clone https://github.com/jenv/jenv.git ~/.jenv From 2ec564ea4649d71820094571ef513930da6740b1 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 14:55:44 +0200 Subject: [PATCH 54/66] add polaris to CI --- .github/workflows/LocalTesting.yml | 19 ++++--------------- .github/workflows/PolarisTesting.yml | 19 ++++--------------- Makefile | 3 +-- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 82627b5a..1fac0114 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -104,22 +104,11 @@ jobs: # install python virtual environment (is this needed?) sudo apt-get install -y -qq python3-venv - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - - name: Setup Jenv - run: | - git clone https://github.com/jenv/jenv.git ~/.jenv - echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bashrc - echo 'eval "$(jenv init -)"' >> ~/.bashrc - source ~/.bashrc - jenv enable-plugin export - jenv add /usr/lib/jvm/java-21-openjdk-amd64 - - name: Wait for polaris initialization + env: + JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 run: | - source ~/.bashrc - make setup_polaris + make setup_polaris_ci # let polaris initialize max_attempts=50 attempt=1 @@ -157,4 +146,4 @@ jobs: run: | export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) - make test_release + make test_release \ No newline at end of file diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index 05d99af9..25e57e92 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -1,5 +1,5 @@ name: Polaris tests -on: [push, pull_request,repository_dispatch] +on: [push, repository_dispatch] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} cancel-in-progress: true @@ -89,22 +89,11 @@ jobs: # install python virtual environment (is this needed?) sudo apt-get install -y -qq python3-venv - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - - name: Setup Jenv - run: | - git clone https://github.com/jenv/jenv.git ~/.jenv - echo 'export PATH="$HOME/.jenv/bin:$PATH"' >> ~/.bashrc - echo 'eval "$(jenv init -)"' >> ~/.bashrc - source ~/.bashrc - jenv enable-plugin export - jenv add /usr/lib/jvm/java-21-openjdk-amd64 - - name: Wait for polaris initialization + env: + JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 run: | - source ~/.bashrc - make setup_polaris + make setup_polaris_ci # let polaris initialize max_attempts=50 attempt=1 diff --git a/Makefile b/Makefile index fae5231a..ebdf70c4 100644 --- a/Makefile +++ b/Makefile @@ -24,10 +24,9 @@ data_large: data data_clean python3 scripts/data_generators/generate_data.py # setup polaris server. See PolarisTesting.yml to see instructions for a specific machine. -setup_polaris: +setup_polaris_ci: mkdir polaris_catalog git clone https://github.com/apache/polaris.git polaris_catalog - cd polaris_catalog && jenv local 21 cd polaris_catalog && ./gradlew clean :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true --no-build-cache cd polaris_catalog && ./gradlew --stop cd polaris_catalog && nohup ./gradlew run > polaris-server.log 2> polaris-error.log & From 7511616e6f4d96b84bc5bf8ced1abe0c3bd1a8a6 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 14:59:12 +0200 Subject: [PATCH 55/66] run polaris testing in it's own CI --- .github/workflows/LocalTesting.yml | 52 ---------------------------- .github/workflows/PolarisTesting.yml | 4 +-- 2 files changed, 2 insertions(+), 54 deletions(-) diff --git a/.github/workflows/LocalTesting.yml b/.github/workflows/LocalTesting.yml index 1fac0114..4bb021b3 100644 --- a/.github/workflows/LocalTesting.yml +++ b/.github/workflows/LocalTesting.yml @@ -94,56 +94,4 @@ jobs: ICEBERG_SERVER_AVAILABLE: 1 DUCKDB_ICEBERG_HAVE_GENERATED_DATA: 1 run: | - make test_release - - - name: Set up for Polaris - run: | - # install java - sudo apt install -y -qq openjdk-21-jre-headless - sudo apt install -y -qq openjdk-21-jdk-headless - # install python virtual environment (is this needed?) - sudo apt-get install -y -qq python3-venv - - - name: Wait for polaris initialization - env: - JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 - run: | - make setup_polaris_ci - # let polaris initialize - max_attempts=50 - attempt=1 - while ! (curl -sf http://localhost:8182/healthcheck || curl -sf http://localhost:8182/q/health); do - if [ $attempt -gt $max_attempts ]; then - echo "Polaris failed to initialize after $max_attempts attempts" - exit 1 - fi - echo "Waiting for Polaris to initialize (attempt $attempt/$max_attempts)..." - sleep 5 - attempt=$((attempt + 1)) - done - echo "Polaris is healthy" - - - name: Generate Polaris Data - run: | - python3 -m venv . - source ./bin/activate - python3 -m pip install poetry - python3 -m pip install pyspark==3.5.0 - python3 scripts/polaris/get_polaris_root_creds.py - # needed for setup_polaris_catalog.sh - export POLARIS_ROOT_ID=$(cat polaris_root_id.txt) - export POLARIS_ROOT_SECRET=$(cat polaris_root_password.txt) - cd polaris_catalog && ../scripts/polaris/setup_polaris_catalog.sh > user_credentials.json - cd .. - python3 scripts/polaris/get_polaris_client_creds.py - export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) - export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) - python3 scripts/polaris/create_data.py - - - name: Test with rest catalog - env: - POLARIS_SERVER_AVAILABLE: 1 - run: | - export POLARIS_CLIENT_ID=$(cat polaris_client_id.txt) - export POLARIS_CLIENT_SECRET=$(cat polaris_client_secret.txt) make test_release \ No newline at end of file diff --git a/.github/workflows/PolarisTesting.yml b/.github/workflows/PolarisTesting.yml index 25e57e92..99d13243 100644 --- a/.github/workflows/PolarisTesting.yml +++ b/.github/workflows/PolarisTesting.yml @@ -1,5 +1,5 @@ -name: Polaris tests -on: [push, repository_dispatch] +name: Local Polaris Testing +on: [push, pull_request,repository_dispatch] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} cancel-in-progress: true From 836a3bdb033031a14dd5063a8b0db3ff2746f8bf Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 15:02:27 +0200 Subject: [PATCH 56/66] run make format-fix --- scripts/polaris/create_data.py | 55 +++++++++++---------- scripts/polaris/setup_polaris_catalog.py | 63 +++++++++++++++--------- 2 files changed, 69 insertions(+), 49 deletions(-) diff --git a/scripts/polaris/create_data.py b/scripts/polaris/create_data.py index d76faeea..6c40e438 100644 --- a/scripts/polaris/create_data.py +++ b/scripts/polaris/create_data.py @@ -8,28 +8,32 @@ print("no client_id or client_secret") exit(1) -spark = (SparkSession.builder - .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") - .config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19") - .config('spark.sql.iceberg.vectorization.enabled', 'false') - # Configure the 'polaris' catalog as an Iceberg rest catalog - .config("spark.sql.catalog.quickstart_catalog.type", "rest") - .config("spark.sql.catalog.quickstart_catalog", "org.apache.iceberg.spark.SparkCatalog") - # Specify the rest catalog endpoint - .config("spark.sql.catalog.quickstart_catalog.uri", "http://localhost:8181/api/catalog") - # Enable token refresh - .config("spark.sql.catalog.quickstart_catalog.token-refresh-enabled", "true") - # specify the client_id:client_secret pair - .config("spark.sql.catalog.quickstart_catalog.credential", f"{client_id}:{client_secret}") - # Set the warehouse to the name of the catalog we created - .config("spark.sql.catalog.quickstart_catalog.warehouse", "quickstart_catalog") - # Scope set to PRINCIPAL_ROLE:ALL - .config("spark.sql.catalog.quickstart_catalog.scope", 'PRINCIPAL_ROLE:ALL') - # Enable access credential delegation - .config("spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation", 'vended-credentials') - .config("spark.sql.catalog.quickstart_catalog.io-impl", "org.apache.iceberg.io.ResolvingFileIO") - .config("spark.sql.catalog.quickstart_catalog.s3.region", "us-west-2") - .config("spark.history.fs.logDirectory", "/home/iceberg/spark-events")).getOrCreate() +spark = ( + SparkSession.builder.config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config( + "spark.jars.packages", + "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19", + ) + .config('spark.sql.iceberg.vectorization.enabled', 'false') + # Configure the 'polaris' catalog as an Iceberg rest catalog + .config("spark.sql.catalog.quickstart_catalog.type", "rest") + .config("spark.sql.catalog.quickstart_catalog", "org.apache.iceberg.spark.SparkCatalog") + # Specify the rest catalog endpoint + .config("spark.sql.catalog.quickstart_catalog.uri", "http://localhost:8181/api/catalog") + # Enable token refresh + .config("spark.sql.catalog.quickstart_catalog.token-refresh-enabled", "true") + # specify the client_id:client_secret pair + .config("spark.sql.catalog.quickstart_catalog.credential", f"{client_id}:{client_secret}") + # Set the warehouse to the name of the catalog we created + .config("spark.sql.catalog.quickstart_catalog.warehouse", "quickstart_catalog") + # Scope set to PRINCIPAL_ROLE:ALL + .config("spark.sql.catalog.quickstart_catalog.scope", 'PRINCIPAL_ROLE:ALL') + # Enable access credential delegation + .config("spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation", 'vended-credentials') + .config("spark.sql.catalog.quickstart_catalog.io-impl", "org.apache.iceberg.io.ResolvingFileIO") + .config("spark.sql.catalog.quickstart_catalog.s3.region", "us-west-2") + .config("spark.history.fs.logDirectory", "/home/iceberg/spark-events") +).getOrCreate() spark.sql("USE quickstart_catalog") @@ -37,12 +41,13 @@ spark.sql("CREATE NAMESPACE IF NOT EXISTS COLLADO_TEST") spark.sql("USE NAMESPACE COLLADO_TEST") -spark.sql(""" +spark.sql( + """ CREATE TABLE IF NOT EXISTS quickstart_table ( id BIGINT, data STRING ) USING ICEBERG -""") +""" +) spark.sql("INSERT INTO quickstart_table VALUES (1, 'some data'), (2, 'more data'), (3, 'yet more data')") spark.sql("SELECT * FROM quickstart_table").show() - diff --git a/scripts/polaris/setup_polaris_catalog.py b/scripts/polaris/setup_polaris_catalog.py index 8e35061d..65bcf525 100644 --- a/scripts/polaris/setup_polaris_catalog.py +++ b/scripts/polaris/setup_polaris_catalog.py @@ -5,35 +5,40 @@ from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration # some of this is from https://github.com/apache/polaris/blob/e32ef89bb97642f2ac9a4db82252a4fcf7aa0039/getting-started/spark/notebooks/SparkPolaris.ipynb -polaris_credential = 'root:s3cr3t' # pragma: allowlist secret +polaris_credential = 'root:s3cr3t' # pragma: allowlist secret client_id = os.getenv('POLARIS_ROOT_ID', '') client_secret = os.getenv('POLARIS_ROOT_SECRET', '') if client_id == '' or client_secret == '': Print("could not find polaris root id or polaris root secret") -client = CatalogApiClient(CatalogApiClientConfiguration(username=client_id, - password=client_secret, - host='http://polaris:8181/api/catalog')) +client = CatalogApiClient( + CatalogApiClientConfiguration(username=client_id, password=client_secret, host='http://polaris:8181/api/catalog') +) oauth_api = IcebergOAuth2API(client) -token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', - client_id=client_id, - client_secret=client_secret, - grant_type='client_credentials', - _headers={'realm': 'default-realm'}) +token = oauth_api.get_token( + scope='PRINCIPAL_ROLE:ALL', + client_id=client_id, + client_secret=client_secret, + grant_type='client_credentials', + _headers={'realm': 'default-realm'}, +) # create a catalog from polaris.management import * -client = ApiClient(Configuration(access_token=token.access_token, - host='http://polaris:8181/api/management/v1')) +client = ApiClient(Configuration(access_token=token.access_token, host='http://polaris:8181/api/management/v1')) root_client = PolarisDefaultApi(client) storage_conf = FileStorageConfigInfo(storage_type="FILE", allowed_locations=["file:///tmp"]) catalog_name = 'polaris_demo' -catalog = Catalog(name=catalog_name, type='INTERNAL', properties={"default-base-location": "file:///tmp/polaris/"}, - storage_config_info=storage_conf) +catalog = Catalog( + name=catalog_name, + type='INTERNAL', + properties={"default-base-location": "file:///tmp/polaris/"}, + storage_config_info=storage_conf, +) catalog.storage_config_info = storage_conf root_client.create_catalog(create_catalog_request=CreateCatalogRequest(catalog=catalog)) resp = root_client.get_catalog(catalog_name=catalog.name) @@ -52,17 +57,21 @@ def create_principal(api, principal_name): else: raise e + # Create a catalog role with the given name def create_catalog_role(api, catalog, role_name): catalog_role = CatalogRole(name=role_name) try: - api.create_catalog_role(catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role)) + api.create_catalog_role( + catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role) + ) return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) except ApiException as e: return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) else: raise e + # Create a principal role with the given name def create_principal_role(api, role_name): principal_role = PrincipalRole(name=role_name) @@ -86,18 +95,24 @@ def create_principal_role(api, role_name): # Grant the catalog role to the principal role # All principals in the principal role have the catalog role's privileges -root_client.assign_catalog_role_to_principal_role(principal_role_name=engineer_role.name, - catalog_name=catalog.name, - grant_catalog_role_request=GrantCatalogRoleRequest(catalog_role=manager_catalog_role)) +root_client.assign_catalog_role_to_principal_role( + principal_role_name=engineer_role.name, + catalog_name=catalog.name, + grant_catalog_role_request=GrantCatalogRoleRequest(catalog_role=manager_catalog_role), +) # Assign privileges to the catalog role # Here, we grant CATALOG_MANAGE_CONTENT -root_client.add_grant_to_catalog_role(catalog.name, manager_catalog_role.name, - AddGrantRequest(grant=CatalogGrant(catalog_name=catalog.name, - type='catalog', - privilege=CatalogPrivilege.CATALOG_MANAGE_CONTENT))) +root_client.add_grant_to_catalog_role( + catalog.name, + manager_catalog_role.name, + AddGrantRequest( + grant=CatalogGrant(catalog_name=catalog.name, type='catalog', privilege=CatalogPrivilege.CATALOG_MANAGE_CONTENT) + ), +) # Assign the principal role to the principal -root_client.assign_principal_role(engineer_principal.principal.name, grant_principal_role_request=GrantPrincipalRoleRequest(principal_role=engineer_role)) - - +root_client.assign_principal_role( + engineer_principal.principal.name, + grant_principal_role_request=GrantPrincipalRoleRequest(principal_role=engineer_role), +) From 02b32a7f8fa046c68b85fa12d099b4f64a50c15e Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 3 Apr 2025 16:43:48 +0200 Subject: [PATCH 57/66] we can test polaris against tpch --- scripts/data_generators/generate_data.py | 30 +++- .../generate_polaris_rest/__init__.py | 1 + .../generate_polaris_rest/empty_table/q00.sql | 9 ++ .../generate_iceberg_polaris_rest.py | 129 ++++++++++++++++++ .../lineitem_001_deletes/q00.sql | 6 + .../lineitem_001_deletes/q01.sql | 11 ++ .../lineitem_001_deletes/setup.py | 8 ++ .../lineitem_partitioned_l_shipmode/q00.sql | 8 ++ .../lineitem_partitioned_l_shipmode/q01.sql | 1 + .../lineitem_partitioned_l_shipmode/setup.py | 8 ++ .../q00.sql | 8 ++ .../q01.sql | 6 + .../setup.py | 8 ++ .../lineitem_sf1_deletes/q00.sql | 6 + .../lineitem_sf1_deletes/q01.sql | 11 ++ .../lineitem_sf1_deletes/setup.py | 8 ++ .../lineitem_sf_01_1_delete/q00.sql | 6 + .../lineitem_sf_01_1_delete/q01.sql | 1 + .../lineitem_sf_01_1_delete/setup.py | 8 ++ .../lineitem_sf_01_no_deletes/q00.sql | 6 + .../lineitem_sf_01_no_deletes/setup.py | 8 ++ .../pyspark_iceberg_table_v1/q00.sql | 1 + .../pyspark_iceberg_table_v1/q01.sql | 13 ++ .../pyspark_iceberg_table_v1/q02.sql | 3 + .../pyspark_iceberg_table_v1/q03.sql | 2 + .../pyspark_iceberg_table_v1/q04.sql | 3 + .../pyspark_iceberg_table_v1/q05.sql | 3 + .../pyspark_iceberg_table_v1/q06.sql | 2 + .../pyspark_iceberg_table_v1/q07.sql | 3 + .../pyspark_iceberg_table_v1/q08.sql | 2 + .../pyspark_iceberg_table_v1/setup.py | 30 ++++ .../pyspark_iceberg_table_v2/q00.sql | 1 + .../pyspark_iceberg_table_v2/q01.sql | 13 ++ .../pyspark_iceberg_table_v2/q02.sql | 3 + .../pyspark_iceberg_table_v2/q03.sql | 2 + .../pyspark_iceberg_table_v2/q04.sql | 3 + .../pyspark_iceberg_table_v2/q05.sql | 3 + .../pyspark_iceberg_table_v2/q06.sql | 2 + .../pyspark_iceberg_table_v2/q07.sql | 3 + .../pyspark_iceberg_table_v2/q08.sql | 2 + .../pyspark_iceberg_table_v2/setup.py | 30 ++++ .../table_more_deletes/q00.sql | 12 ++ .../table_more_deletes/q01.sql | 14 ++ .../table_more_deletes/q02.sql | 2 + .../table_partitioned/q00.sql | 7 + .../table_partitioned/q01.sql | 14 ++ .../table_unpartitioned/q00.sql | 7 + .../table_unpartitioned/q01.sql | 14 ++ scripts/data_generators/tmp_data/tmp.parquet | Bin 1822358 -> 802580 bytes scripts/polaris/create_data.py | 3 +- test/sql/local/irc/test_polaris_tpch.test | 56 ++++++++ 51 files changed, 536 insertions(+), 4 deletions(-) create mode 100644 scripts/data_generators/generate_polaris_rest/__init__.py create mode 100644 scripts/data_generators/generate_polaris_rest/empty_table/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/generate_iceberg_polaris_rest.py create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q02.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q03.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q04.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q05.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q06.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q07.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q08.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q02.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q03.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q04.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q05.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q06.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q07.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q08.sql create mode 100644 scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/setup.py create mode 100644 scripts/data_generators/generate_polaris_rest/table_more_deletes/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/table_more_deletes/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/table_more_deletes/q02.sql create mode 100644 scripts/data_generators/generate_polaris_rest/table_partitioned/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/table_partitioned/q01.sql create mode 100644 scripts/data_generators/generate_polaris_rest/table_unpartitioned/q00.sql create mode 100644 scripts/data_generators/generate_polaris_rest/table_unpartitioned/q01.sql create mode 100644 test/sql/local/irc/test_polaris_tpch.test diff --git a/scripts/data_generators/generate_data.py b/scripts/data_generators/generate_data.py index 5375e63e..c936ea2a 100644 --- a/scripts/data_generators/generate_data.py +++ b/scripts/data_generators/generate_data.py @@ -1,17 +1,43 @@ from generate_spark_local.generate_iceberg_spark_local import IcebergSparkLocal from generate_spark_rest.generate_iceberg_spark_rest import IcebergSparkRest +from generate_polaris_rest.generate_iceberg_polaris_rest import IcebergPolarisRest +import sys -# Example usage: -if __name__ == "__main__": +def GenerateSparkRest(): db2 = IcebergSparkRest() conn2 = db2.GetConnection() db2.GenerateTables(conn2) db2.CloseConnection(conn2) del db2 del conn2 + +def GenerateSparkLocal(): db = IcebergSparkLocal() conn = db.GetConnection() db.GenerateTables(conn) db.CloseConnection(conn) del db del conn + + +def GeneratePolarisData(): + db = IcebergPolarisRest() + conn = db.GetConnection() + db.GenerateTables(conn) + db.CloseConnection(conn) + del db + del conn + +if __name__ == "__main__": + import pdb + pdb.set_trace() + argv = sys.argv + for i in range(1, len(argv)): + if argv[i] == "polaris": + GeneratePolarisData() + elif argv[i] == "local": + GenerateSparkLocal() + elif argv[i] == "spark-rest": + GenerateSparkRest + else: + print(f"{argv[i]} not recognized, skipping") diff --git a/scripts/data_generators/generate_polaris_rest/__init__.py b/scripts/data_generators/generate_polaris_rest/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/__init__.py @@ -0,0 +1 @@ + diff --git a/scripts/data_generators/generate_polaris_rest/empty_table/q00.sql b/scripts/data_generators/generate_polaris_rest/empty_table/q00.sql new file mode 100644 index 00000000..a3cb6724 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/empty_table/q00.sql @@ -0,0 +1,9 @@ +CREATE or REPLACE TABLE default.empty_table ( + col1 date, + col2 integer, + col3 string +) +TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' +); \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/generate_iceberg_polaris_rest.py b/scripts/data_generators/generate_polaris_rest/generate_iceberg_polaris_rest.py new file mode 100644 index 00000000..9a4984af --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/generate_iceberg_polaris_rest.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyspark.sql import SparkSession + +#!/usr/bin/python3 +import pyspark +import pyspark.sql +import sys +import duckdb +import os +from pyspark import SparkContext +from pathlib import Path +import duckdb +import shutil + +DATA_GENERATION_DIR = f"./data/generated/iceberg/polaris-rest/" +SCRIPT_DIR = f"./scripts/data_generators/" +INTERMEDIATE_DATA = "./data/generated/intermediates/polaris-rest/" +PARQUET_SRC_FILE = f"scripts/data_generators/tmp_data/tmp.parquet" + + +class IcebergPolarisRest: + def __init__(self): + pass + + ### + ### Configure everyone's favorite apache product + ### + def GetConnection(self): + os.environ["PYSPARK_SUBMIT_ARGS"] = ( + "--packages org.apache.iceberg:iceberg-spark-runtime-3.4_2.12:1.4.2,org.apache.iceberg:iceberg-aws-bundle:1.4.2 pyspark-shell" + ) + + client_id = os.getenv('POLARIS_CLIENT_ID', '') + client_secret = os.getenv('POLARIS_CLIENT_SECRET', '') + os.environ["AWS_REGION"] = "us-east-1" + os.environ["AWS_ACCESS_KEY_ID"] = "admin" + os.environ["AWS_SECRET_ACCESS_KEY"] = "password" + + if client_id == '' or client_secret == '': + print("could not find client id or client secret to connect to polaris, aborting") + return + + spark = ( + SparkSession.builder.config("spark.sql.catalog.quickstart_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config( + "spark.jars.packages", + "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19", + ) + .config('spark.sql.iceberg.vectorization.enabled', 'false') + # Configure the 'polaris' catalog as an Iceberg rest catalog + .config("spark.sql.catalog.quickstart_catalog.type", "rest") + .config("spark.sql.catalog.quickstart_catalog", "org.apache.iceberg.spark.SparkCatalog") + # Specify the rest catalog endpoint + .config("spark.sql.catalog.quickstart_catalog.uri", "http://localhost:8181/api/catalog") + # Enable token refresh + .config("spark.sql.catalog.quickstart_catalog.token-refresh-enabled", "true") + # specify the client_id:client_secret pair + .config("spark.sql.catalog.quickstart_catalog.credential", f"{client_id}:{client_secret}") + # Set the warehouse to the name of the catalog we created + .config("spark.sql.catalog.quickstart_catalog.warehouse", "quickstart_catalog") + # Scope set to PRINCIPAL_ROLE:ALL + .config("spark.sql.catalog.quickstart_catalog.scope", 'PRINCIPAL_ROLE:ALL') + # Enable access credential delegation + .config("spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation", 'vended-credentials') + .config("spark.sql.catalog.quickstart_catalog.io-impl", "org.apache.iceberg.io.ResolvingFileIO") + .config("spark.sql.catalog.quickstart_catalog.s3.region", "us-west-2") + .config("spark.history.fs.logDirectory", "/home/iceberg/spark-events") + ).getOrCreate() + spark.sql("USE quickstart_catalog") + spark.sql("CREATE NAMESPACE IF NOT EXISTS default") + spark.sql("USE NAMESPACE default") + return spark + + def GetSQLFiles(self, table_dir): + sql_files = [f for f in os.listdir(table_dir) if f.endswith('.sql')] # Find .sql files + sql_files.sort() # Order matters obviously # Store results + return sql_files + + def GetTableDirs(self): + dir = "./scripts/data_generators/generate_polaris_rest/" + subdirectories = [d for d in os.listdir(dir) if os.path.isdir(dir + d) and d != "__pycache__"] + return subdirectories + + def GetSetupFile(self, dir): + setup_files = [f for f in os.listdir(dir) if 'setup' in f.lower()] + if len(setup_files) == 0: + return "" + return setup_files[0] + + def GenerateTPCH(self, con): + duckdb_con = duckdb.connect() + duckdb_con.execute("call dbgen(sf=1)") + + for tbl in ['lineitem', 'customer', 'nation', 'orders', 'part', 'partsupp', 'region', 'supplier']: + create_statement = f""" + CREATE or REPLACE TABLE default.{tbl} + TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' + ) + AS SELECT * FROM parquet_file_view; + """ + duckdb_con.execute(f"copy {tbl} to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") + con.read.parquet(PARQUET_SRC_FILE).createOrReplaceTempView('parquet_file_view') + con.sql(create_statement) + + def GenerateTables(self, con): + # Generate the tpch tables + self.GenerateTPCH(con) + + + def CloseConnection(self, con): + del con diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q00.sql b/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q00.sql new file mode 100644 index 00000000..c581d869 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q00.sql @@ -0,0 +1,6 @@ +CREATE or REPLACE TABLE default.lineitem_001_deletes + TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' + ) +AS SELECT * FROM parquet_file_view; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q01.sql b/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q01.sql new file mode 100644 index 00000000..472849d5 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/q01.sql @@ -0,0 +1,11 @@ +update default.lineitem_001_deletes +set l_orderkey=NULL, + l_partkey=NULL, + l_suppkey=NULL, + l_linenumber=NULL, + l_quantity=NULL, + l_extendedprice=NULL, + l_discount=NULL, + l_shipdate=NULL, + l_comment=NULL +where l_partkey % 2 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/setup.py b/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/setup.py new file mode 100644 index 00000000..78489008 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_001_deletes/setup.py @@ -0,0 +1,8 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=0.01)") +duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q00.sql b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q00.sql new file mode 100644 index 00000000..fa22eb3f --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q00.sql @@ -0,0 +1,8 @@ +CREATE OR REPLACE TABLE default.lineitem_partitioned_l_shipmode +USING iceberg +PARTITIONED BY (l_shipmode) +TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' +) +as select * from parquet_file_view; diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q01.sql b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q01.sql new file mode 100644 index 00000000..c7047b5f --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/q01.sql @@ -0,0 +1 @@ +delete from default.lineitem_partitioned_l_shipmode where l_shipmode = 'TRUCK'; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/setup.py b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/setup.py new file mode 100644 index 00000000..78489008 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode/setup.py @@ -0,0 +1,8 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=0.01)") +duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q00.sql b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q00.sql new file mode 100644 index 00000000..e1efd191 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q00.sql @@ -0,0 +1,8 @@ +CREATE OR REPLACE TABLE default.lineitem_partitioned_l_shipmode_deletes +USING iceberg +PARTITIONED BY (l_shipmode) +TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' +) +as select * from parquet_file_view; diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q01.sql b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q01.sql new file mode 100644 index 00000000..81163885 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/q01.sql @@ -0,0 +1,6 @@ +UPDATE default.lineitem_partitioned_l_shipmode_deletes +Set l_comment=NULL, + l_quantity=NULL, + l_discount=NULL, + l_linestatus=NULL +where l_linenumber = 3 or l_linenumber = 4 or l_linenumber = 5; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/setup.py b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/setup.py new file mode 100644 index 00000000..78489008 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_partitioned_l_shipmode_deletes/setup.py @@ -0,0 +1,8 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=0.01)") +duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q00.sql b/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q00.sql new file mode 100644 index 00000000..4112dadb --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q00.sql @@ -0,0 +1,6 @@ +CREATE or REPLACE TABLE default.lineitem_sf1_deletes + TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' + ) +AS SELECT * FROM parquet_file_view; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q01.sql b/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q01.sql new file mode 100644 index 00000000..4f789bda --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/q01.sql @@ -0,0 +1,11 @@ +update default.lineitem_sf1_deletes +set l_orderkey=NULL, + l_partkey=NULL, + l_suppkey=NULL, + l_linenumber=NULL, + l_quantity=NULL, + l_extendedprice=NULL, + l_discount=NULL, + l_shipdate=NULL, + l_comment=NULL +where l_partkey % 2 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/setup.py b/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/setup.py new file mode 100644 index 00000000..12ce8006 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf1_deletes/setup.py @@ -0,0 +1,8 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=1)") +duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q00.sql b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q00.sql new file mode 100644 index 00000000..78934704 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q00.sql @@ -0,0 +1,6 @@ +CREATE or REPLACE TABLE default.lineitem_sf_01_1_delete + TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' + ) +AS SELECT * FROM parquet_file_view; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q01.sql b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q01.sql new file mode 100644 index 00000000..8fcc58ed --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/q01.sql @@ -0,0 +1 @@ +delete from default.lineitem_sf_01_1_delete where l_orderkey=10053 and l_partkey = 77; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/setup.py b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/setup.py new file mode 100644 index 00000000..78489008 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_1_delete/setup.py @@ -0,0 +1,8 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=0.01)") +duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/q00.sql b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/q00.sql new file mode 100644 index 00000000..b21fa208 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/q00.sql @@ -0,0 +1,6 @@ +CREATE or REPLACE TABLE default.lineitem_sf_01_no_deletes + TBLPROPERTIES ( + 'format-version'='2', + 'write.update.mode'='merge-on-read' + ) +AS SELECT * FROM parquet_file_view; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/setup.py b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/setup.py new file mode 100644 index 00000000..78489008 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/lineitem_sf_01_no_deletes/setup.py @@ -0,0 +1,8 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=0.01)") +duckdb_con.execute(f"copy lineitem to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q00.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q00.sql new file mode 100644 index 00000000..cb12b680 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q00.sql @@ -0,0 +1 @@ +CREATE or REPLACE TABLE default.pyspark_iceberg_table_v1 TBLPROPERTIES ('format-version'='1') AS SELECT * FROM parquet_file_view; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q01.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q01.sql new file mode 100644 index 00000000..b84bc2a2 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q01.sql @@ -0,0 +1,13 @@ +update default.pyspark_iceberg_table_v1 +set l_orderkey_bool=NULL, + l_partkey_int=NULL, + l_suppkey_long=NULL, + l_extendedprice_float=NULL, + l_extendedprice_double=NULL, + l_shipdate_date=NULL, + l_partkey_time=NULL, + l_commitdate_timestamp=NULL, + l_commitdate_timestamp_tz=NULL, + l_comment_string=NULL, + l_comment_blob=NULL +where l_partkey_int % 2 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q02.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q02.sql new file mode 100644 index 00000000..e31e7597 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q02.sql @@ -0,0 +1,3 @@ +insert into default.pyspark_iceberg_table_v1 +select * FROM default.pyspark_iceberg_table_v1 +where l_extendedprice_double < 30000 \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q03.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q03.sql new file mode 100644 index 00000000..637d4219 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q03.sql @@ -0,0 +1,2 @@ +update default.pyspark_iceberg_table_v1 +set l_orderkey_bool = not l_orderkey_bool; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q04.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q04.sql new file mode 100644 index 00000000..94605930 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q04.sql @@ -0,0 +1,3 @@ +update default.pyspark_iceberg_table_v1 +set l_orderkey_bool = false +where l_partkey_int % 4 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q05.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q05.sql new file mode 100644 index 00000000..13dd2e38 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q05.sql @@ -0,0 +1,3 @@ +update default.pyspark_iceberg_table_v1 +set l_orderkey_bool = false +where l_partkey_int % 5 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q06.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q06.sql new file mode 100644 index 00000000..12b8c7c7 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q06.sql @@ -0,0 +1,2 @@ +ALTER TABLE default.pyspark_iceberg_table_v1 + ADD COLUMN schema_evol_added_col_1 INT DEFAULT 42; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q07.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q07.sql new file mode 100644 index 00000000..b3b40977 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q07.sql @@ -0,0 +1,3 @@ +UPDATE default.pyspark_iceberg_table_v1 +SET schema_evol_added_col_1 = l_partkey_int +WHERE l_partkey_int % 5 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q08.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q08.sql new file mode 100644 index 00000000..f018d3f0 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/q08.sql @@ -0,0 +1,2 @@ +ALTER TABLE default.pyspark_iceberg_table_v1 +ALTER COLUMN schema_evol_added_col_1 TYPE BIGINT; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/setup.py b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/setup.py new file mode 100644 index 00000000..1474e8c1 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v1/setup.py @@ -0,0 +1,30 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=0.001)") +duckdb_con.query( + """CREATE VIEW test_table as + SELECT + (l_orderkey%2=0) as l_orderkey_bool, + l_partkey::INT32 as l_partkey_int, + l_suppkey::INT64 as l_suppkey_long, + l_extendedprice::FLOAT as l_extendedprice_float, + l_extendedprice::DOUBLE as l_extendedprice_double, + l_extendedprice::DECIMAL(9,2) as l_extendedprice_dec9_2, + l_extendedprice::DECIMAL(18,6) as l_extendedprice_dec18_6, + l_extendedprice::DECIMAL(38,10) as l_extendedprice_dec38_10, + l_shipdate::DATE as l_shipdate_date, + l_partkey as l_partkey_time, + l_commitdate::TIMESTAMP as l_commitdate_timestamp, + l_commitdate::TIMESTAMPTZ as l_commitdate_timestamp_tz, + l_comment as l_comment_string, + gen_random_uuid()::VARCHAR as uuid, + l_comment::BLOB as l_comment_blob + FROM + lineitem;""" +) + +duckdb_con.execute(f"copy test_table to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q00.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q00.sql new file mode 100644 index 00000000..1743f22d --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q00.sql @@ -0,0 +1 @@ +CREATE or REPLACE TABLE default.pyspark_iceberg_table_v2 TBLPROPERTIES ('format-version'='2', 'write.update.mode'='merge-on-read') AS SELECT * FROM parquet_file_view; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q01.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q01.sql new file mode 100644 index 00000000..205669c6 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q01.sql @@ -0,0 +1,13 @@ +update default.pyspark_iceberg_table_v2 +set l_orderkey_bool=NULL, + l_partkey_int=NULL, + l_suppkey_long=NULL, + l_extendedprice_float=NULL, + l_extendedprice_double=NULL, + l_shipdate_date=NULL, + l_partkey_time=NULL, + l_commitdate_timestamp=NULL, + l_commitdate_timestamp_tz=NULL, + l_comment_string=NULL, + l_comment_blob=NULL +where l_partkey_int % 2 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q02.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q02.sql new file mode 100644 index 00000000..1a7aa136 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q02.sql @@ -0,0 +1,3 @@ +insert into default.pyspark_iceberg_table_v2 +select * FROM default.pyspark_iceberg_table_v2 +where l_extendedprice_double < 30000 \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q03.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q03.sql new file mode 100644 index 00000000..133da8ba --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q03.sql @@ -0,0 +1,2 @@ +update default.pyspark_iceberg_table_v2 +set l_orderkey_bool = not l_orderkey_bool; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q04.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q04.sql new file mode 100644 index 00000000..6bbbb058 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q04.sql @@ -0,0 +1,3 @@ +delete +from default.pyspark_iceberg_table_v2 +where l_extendedprice_double < 10000; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q05.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q05.sql new file mode 100644 index 00000000..2780a8a9 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q05.sql @@ -0,0 +1,3 @@ +delete +from default.pyspark_iceberg_table_v2 +where l_extendedprice_double > 70000; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q06.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q06.sql new file mode 100644 index 00000000..1a9eafd8 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q06.sql @@ -0,0 +1,2 @@ +ALTER TABLE default.pyspark_iceberg_table_v2 + ADD COLUMN schema_evol_added_col_1 INT DEFAULT 42; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q07.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q07.sql new file mode 100644 index 00000000..66ed1232 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q07.sql @@ -0,0 +1,3 @@ +UPDATE default.pyspark_iceberg_table_v2 +SET schema_evol_added_col_1 = l_partkey_int +WHERE l_partkey_int % 5 = 0; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q08.sql b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q08.sql new file mode 100644 index 00000000..99a53dd4 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/q08.sql @@ -0,0 +1,2 @@ +ALTER TABLE default.pyspark_iceberg_table_v2 +ALTER COLUMN schema_evol_added_col_1 TYPE BIGINT; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/setup.py b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/setup.py new file mode 100644 index 00000000..1474e8c1 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/pyspark_iceberg_table_v2/setup.py @@ -0,0 +1,30 @@ +import duckdb +import os + +PARQUET_SRC_FILE = os.getenv('PARQUET_SRC_FILE') + +duckdb_con = duckdb.connect() +duckdb_con.execute("call dbgen(sf=0.001)") +duckdb_con.query( + """CREATE VIEW test_table as + SELECT + (l_orderkey%2=0) as l_orderkey_bool, + l_partkey::INT32 as l_partkey_int, + l_suppkey::INT64 as l_suppkey_long, + l_extendedprice::FLOAT as l_extendedprice_float, + l_extendedprice::DOUBLE as l_extendedprice_double, + l_extendedprice::DECIMAL(9,2) as l_extendedprice_dec9_2, + l_extendedprice::DECIMAL(18,6) as l_extendedprice_dec18_6, + l_extendedprice::DECIMAL(38,10) as l_extendedprice_dec38_10, + l_shipdate::DATE as l_shipdate_date, + l_partkey as l_partkey_time, + l_commitdate::TIMESTAMP as l_commitdate_timestamp, + l_commitdate::TIMESTAMPTZ as l_commitdate_timestamp_tz, + l_comment as l_comment_string, + gen_random_uuid()::VARCHAR as uuid, + l_comment::BLOB as l_comment_blob + FROM + lineitem;""" +) + +duckdb_con.execute(f"copy test_table to '{PARQUET_SRC_FILE}' (FORMAT PARQUET)") diff --git a/scripts/data_generators/generate_polaris_rest/table_more_deletes/q00.sql b/scripts/data_generators/generate_polaris_rest/table_more_deletes/q00.sql new file mode 100644 index 00000000..fae1cc3e --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/table_more_deletes/q00.sql @@ -0,0 +1,12 @@ +CREATE OR REPLACE TABLE default.table_more_deletes ( + dt date, + number integer, + letter string + ) + USING iceberg + TBLPROPERTIES ( + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read', + 'format-version'='2' + ); \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/table_more_deletes/q01.sql b/scripts/data_generators/generate_polaris_rest/table_more_deletes/q01.sql new file mode 100644 index 00000000..912dae78 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/table_more_deletes/q01.sql @@ -0,0 +1,14 @@ +INSERT INTO default.table_more_deletes +VALUES + (CAST('2023-03-01' AS date), 1, 'a'), + (CAST('2023-03-02' AS date), 2, 'b'), + (CAST('2023-03-03' AS date), 3, 'c'), + (CAST('2023-03-04' AS date), 4, 'd'), + (CAST('2023-03-05' AS date), 5, 'e'), + (CAST('2023-03-06' AS date), 6, 'f'), + (CAST('2023-03-07' AS date), 7, 'g'), + (CAST('2023-03-08' AS date), 8, 'h'), + (CAST('2023-03-09' AS date), 9, 'i'), + (CAST('2023-03-10' AS date), 10, 'j'), + (CAST('2023-03-11' AS date), 11, 'k'), + (CAST('2023-03-12' AS date), 12, 'l'); diff --git a/scripts/data_generators/generate_polaris_rest/table_more_deletes/q02.sql b/scripts/data_generators/generate_polaris_rest/table_more_deletes/q02.sql new file mode 100644 index 00000000..0b85ca8b --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/table_more_deletes/q02.sql @@ -0,0 +1,2 @@ +Delete from default.table_more_deletes +where number > 3 and number < 10; \ No newline at end of file diff --git a/scripts/data_generators/generate_polaris_rest/table_partitioned/q00.sql b/scripts/data_generators/generate_polaris_rest/table_partitioned/q00.sql new file mode 100644 index 00000000..2aba8da4 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/table_partitioned/q00.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE TABLE default.table_partitioned ( + dt date, + number integer, + letter string +) +USING iceberg +PARTITIONED BY (days(dt)) diff --git a/scripts/data_generators/generate_polaris_rest/table_partitioned/q01.sql b/scripts/data_generators/generate_polaris_rest/table_partitioned/q01.sql new file mode 100644 index 00000000..ecbcd5e8 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/table_partitioned/q01.sql @@ -0,0 +1,14 @@ +INSERT INTO default.table_partitioned +VALUES + (CAST('2023-03-01' AS date), 1, 'a'), + (CAST('2023-03-02' AS date), 2, 'b'), + (CAST('2023-03-03' AS date), 3, 'c'), + (CAST('2023-03-04' AS date), 4, 'd'), + (CAST('2023-03-05' AS date), 5, 'e'), + (CAST('2023-03-06' AS date), 6, 'f'), + (CAST('2023-03-07' AS date), 7, 'g'), + (CAST('2023-03-08' AS date), 8, 'h'), + (CAST('2023-03-09' AS date), 9, 'i'), + (CAST('2023-03-10' AS date), 10, 'j'), + (CAST('2023-03-11' AS date), 11, 'k'), + (CAST('2023-03-12' AS date), 12, 'l'); diff --git a/scripts/data_generators/generate_polaris_rest/table_unpartitioned/q00.sql b/scripts/data_generators/generate_polaris_rest/table_unpartitioned/q00.sql new file mode 100644 index 00000000..b7f2c4c9 --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/table_unpartitioned/q00.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE TABLE default.table_unpartitioned ( + dt date, + number integer, + letter string +) +USING iceberg +; diff --git a/scripts/data_generators/generate_polaris_rest/table_unpartitioned/q01.sql b/scripts/data_generators/generate_polaris_rest/table_unpartitioned/q01.sql new file mode 100644 index 00000000..cb58794b --- /dev/null +++ b/scripts/data_generators/generate_polaris_rest/table_unpartitioned/q01.sql @@ -0,0 +1,14 @@ +INSERT INTO default.table_unpartitioned + VALUES + (CAST('2023-03-01' AS date), 1, 'a'), + (CAST('2023-03-02' AS date), 2, 'b'), + (CAST('2023-03-03' AS date), 3, 'c'), + (CAST('2023-03-04' AS date), 4, 'd'), + (CAST('2023-03-05' AS date), 5, 'e'), + (CAST('2023-03-06' AS date), 6, 'f'), + (CAST('2023-03-07' AS date), 7, 'g'), + (CAST('2023-03-08' AS date), 8, 'h'), + (CAST('2023-03-09' AS date), 9, 'i'), + (CAST('2023-03-10' AS date), 10, 'j'), + (CAST('2023-03-11' AS date), 11, 'k'), + (CAST('2023-03-12' AS date), 12, 'l'); diff --git a/scripts/data_generators/tmp_data/tmp.parquet b/scripts/data_generators/tmp_data/tmp.parquet index 3600bba4c2d147148b5a3d4709e2bfe6b8451478..63c4314a4ce7400ddf5cc0e98f6e3a504a46ba8b 100644 GIT binary patch literal 802580 zcmeF)1$-RWfj@A&r>5?;DW{Mtqzsp9&2a5q8-rxawrtC?Ewg2|OfoYwGcz+YGcz+Y zGt>XOo-|JTPn)>OrA@N_e7^7ZwEJdf_RYNU?r5Jj?_05{UQ6~`_mf|IZQ+xMH(wh* zIJqBRdF_>C$=ctD_*q1#vn2U@!vuGK8QaA@jpTQZxDYB;!i=m6-4GBvILPe zh-^Vb1`!oR_8@Wu5gkO%AaVtfJBXMd@&u7Lh2+^B`IT(K3iuL9`B{O%QE^ zXct8LAUXumF^EnF+GSG zLCg$dRuHp;m=na@Am#-zKZpfEEDT~%5Q~FY62#IVmIbjqh!sJs3}RIftAkh*#M&U% z1+hMe4MA)SVp9;CgV++p)*!Y8u|0?#LF^1-R}i~{*b~IwAoc~ZKZpZC91P-65Ql>} z62#FUjs=2gDJSNR%W=No*JZ}A%6<~w|s@9{eS!T0$AKjcTe!H@X~Kjmlq zoL}%ue#Ni(4gblT{1?CFccdg0sYydx(vhAFWF!;+&F}dG|HJ?CNB+cHWF`w)$wnkm zWG4sF|nJQGJ z8r4an1~sWgZR${$deo-@4QWJUn$VPHG^YhEX+>+=(3WeG#AU8YE&nQ8q}l~wW&j0>QSEtG^7!YX+l$)(VP~vq!q1cLtEO> zo(^=R6P@WoSGv)i9`vLaz3D?=`q7^O3}g_48NyJ8F`N;MWE7(r!&t^Ko(W835|f$2 zRHiYV8O&rBvzfzO<}sfIEMyUjS;A75v78mGWEHDf!&=s{o(*hd6Pww>R<^O79qeQm zyV=8D_OYJ>9OMv(Il@tnahwyJojsg^<5b+eI z2t_GIaY|5F`or2WD$#5!cvy8oE5BO6{}gpTGp|i4Qyl+ zo7uuvwy~WZ>|__a*~4D;v7ZARL>+EMz4ckwlT597L0oT;wK(Jme)G zvE(O?0u-bW@f4;AMJYycN>GwgBv6_%l%*WysX#?4kw|5#P?c&_Cy5%=q!zWQLtW}o zp9VCf5shg=Q<~A77PO=lt!YDB+R>g4bfgoV=|We!(VZUjq!+#ELtpyQp8*VH5Q7=Q zP=+y_5sYLMqZz|k#xb4=Ok@(1nZi`2F`XIAWEQiT!(8Sup9L&r5sO*EQkJot6|7_x zt69TZ*0G)qY-AIg*}_(~v7H_4WEZ>H!(R5Wp937^5QjO!QI2t(6P)A}r#Zt}&T*a# zT;vj$xx!Vhah)67RG6s8D8DMoQhP?Az4P?|E7r5xp{Kt(E%NM))}m1UG8z82R!5vk9opVo*Dl;erNn2_+5DX z>yIpCB^!}Mk)0ewlapNJCWbuZB_FZmCyoLXq!95GrU*qTMsZ3|l2Rm4nlhB79ObD% zMJkaYE&nQ8q}l~wW&j0>QSEtG^7!YX+l$)(VP~vq!q1cLtEO>o(^=R6P@Wo zSGv)i9`vLaz3D?=`q7^O3}g_48NyJ8F`N;MWE7(r!&t^Ko(W835|f$2RHiYV8O&rB zvzfzO<}sfIEMyUjS;A75v78mGWEHDf!&=s{o(*hd6Pww>R<^O79qeQmyV=8D_OYJ> z9OMv(Il@tnahwyJvmgvY=B$U;`K5lIx;$w4$Z$wh8r$U|Q85lepJC_q695l>-?P?Ta6rvxP_ zMFOQMLs`mEo(fc?5{Xo%3RS5_b&{w-O=?k_I@F~e^=Uvu8qt_0G^H8MX+cX`(V8~2 zr5)|*Ku0>!nJ#pt8{O$aPkPatKJ=v@{TaYO1~Hf+3}qO@8NoS|UJKW_S z_j$lW9`TqbJms13zv7R^|As$?$G`r_LRPX7Nfg=1K{PqZMQ&oqLtgR`OMc=gKtT!- zPhpBslwuU81SKg&0;MTKS;|qK3RI*LiBzTvRjEdGlBhvVYEhdy)TJKvX+T37(U>MQ zr5Vj>K}%ZEnl`kh9qs8rM>^4&E_9_E-RVJ3deNIc^ravD8NfgWF_<9?Wf;R5!AM3i znlX%J9OIe5L?$trDNJP=)0x3cW-*&N%w-<)S-?UTv6v++Wf{v^!Ae%Knl-Ft9qZY^ zMmDjTEo@~Q+u6ZRcCnj1>}4POIlw^fMJ{ofD_rFo z*SWz>ZgHDC+~pqkdB8&+@t7w(<(cunAj|nJQGJ8r4an1~sWgZR${$ zdeo-@4QWJUn$VPHG^YhEX+>+=(3WeG#AU8#VAe*N>YjhN>hfil%qTqs7NIesZ15BQjO{)QG=S) zqBeD?OFin-fQB@pF->SnGn&(amb9WZZD>n7+S7rKbfPm|=t?)b(}SM$qBni$OF#NE zfPoBRFhdy1ForXNk&I$AV;IXg#xsG5Oky%qn94M!GlQATVm5P_%RJ_@fQ2k#F-us= zGM2M~m8@blYgo%V*0X_)Y+^H8*vdAxvxA-NVmEu(%RcsVfP)<3Fh@AbF^+SBlbqr- zXE@6_&U1l_T;eiUxXLxIbAy}Q;x>1<%RTP%fQLNdF;95PGm^bNp-}h)pd&Jqg{)*F zk|?s1gJ^P+i`>MJhrHw?mi)v~fPxevp28HND8(pF2})9m1WHqevXrAd6{tuh5~)lT zs#1;WBvFH!)S@Q6 z^rAO?=u1EPGk}2%VlYD($}omAf{~13G-DXcIL0%9iA-WLQ<%y$rZa|!^2*vmflbAW>!;xI=z z$}x^}f|H!$G-o)=InHx|i(KL|SGdYGu5*K%+~PKOxXV56^MHpu;xSKn$}{f(MwIjp zU_#083jqC*g{)*Fk|?s1gJ^P+i`>MJhrHw?mi)v~fPxevp28HND8(pF2})9m1WHqe zvXrAd6{tuh5~)lTs#1;WBvFH!)S@Q6^rAO?=u1EPGk}2%VlYD($}omAf{~13G-DXcIL0%9iA-WLQ<%y$ zrZa|!^2 z*vmflbAW>!;xI=z$}x^}f|H!$G-o)=InHx|i(KL|SGdYGu5*K%+~PKOxXV56^MHpu z;xSKn$}{7CK!u3puYTp#QeFyt&Q1SD1fS;>zCa4T$iMR?zRat9g|G58zRoxJCg0*U zzRh>|F5lyI{)6xH1AfSlc!MAF6Mo9i_&LAem;8!f^Bew?H~BAq%kM}@DpHe%w4@_F z8OTT`{+r+P2mXiu<&XS{x5!KuvXYHRqR37TqRB}vauY)y@{*5O@)Jh^3Q~x83R8rl z6r(sLC`l<2C`}p4QjYRepdyt>q%u{gN;RsJL=9?Ei`vwoF7>ES0~*qZ#x$WR&1g;w zTGEQvw4p8SXio<^(uvM=p)1|!P7iw0i{A91Fa7Ax00uIM!3<$2!x+v8Mly=gjA1O} z7|#SIGKtAdVJg#@&J1QUi`mR!F7uer0v57}#Vlbd%UI3|RiB0D*VCMUVbO$>R+OFm-B zPaFj(NFm}WOc9DwjN+7_B&A58G-W7DIm%Okic}(z%2c5$)u>JqHK<7~YEy^0)T2HP zXhlxi$tXrMhOvxeJQJA6BqlS3sZ3)!GnmONW;2Jm%ws+aSjZw4vxKEAV>v5W$tqT} zhPA9?Jsa4_CN{H$t!!gEJJ`uCcC&}Q>|;L%ILILmbA+QD<2WZc$tg~AhO?aGJQujg zB`$M?t6bwcH@L|yZgYpb+~YnEc*r9j^Mt27GyXf&GyZ$j3y**Ok%g>eBa$exlY?k- zl8fBLkcYhFBbNNcQGkLJBA&t&p(w>DP6LgKv zn$)5;b*M`{>eGORG@>z0Xi77h(}I??qBU)3OFP=rfsS;dGhOIPH@eeEMhTB zSjsY%vx1eZVl``6%R1JxfsJfpGh5ioHny{ao$O*ad)Ui9_H%%P9O5uXILa}ObApqc z;xuPC%Q?<-fs0(?GFQ0DHLi1mo800yceu+v?(=|$JmN7=c*--9z22f(_!^*&$V?Wp zl8s2B$W9KT$w@A96GI;Il8;#O6Gs6GQiym8Q-q=vqc|lfNhuO2O&Q8kj`CEXB9%y_ zGF7NbHL8vz-t?g_{pimC1~Q1j3}Gn47|sYrGK$fRVJzbq&jcniiOEc1D$|(G3}!Nm+00=s z^O(;97P5%NEMY0jSk4MovWnHLVJ+)e&jvQKiOpt?FvXYHRqR37TqRB}vauY)y@{*5O@)Jh^3Q~x83R8rl6r(sLC`l<2C`}p4 zQjYRepdyt>q%u{gN;RsJL=9?Ei`vwoF7>ES0~*qZ#x$WR&1g;wTGEQvw4p8SXio<^ z(uvM=p)1|!P7iw0i{A91Fa7Ax00uIM!3<$2!x+v8Mly=gjA1O}7|#SIGKtAdVJg#@ z&J1QUi`mR!F7uer0v57}#Vlbd%UI3|R%2JNGbfGKV=uQuM(u>~op)dXD&j1E8h`|hDD8m@e2u3oB(Trg%;~38b zCNhc1Okpb1n9dAlGK<;FVJ`ES&jJ>*h{Y^nDa%;S3Rbd;)vRGH>sZeQHnNG$Y+)*>T;VF$xXul3a*NyC z;V$>M&jTLvh{rtPDbHL2sMOCTfExY6mjLuf7P69!NTSG24x-6PE^-q?9`cfpSn?A` z0SZ!xcnVX5q7%2JNGbfGKV=uQuM(u>~op)dXD&j1E8h`|hDD8m@e z2u3oB(Trg%;~38bCNhc1Okpb1n9dAlGK<;FVJ`ES&jJ>*h{Y^nDa%;S3Rbd;)vRGH z>sZeQHnNG$Y+)*> zT;VF$xXul3a*NyC;V$>M&jTLvh{rtPDbI}mgki>irD5UmuRpSom25;3MRsx!O-^!= zn;7ztmwd#MpEwFokV3>$m?9LV7{w_;NlKAGY06NRa+Ie66{$obm8n8is!^RJYEY9} z)TRz~sYiVp(2zznrU^}HMsr%wl2){)4Q**hdpgjOPIRUVUFk-5deDAZhTiM2TcCeFO>}C&p*~fkkaF9bB<_JeQ#&J$?l2e@K z3}-pVc`k5~OI+p(SGmS@Zg7)Z+~y8e zBa$exlY?k-l8fBLkcYhFBbNNcQGkLJBA&t&p(w>DP6LgKvn$)5;b*M`{>eGORG@>z0Xi77h(}I??qBU)3OFP=rfsS;dGhOIPH@ee< zp7f$OedtR+`ZIum3}P@t7|Jk)GlG$fVl-nI%Q(g}fr(6FGEEMhTBSjsY%vx1eZVl``6%R1JxfsJfpGh5ioHny{ao$O*ad)Ui9_H%%P9O5uX zILa}ObApqc;xuPC%Q?<-fs0(?GFQ0DHLi1mo800yceu+v?(=|$JmN7=c*-;5Ki@Rt zzwos1_}3p<$VxUMi6T2Wh$bhw$W07+$V)zA$xj>wC`cjVDNGTHQjFr1pd_V8pfqJD zOF7C@fr?Zjk;+t|D%Ge?5;dqvEoxJTy40gS4QNOs8q}a>$Rs8+g{e$q zIy0EbEM_x@xy)le3s}e^7PEw-V?7(#$R;+kg{^F3J3H9PE_Snr zz3gK@2RO(f4s(Q~9OF1AILRqabB42=<2)C*$R#dwg{xfSIybnag{oAeI!V-^Cbg(d9qLk#`ZS;+jc800n$nEsw4f!e zXiXd1(vJ3Ypd+2=Oc%P+jqdcIC%x!RANtad{tRFsgBZ*ZhBA!dj9?_A7|j^QGLG>~ zU?P*4%oL_Fjp@u_CbO8$9Og2Q`7B@|i&)GOma>fHtY9UpSj`&NvX1p^U?ZE@%oet? zjqU7UC%f3q9`>@2{T$#Rhd9g;j&h9SoZuvYjOMhUC9P;p8`{#2_H>{lo#;##y3&pA^q?ob=uIE`(vSWOU?77S%n*h$ zjNy!6B%>J37{)S=@l0SMlbFmDrZSD`%wQ(7n9UsKGLQKzU?GcG%o3KejODCgC97D? z8rHIo^=x1xo7l`2wz7@w>|iIm*v%gHvXA{7;2?)M%n^=qjN_c(B&Rsd8P0N!^IYH} zm$=Lou5yj*+~6j+xXm5za*z8w;31EA%oCpSjAXB8S{Hr^pd&Jqg{)*Fk|?s1gJ^P+ zi`>MJhrHw?mi)v~fPxevp28HND8(pF2})9m1WHqevXrAd6{tuh5~)lTs#1;WBvFH! z)S@Q6^rAO?=u1EP zGk}2%VlYD($}omAf{~13G-DXcIL0%9iA-WLQ<%y$rZa|!^2*vmflbAW>!;xI=z$}x^}f|H!$ zG-o)=InHx|i(KL|SGdYGu5*K%+~PKOxXV56^MHpu;xSKn%Cpc*fY)DtK9%Qle*QNi z_&l%h1yb-u{+%!JWnSege3h^9b-uwj`4+G7ZN9^I`5v$HAAFx5@I!vY8~m7`@Kb)q z&-n$vKt?k0-~66G@IU-7f8$m?9LV7{w_;NlKAGY06NRa+Ie66{$obm8n8i zs!^RJYEY9})TRz~sYiVp(2zznrU^}HMsr%wl2){)4Q**hdpgjOPIRUVUFk-5deDAZhTiM2TcCeFO>}C&p*~fkkaF9bB<_JeQ z#&J$?l2e@K3}-pVc`k5~OI+p(SGmS@Zg7)Z+~y8=yOIp#IHngQ3?dd>A zI?r62tnz(58um>~>h7{eLCNJcT5F^pv#;I&HLPVF>)F6YHnEv4Y-JnU*}+bBv70^Y zWgq)Fz(Edim?IqJ7{@umNltN^Go0ld=efW|E^(PFT;&?qxxr0tahp5bVaBvJpuX*~vjPImtzCV#q^Y@)1jZ;wV5t3K36Xicpkd z6sH6wDMbRMDMMMxQJxA^q!Ni#rV3T5Ms<>?K}~8=n>y5`9`$KJLmJVTCN!lP&1peP zTG5&|w51*G=|D$1(U~rEr5oMpK~H+on?CfVAN?7?Kn5|GAq-_0!x_OyMlqT(jAb0- znZQIQF_|e$Wg63&!Axc`n>oy79`jkiLKd-@B`jqb%UQunR$y!A)*) zn>*a)9`|{`Lmu&%Cp_gD$zK27x$p%59g&$VWF;GsM3J2wM3a+TRG6s8D8DMoQhP?Az4P?|E7r5xp{Kt(E%NM))}m1UG8z82R!5vk9opVo_PWA^=n=LeD7NL1%Up@LRPX7Nfg=1K{PqZ zMQ&oqLtgR`OMc=gKtT!-PhpBslwuU81SKg&0;MTKS;|qK3RI*LiBzTvRjEdGlBhvV zYEhdy)TJKvX+T37(U>MQr5Vj>K}%ZEnl`kh9qs8rM>^4&E_9_E-RVJ3deNIc^ravD z8NfgWF_<9?Wf;R5!AM3inlX%J9OIe5L?$trDNJP=)0x3cW-*&N%w-<)S-?UTv6v++ zWf{v^!Ae%Knl-Ft9qZY^MmDjTEo@~Q+u6ZRcCnj1>}4POIlw^fMJ{ofD_rFo*SWz>ZgHDC+~pqkdB8&+@t7w(ojsg^<5b+eI2t_GIaY|5 zF`or2WD$#5!cvy8oE5BO6{}gpTGp|i4Qyl+o7uuvwy~WZ>|__a*~4D;v7ZARQI+@)TaRrX+&e1(3EC0rv)u( zMQhs7mUgtK10Cr^XS&dpZgi&yJ?TYn`p}nt^k)DA8N^_QFqB~oX9Ob|#c0MbmT`<{ z0u!0UWTr5cX-sDZGnvI~<}jCe%x3`$S;S(Nu#{yiX9X)+#cI~DmUXOW0~^`IX11`E zZER-;JK4o<_OO?I?B@UnImBU(aFk;l=L9D?#c9rPmUEov0vEZ&Wv+0QYh33BH@U@a z?r@iT+~)xgdBkI$@RVnvUk3j8%(IB_89;AjCJR}~MkG;WCkN5wBp11fArE=UM=be? zqW}ddL_CElLQ#rQoD!6z6bY233}q=tc`8tmN+eR5DpaK!)k&fTHK|2y>QI+@)TaRr zX+&e1(3EC0rv)u(MQhs7mUgtK10Cr^XS&dpZgi&yJ?TYn`p}nt^k)DA8N^_QFqB~o zX9Ob|#c0MbmT`<{0u!0UWTr5cX-sDZGnvI~<}jCe%x3`$S;S(Nu#{yiX9X)+#cI~D zmUXOW0~^`IX11`EZER-;JK4o<_OO?I?B@UnImBU(aFk;l=L9D?#c9rPmUEov0vEZ& zWv+0QYh33BH@U@a?r@iT+~)xgdBkI$@RVnvGr&(?fBq}~&-wY^h~V?Q!WT%v7x{O- z#Fu%MukcmA#@G1<-{f1o#<%$n-{pI}&VTTIe!vg;5pVEge!@@r89(P2{E}bsYktFj z@+SYqZ}}Z5NkwYXkd}0$Cj%MD#DDX9{=on6zxMQr5Vj>K}%ZEnl`kh9qs8rM>^4&E_9_E-RVJ3deNIc^ravD8NfgW zF_<9?Wf;R5!AM3inlX%J9OIe5L?$trDNJP=)0x3cW-*&N%w-<)S-?UTv6v++Wf{v^ z!Ae%Knl-Ft9qZY^MmDjTEo@~Q+u6ZRcCnj1>}4POIlw^fMJ{ofD_rFo*SWz>ZgHDC+~pqkdB8&+@t7w(=Q-_jSE$wF4L5lIx; z$w4$Z$wh8r$U|Q85vwor|1?Mc@CiU?giipUpcBB)CR7Z+18~rO1ZRNoD**i%eg*Ie zdj;_G@GF3i=oLVC{D;T?C+{7=FT&$LJpRL106w!<0Dk!okN-b>^_Rh~6uP-IV+rGJl<U8pW~mQWqz<)vsUl5iiI*zUhL(7# zB3)>SmnzbSmUyWmLuiSYDl&$ac&Q>&Xo(M8#2gOoK&8Fc-(}ly@fqoZ)S*57P7#qh zWlZfazW6y@dw1`AYI-A0Xs3sZ&r~0z4ej%A@tNv_bfJA7EE-I zdARsY^+CqaJ`We4sXoXQ+UMcoGt&n#{ZoXl%KW3HOBmM+1P}1M8ZNFE2=_s_4?@Et zd>n-PAbcEzhrws{R1lFWZOr}OJUNdzPA^9UP+NEL#pVEonEwk`C3G(3^B{IK3`7tJvM9o_ugLPpDXWf{^#3b z|90hNn!mFdfBWHrHNAMAp^g6FtzRq`u6>N>uW)CFJNwh}&MURkp-+W;GKvDKP$&>MC!Dmch;c;^ zNZ+acgPR_Du;v}!d9c4*eX)x7-h}tQ^}{apV)_5$T8h-^Q%3)3>+`idug~81ydIG{ z{fjPor}|=JLifM@iu(VnhHi+@?a@+2ci;b)i+s0d&^lE9WnJ^s(Nzw-v+@u3ay@wb zcOLA!J^#A;Vza`v@QLnU?L}Hd>I^SN*ZWjo9KZkQQW;}Q{ltI%DN<)l8GZWv^T&I4 zJtB3+KX1(Q`$9iYANc3$+m}vLXZ$;rnCzoos`)!7mUmYCu5D0#`^x9LDu3R#Zd>vB)Qwf>MgQ-+N1o^u-U)0D}Rhi=XE z?pwe9?TCz_sim$PE8c%L>PyXgeBLQl|Wn5qhW(I`#EMyO)~(`n>T=&3}E~ z_@(B*K5zU|^IxAgeyRDd&l|tY{MXP&Ak#emAmqneI_21>;o`H_2k%UXb-uCp!^LNy z55n_dKU4EzpYQGuJ0CV8ZK{}zKL}5&b&mY7UHS1nz(+D6HX>cBnC(CR*e2ITq)QXi z{+A!y>OO@l`sk*_Mx;*_Q}MSS+Z@}7^l4(=Ncpj??$foRk9bmSM21u`^}iaPYU_Oe z5%=sz`FQ^qvtlDMKHu~I7Y`kt#p`hT&>b6|7W=OrM&=xh;@_W4ZQzpAE4 zn?7xH>-e`%E$_4lA2i{E=I{O&6Os1$MBMji^R9c|xBB8Rd*7y^2JyLHEby;ci%6UC z&s+I7tFg8O{6lpW2jxO8x`4f6x@169#{+AE%*Gv87y?=kDAG)vpv6Wfs?N7x0 zf0YltMd1espP=63Ja${7D&Ex3 zer$aokuFor@k!5D_cyC=Uom~>*@POJq8ywJ->3TGiTq`7Aa5NaoFY^=O;@J{KPy!M8>o+r?Y(c38x?M2}fj1A5%Z_V_V(Jt>|SYXnqR5 zcs3$a+R#TiU*=hdez}uAw7@%+53snl{=55EEHZ?)DqMv7AbcEzhe7yM@L4<)Mr246 zQ|c!{`9a6PKfcTdUBmNsFSk+eS|mk=G-;#%^y`;eS-A11ar}qg*ZE%C(7=9IHN3As z75h3OL%O#=pZi%naeoR9;)o3C-~NC_cn5~xgM{CMd zFSks5?hm|yDKe%_7hSC5^DTN_FI)1(*1c=Z?n@v2{cZ0&l@Go_FCOk+xE7Hy^f|cC z!r}feyc5Ek_UV4r5&Ha%FXjxL{NB47J^()52S9lLhWGCWeChM4I#k}wm?=&AKhIPB zR+UP@Jk`*x@629J?F&^ihQ3ubQ_Q!M`QC%~{g&0xEY26-Ci<`TEupSqeoXUWGX?Wu zKSccN%!dt4Q_b{znrgWGxO(7IG)eZ8J74vmFwyqYG)4BKoJ9H&_e(_T3^BvM^{=vl z;hCf#-xte9q{$HaLe_tJmxsPYYV(giOS41&>9b}-f5O~1vVL6K5s}s(x%uT^uP$^! zFCtyan6keL&vE@2=lMpY%MkiX)_>Nt;&*0)hW_45vG0e<)!z(Hcy$20(=-2fwTSc? zLbLWihBf{5I^JW(XGDhQyZ$}y@#a*2go{slAN;LppAi|K@Aq)Cmp}c@Z3_KHGSg?bh`Co&QAO3YZnjy zFej_F()oYr?#pbS9Fnhx^@s%R$R7^;!U#Cpo>ZyM0j?6_;R*o-| zy+E1liS^T`&XplcVv+ncs${58x=_CKv3aYdi~pWRHKOA3MaNgKkUwLZgbcYeSIbtf zTvYa)p<1?lRmx<|le$pBH!@|Ylciwo z`dKP}SBp0*M5oW0I(N>xwJ>TJ2A(j+yk5W2pcDPQfJ`EthROr5rT=D5N+ zYL!m?HMdsERX0P!ROM5>kuFuuaw%hzzT~#p@|Duoto7ELiK+91R$IA3T#>}mwbN#K zDnZrWzJAGdwh;mnX-kRMApbO>GGCK z%9XBG!dp4BM7>ocL&{1qKXXT#?4=7*ai1yS#;CQdZ5Bm?>ja#rU+Tv)0X=vu=UdGH;cAGjnLO%T%tMu49wEHr|$23ftHK}T~{EqHE`^QPp;3s2%x6s&uJitE8%z zuSmwkl-Wbu6IY~qzIri{k$K*z^k(UV_@vTFm8zvo{bt3wbt-0iGb*-Vj+|BB_>Oj2 zqjKa(s2Wu>e`I8Vs_D`cOn5V8-fDTu{K#EzG|b(gey$98(&R~*zrb6O)#GXw&QPIX zo;Rvi%bhX4R`$@2Pb!qPbiE31q{~*JOzC`it0ZOrapCKo5~?RuC{r#?xhh%T$`Vtn zR>8V)xyt0Lk}h3=r1XxG&?85uOIa?lc8xj(Qss}2tzWBZ*{HYP%wO)!d_`*M+Qb4s z&?KsQq3YQy)-9Wu{;dM#3)M-?T&qk{#X@Bx6SG&zkn<}VR7r}imn9)OWnAjSvNh9v z%Pj>9=5J6nbDHv{>m|e#sFS&B%v;q9XM3=`P}ywN-+UumLawrJ=E@#lJ1R@w@)<+j zmpfC%9NB8d)yflHr$*+tGF3>-laM=Wo^+W@CvTBdr(Tw7@frUgM{gc1yPn^L%`s*2 z-Yh*ccV_JIt`ol*KMKXDwTpTWgLvO zMqShf56?-|)3fX0PTh{y%?x%;W!cuJ*EiE?v9H~Iz+q44=YG`&*Bx(Q ziJl2tv-lA$k*D7Yr%=mLFmsJCAn2Rx*{DpA7E5g}iU-_1oS&XU-=7X>uj|Z?#5{Wx z0*)ukdLU)Z&zYzCE8C#VFO=&!-961qp`ZWN^;Hmdg!@fq4xZ$XI^MvdM~>M(KL&+x zzh?8TdiUuH96p;`mcXxS4m)GuV;tAIH_L9>qgV#nn8Lh(So^0QAYQ3K$Zwo*f(tb{ zI$aZ;MWN50N;%hZZZR$7GM{4*g{j)!?-JnL0(?iw!PnNGP(hr+QRNoHnVKI0XY1@8 zYECJ=zwm3LxcQoBGW5^BX?!FNhHaz`^a!1E7fl3N=eMbz$@MvFerSCwNe1!unnY<> zP5qQl1eYPGktrS~9OF4jnj;?xXz|oQ(vfbDf}xdeITvI1rS&EH!s;}Yrf>({03ed9 z3;+9nnZE66!~laRKIdY5rzP>|u=-@E<~tlG6meTKWb4b-uyNLFy+A`G=CfgXPR zZ|%*|Yi|o&DS2Mv$M`f`(tcmZ=ce}TKD2f3QrXn}bS_T80-<-^VdRJ3uOzezllwS4 zT)e|YI{HUKdKmFs>+#;}q2@kf8f4 zXu(gP6#~_3Q=PVKMPyq<4&^$vgj`3&l*4w{Gf%`6U&~7-Ar=i3L-;3kT?Z|0xe*P; zxi-r4wFV>e!=HaA>A*B#YRWu4{KPj5k{3z@&+9|D(8VxTX4_F~b(h*zam19+zy12= zw+k2;NBxP7rf7F}T6jV)9FIVIh312&b2eIFA@ZlF6-qW!afzB{eX;`ivZHAR^`fvo zBr^%)Pc67v8tzQ2hrFQ~CI_;I_kZp5z`!anU5l`Q5h)+J6ysr~#(EWNEL}sl4GQ57 z8Drj4M?J#8H9v9MxfGpSiypTIt+WXCbRERum^jdQ_N#I2@{vMZGTxjaNiShoxN))W z4bgp;^{28W{hMHJ71KWWo_AQ!s^oXwQ`w#qG&u)%D2_2XxzSKVRa$e8(@;M~YFTOS z=hh!WO4c%XJ38L?ZTNI*g`8;$PMnRq=o#;_U)Qj}W<1r#za<&?_P>fp1}^{J@l)ax-fiM6BQ+!K`@%A;AMtrbqUwS0<6 zr;4R{>eWZZlM@e!0%1R%PEXDERPV{!^t23~68VfHGk)fmYgOIozx(D-jM%IJ(0ul) z0U#wb%NCkzK$Gk_2%on1C_U85UVy*8zK_*=@iwTic3%Pu5Z=4m#{7<$d*zIsh0-H$ z#{uY`nkNfR-rUb5f>(ZFjaMa0$~0d9DjaRPCuX81+9x!n7k+xCu0`O{r_4ts@8(F( zo?<;{yOk0G+0)!>@k1(G%i7q>OfFysBwxM7fTzGefMFzJJ&`xZEmH>W92iw95Z(j& zM#889-}My-IuLNcI@e|!h1RR-{ypn&tu$CfGiiGXnQI;wd-vY(XAHC?ad^T~^-fsK za2C3pDnIk7@D~6@mfg^xxinP&7FOMDJiLik$8GJIu04(6k5B(=h?hNV-|tf=%eC8~TL24&ja|WOC$YDoZ zF22RVz)XPKwKbL2`SIg|J_##`maJ~Xd|&VZLDz=vT1D<5d;a1Zc8(Jre)s%n^rD;H z?IE~;h9zd3xigEy_VnrtBzxnRt&dr#ol?Oaef;%Au9?s+VO{9hb56)Dkwp!4WcJ?I zzV_=sC%AnHboA)Q(Sb@_Q_p4FLip%O})}A zSkZ`tLAdmtc=#^;Kyy+Pj8%ZaxO3Z%ZQX~T^%FRD~ z`;D}=mKw-F*6o*L2AB8kso&Zr!YOvPO*kjlVC@OUkjcOjBS=cA5Ox;@hE0W_cA38u z=IA~yl%IVu{LkJ<=mn6rbAD=%@=dz|rzgA;P&j$MEDWjD7=cz!*N*o9!Kfz^aHc8F zhwh8$>~*>cCH)8|kpd2e{J48#PCRC)^|l?(u^xs^pkxE!SVREJyoixAg%^1ZX!P-^ z|Dl)m5POL35>#meqDLK3Dhft_Q3;`!O>Z7c*r-1L@RO(!yYReiSyJo{Joj+=CVr(^ z@3Ji*P?0rbnCLBs9x2)uwpv6hg^c+6nKH)ob2I=#C(szO=;UAWEM$J!+M)Q)&0Cvj zDHl{RhrdW|Mc-fZZ^HFS->kBZN&%)Pr_v#TDT~NpG>pI6Xz98Rp3w%*>nKF zfSAuGUOosRj!e&g0R(Wf(J1hY5bXkpv(6Jdi=ciGJ&Ssz)()9i$~dHpe<`$?$*uhGYe4fQQwX>nDt3 zGVX+PO@iUE03vWP7x3#p`*NOjMrip~wnKiv6ogIGy$*yIbnYlRQbGDmj(UEly^lMe z53^{kNejzrlG}k)tD}I{6AF#w04vCY{_sn0u7bKAAJxL!h0=lbCb_31P$!Wt_k)rT zEFkP)kI3u_NU^Be3iQ+5yo}13iyfQkpz$a?AqdkOp<}pB6O(Prjv`eIQUv?3XL0$u zmktN)BBREy!Y_YM=ZHuR1*tHj!z?^NzFAM?mf7_ z*}JASVCC_j6m)%C`xgMZ0|Y2)S2>VeCevI=rT_fj)wxU+S}J4|2u2QiJvyZ)ThFKW zj9pc4`@?-Y(06M%PvLicz+yeRNbyR7FFE5Tha7kN{Lu3o>vJ!}ts(NlT-2@tafx7_ z2Yf^`Ws5>GHglopWO@6Q@OLPC74niMk96NW!FGpC%#1w$k6%c{wzpz2E9gTu$Y;TZ zlp9w}_BLBuVM=_MPaH4&yKf|BSDhOo7|m+A*6{`1tL55Y2Pbo7Ou__b2R+SwrFMci z7T~tu9m*TGWw;w;g)bBQQgZH2O zchFs#!^s_Y(#@&M$N%)B{Bp!nzqHbQ$?@!#V0!wjEEsr-k8*4%`^Wr)!!jRkUOWjW z6Hkox93j{NIAs|9e|-8q6{8g#X_$B6Z ztvrkefxq~4@LrmX$(}!|Xh&P%Nv7%l*&ei>2M)DUVBm;D%NQZD+|Ca_OosDZ}^XMI6*8R-gt+Dvk|(q_qB&AKE84mCIu*BYLe)9cg~M@iN1> z%N!3D?<7jY^zlt}yCMREQ*q)U=Hz&Fz31g{6TK~-@oj6D?cfIi*KJ{U?RC$81jGyT zcH&t~)GPVb_q?Nd!ofog0Qly*N|Zm7cn5t#9+L$e=o>D&{$;Bq4cf5_Rr(J?Xsd2z=QfYQ5wRb>2$7Q>%pNWjci6!n|AYu#Mz&rc z`dW5ZBkz!;5+48HJK62DTNRV@Vn3AR9qs)95%3k=9y2i5{EfFhBtyhivU}~!DBHQ^ z-mR|tHrq?bhhO;!sr*LcxZ_SlR;|pt>cirTE~Y9+1k}`^eE9Pyub~H%Cu+L55ghya zm);j4tF*3{+mv+Pr0To)q4+RMXhY#K$H$BDKJ9$5thjAc?6OaO&!l?7Hts^$4A}}( zHRyr>G+H~;fO4_a($9bB8)=hmMM^msM+$Dc`e}}vNdaGg*3$2k$KbD|kH>MHY`eQz z?9K;}W4K+=RaUcYPCF?AGN}W4;A7+69q_P=4o=Tq$c1HPZe`8{~ku zm3KBj@tq%gQ zSYB%H(OWUBaz0F39}yU^6_2UUrbhr>Opnv&#I1jg;;Ir5%!?ZD;E;0nL2An2An zy!_Di=vXEED@ZjNUiOIOLQ%SN+SI zAOB7<1$OoYU*%78_D(0+1#0Q}pMD3kOi@&N*RkloC~Dj5Fl)g&zXu>vN#<60uODBg z-4!YKQR2)zuY@(k`=B&DXd3-iu5*rsvqzyt)jUJ$_wJ}w-IDj_Fxpj6f1&)4j8;ZK9cjE3Xcy2)a_JpSSv4Kj9D zvSX#pm)PRM-_^@KI$%|tZJATJ!t`w&@iFzpwx(NDOS7BV`*=ItjoYewv@_J5WfVJ) zHCooHG9Px2KP8GY!!1rdPVOPH(5@FWD=g8|u8YSDB?Z7u-EAL#<6k3a$y>IqM0fdc za@Coo6j{_HDe;i08qdV#gdd({fV*bovbjo-^5f>uec>Y===ZsFl&i^V+Q=$*V!}E{ zK8xL)!>k+oa7Fdw-sSgQ(O@JPcaM+1$_(r18>%b5w_@XtsjKHDl)}EjeP244ak7h_ zL`T6097)52s7q{VjvN~eDDCk#{c0nYl(_fT$2-K>q0P=|WYW%9`D*0ulxurI?#-cn za)f&8#B+!~)jGG|2Q5a>KmSh9b#UEZ0zaqgmt=SuD7H9&79Xul3b~z9f?sCK7Ao+( zJwbrL)ge)S@^_#KCw#m*L6?R(HP0VD?7UsY2ZzKfXv4?52}Nsu8-cF=kiT=pRCd~; z;`vJd-(_;3{lnh8Q}lA|&xC1vj&j=+aO4AF^?SgSMSf;ZxNyf(5hT%LX3TW{XW_U2)Wi+mlynsf4tvzBUyZ4B#dNdp2NXVjBG zR;bu*ajq8y#g|AG=W1`R)a`2<;S0a^ZQp)9=#vWQA)gI`UZI@eOL7ij{dwLI$sPBO zZKnb2n<#b2?{Hod@KP7=XM6@!zTTO)JEQuyLmxbe*d}(tVhBy{#BEpJ&rq^A!mOz5 zi3hJz7c=yewUn(uCjtPEARV98hK16e)P>ex4ht1Fjp*FRC@QiRadl(0(vl~gyWB%N zcM8Mx1ox-LG&K`AxF^xT+^k1#%N8+XaF<-3xOBEU#urIk0+eHwoa7Qab|^5 zQ>{F?oty$HH0r?Ov|@VdTMnnK;Klw5 znAi+WyZE?fFin^fSo?2QQUsc9d+evoyJT$g-DT(L)**NsZYShiR7YsLFhq2@ z8|T<~a;}i5B$g-J=@YX#Oc?v2o3oyU6(9Gp%hU8FJ_2*5omkE{Q-^v|B>j74!k6+is;ff1_k8T1BYy%5Bh~`YvU2{4yEs>znGD0@E*JC{<=W;td$&?CVxxjWsW&j*A z5?OOs&~Lmjf9eKjC%r@TMMlG<#Mn7I<2)vvpYG2fUT$2SOD-zRtBHQ`IP+T{&_-ao zW?B-8F`Z;g#HTMcw)WTkr4{Uc9Cl9^XAJ<8(|5pxPV&e*Obc9g@FChUz}XqD=$2cB zi0NAuP@~5QIYer4-0@oRbYCx2wqK>nI)V;RA|42d_?B*stagolvK6H1FImln7KU35 z@$|%Q_bH98JKa0cBrVOTrKUS zm;LQd*)v-t==Jq)BH4_S%bn97QvKYsz>we0(Rfiz z#Yijn_1}Ka+LhAEK(t!WWOuj=pM1y5jIHcKw^?G(Zaw!b6w&Ht>+$xU^iiyXyX7KG zgvGut-=R|8ceNJ19H0I5@BU^V$yRI9JRW-Y^QG!6F7~X-0h=VPgLO_W1u8Ib-+k`Y z-vsf2Kh>HwkY3O{{-A>I)RefdB8?w_N3&XkBS+`i-}^k2Fk=kdr88PMVfyfkQ)XaF zi^}(P2uX;`9g0gV0Tw>_eJ-80muKJq`H#-&1>vIB(y%2#M%+C%J9t}6%!2bSe`T$Q zRymRPj=I9OdP{SqKG+96ouB>8SH8&s@WEeA$EMcucEP8#inrD_-Pb1~Vr^g zTj>j%-}2S9y7Wprkt+;wVR%n(z_(up`tc%e0dk<#)7siIdJ>O1=rL`{H{FSo+Zwon z&Q~_S{AuM%#^=`YfGg%)?CsIHJNwh-gSWhR=al9uv@eFg3yvorr~K8Bh9CZomw>99 zeN08ustmQpy;?yhepdbD3?V>gYg`drnw$2y&A;}&psY1u2FGL#veV{|fA(cgP-44i zt7hQ4;^7E|cD94(1h|eo(1TEQLgaY`?$7?|m!K8Vrx|Nx}(M65gEl|J-rH1?jre==yaXSMk|M+v?2m+FEK;*PFC!*Si4LX8V8?ZsN^^ck$ zi`2s(f{If)U`Ox3j>3Ka%=^j*aE1~qVJ7)IkSkY#wDZEssvPO@R4GrUY9o&xE3g*MWs-x)0>UR5s0ARoJz)l`M6XG%-KiXsft;t7;Q z+^xBajBDa*hel0ZiuTtw)|cPBe^8`jcv~eT6GP}x7W8x?*5H%tvV}g2+Z0M?=Bc6L zihY*c+th6xs(|=|fz((^58(Gbd~fOX!~U{AlL!;DnnOgRLySA@b}7Q;viTq0wolHP zj$ls7v}-yG^0-Pp5pjC>vu_mN7>84pxo|{pv9qMQKrF>TZf(0$algj&){wY!{xl#C z)(MA0Ud7f7MrG2cP(*k9s*-f_!L!C%-yJ`*`77TIsPl>(osjpXk6yqGy}iQ{XO;Ob zno~57xfPUWyrnn?aJM}MiG61t{{uAHD;GTGEZpSk!FkM3YHRd6!37frn&SM~mp@aq z^GO}0C7n(7fb~~{VQ&XIA?8C^KEULyB}dZ$LQL_zwCf(eg!-&=Ka$S56R$j}Jo}4Z z`Vh@C!&^eahDdpyvb~S-2YbBq!{+_ZltZ78MSC%EL`PR5Mjg^MS^^}Riqp>K_k8Zf zai@Vf!$kvdnl4+F1`e8CbI={ABXx*WK8d!`_iU)mYUWH%in*8)!}YM#d=A0an$=43VbeyzY@mATDF70Rs`&3q9_tI)gf|M-L4CClS&E%rg~w^itx zEGM)#%H8~xFLGB!-}m)^j#>=Fmas79=EuJLzSLjP?pzR!dInEBHel(!ol6KWiS!a5 zLl-P_ZY1$7D}=vPq{^yT=LLXF04npcxU0xx$$1n-8xQ~V-7ztiVa#RWY;LY!eZgL? zk*?d|DJQlQA(6%k@n@+M=OgT}ICsmunh73XlYI&}fif2$He23pxA}*kgFNbX&+9}x zwnGhv|MH`LC>3k{YnC^2H*4^<=IdF&^K<^RUN^6Icbck5c8Rc z@_+Eu?sC^QXW10q@&L$n6mT*%ef#0zHzjJ24_3zwrJQU8UzesnQ1t&`la68cYB%eBTL8qJ6Zz~7d@c13zN0&hi;bE z=FzpzWpKHi8lEWR;gUwdTmpTEp09_qqw&`(R)YTVy7{SRiOXMZo(5%ye9)TyfasV2 zl&R410E#|~iyJ0BjuWbJQ*A)VTfCMCi94>QVpD#GmTsE68 z)kM{8*JRNa;lll8tDy?VjpAwpW%xDLHeUq#sHp~PQs6EEw&&`w_^`Mv?!iZTF-FQ) z@4sWU9XXptyWAdr_09Wxlf1hX+pj*+K1ZsHS@+PINs#&K$uuJgOwdBD1mqpJuarX* z1k%_FM;-2a<75pQRX+Sg9ftQI^5)eFAC?)en7Dutwn727Ubg%8)?iaU07D_!$U999Y_n~NSBqhV>26Ydz+UYekx%! z76@i&ERc3RAzbc2sL*jXB%q+?~jg{H#%X?m+nGA$*tDa`St`) z9SkaCZz!VsG+a7Lg*!}i)H^|*589ri$g1xw30^hv-=vsl$3wAoV9S&|hU1h*R+s$nIn%@nDpU~C1o=yOBjaNHilHJg3(iYC7OkYjw`nouRDH)i))mIu5~u_ zf=nA;f5)32+3OuJ=S-WNnyw2m+2^%mW?u@?$Na4`=fdeWLf^^=Um>##P39tW5>Hcm zoDhRs_)01buhPSNm7?ej!Tlk^TPHYZ+B6>1`ICPF?m$~TDQ#VDK~PnCgT>6oHm6Dq-;-Tdfhl{r?+Ji8thvRyTwG1nL(d-$qiwmEk# z6-cLV+(q><)g*+6Z-TveI!9sJoUEw~u@~K$Q&(UTn$EUuRJzk;yN(*Y!~_a6fd~>| z)Bb_Yzw+fC5@IS4rCnnX;653m3Es;^e7gZYR!8i^pZU-^5ou=M?He6)E-zgzdFcQj}GUJ0KtSiU!xU>s2yHhW|yc za%y%=Tg#aUFgft~=w_ZkUe9h$_;vZ}cYouauO2AwlUvb`>Fa&PO~?X)c`H(SL5T=l zzWQzNXPAXdJ`b7R`IIox?)vaSf0^jh4gzwwuaASHtB^(rF;LQUQ=MVu;U_=V50TE^ zMq+zhPh@lHEMqy}G5yNJTt+go*{$B!pZW9~QzN)a8>2cWyP`!YVTP{!etmkUT)S(O z1;>DzPw35mDk;Fl6ZYn!x5XNfYX(^~KZ#t~z># z^n;rp{>sPa^;zpUL+z$MGyy43jYvG8ItKIC)Be8Kt?~z6oqD<}a+%Ax5^_(o+c{6N z)DB-SYUxfpj{bunZ?qZW?dV%*?89GKFYi!Dp>pEOF#z0@1XlTF((d?RZT6pesly#w zp||zwQ>ukXKRQ*cpTl`mdOFNQNZ!-t7MmX1kHs~R4A*WEI!lcS(=Kpil{+pxzQxp2 zY|Kbsee#FN#oG^Y#*I$1iZ!Xb>EA+iI$eF!Ok{V{sl2@YLy4mk?%_~WXn()X2;T-< z^)Ao~5S7pV$FICJnAx=A0e8++I3EK8tBfO#6)Z8ILef96z zrli+;v(sn3Doo2-)$3$eH~cSuOhU~4x?OfUECJ4~YH$8k5w7Q?*Y|XpkAbktxyc); zbP?L`s8Yycd%77KDsD|@wN{PwmRoHQ9JS*US3tGhI$|2R&Wx3qE0SDXBZ~9@( zGP{na?|J>BiRQMYi0zEV?iL)QA)WAIjo7gezxnyMKS+fF^G%Q)b&glqMLFXuH&|ri z@!MbCgJrk(r-ObFk<6{n`9(gJH(Vjawz_d{c`7a@?SF3b58egIbDe64C6(L?<@J~M zyNeySZWy9nc5*_cy~g?A7Pq!>D>_)j&WHuWgz2Ao7aob0KKpX2%MX9(1AI1;Jatlf z+B^N;mV7}~-MgW@`h^#sh_$>~-A(c~TVkK%i+ETQYdW9p*L-9%wuVUV9)9{wx~SO> z3-SfLH&K8Iqsm42ea*%-6~F4<>x8+2=5haY^cV}u&MqEGkxz_A5D?;}TpRm2g-(%n z&p_a2t8%M83B33!Rr*zgw$;rQgk|p_A}@EZTo(mR#P%J2DUONjnVdvP3 z=;mvbI7eF132(ag$$xG;`$h^cw+qobkfn?4aB3xAd1SbC1K}th&G~-XRlZYK$L5pI z^v1z8CV1%SMk$?sSrtkI)iqnc0g0?`AG&&LGa$~fc2bY+1hXX1G~vtD z5g+l8u3SS)Ld(Bi({{|#^JVjGU+PnRT@`Aio0P2FK*H|llYtVTwy%+Qck^+6$9qjyR1tccPx$?Xb6{3*g32}0luRcEarkOk}lQSI!7RisR z%FeJDb=^Py?H9=m)2T55-gxAc4pvH-Pw8T$9c0CM>4M0sHt_=wzrDA1J}*%cge z8Adw?t{TN8Nfg}%>HPeyH;PGX@D>Sg4uJ^SO+F9%d^?m?m-*G+p^JDBpU$<+AqjS= z1A}z|;gX2}d30A&yY9CEgWZO%x`r64WPJR5ZN7T3J8tEMX~Na= z&7CrBT+WrI^opsWn_v3Odq}$12ou}w?+_oG_4bnvjod4MY?B1q2KX&{{#`F!B3`Y; zMmN=i&99*?)Ncv(oWcRDD(l+;4y0r#mWy^Mk2c(-leL zF3)!sJ>(7;#3#Rf{P{P6;_K6Gve8Z#lN)~YPl;n3K>=H3x~KyG`@ivhS4JUYobtC}3S<(MgFz zh&GEsUnh?sN)vFRG5eBTQXF!h|2?pr&s&pY`$MjNeuMu2Y5_JR$?MjRL?4so^?&s7 ztmay$9r2(RY!hFXm>Mez)(trfY3b1v#sy&?ek&O*+(qPU2^i{=o_G&`0=$au$lLvT z&v){(-|?N(I`FD=Wc3{GVr{h^6$%kdxv?hkniS*q)xUgkNI+?@eOn9GA~VDrK@Y>c zMue1|J+>IHqU$0Qx7mL7LzOZ8J!xa>v=eT5dR$k#L@3yuvIpS>R+{%!8}0Pkba!KW zZ4L*(It5I40>i@^byGa0yTAT-K3Er5N>5qE2`NP{!`{A4Y$LC>Ncew>rbL@(j$0FR z&c0k@0dR4A^{d~|I;D*axyu=>zDc}G1~nOHb<*8SX4t0q_0fH|SYS9h-~AmLKgeQH zZ$XM$8y$8EBYjQd9EVr`idbHp#4Ar~!VBZd_gx?U{O0d{p5%LJ>7W6zQ4#mApWVFp z&F>DfJRLN?g=C8Uu@`=iQoG&trmqYXnY9c&1H2pH3wEr^qp{IiZv8@ioAoqoSg#`ubHEDnuc_qZYwV#M~)Eaebzhb;gUT5 zk#|if;9JuNK+IQ z3Ojw3>{R2B6}On?1zanbw&;*`Bk=KGlGH9I_VsFR>KBVYjSt=jG)zP~Em?HonE<6+ z`D#D&w^K+5D2REtG(y*mwiFfp=1+d@Q_;gJniEJh^HkW?11(C2)ocni8xM2}-@^dt z6}L}**gH>3PssAxPfA*n`?Gk1}N;y~7=HL%Z zn)m0p7V_8s);nb?Df|H|bR+!9Z=4{CxN9b##3Wz{B{u>Ua52qRu>gHsD#d&F+8fjM zE))hXP=C4Jg&?Vgj70bXqXZco$W_L)p0>1t*lL%-T!olj9ZzdeK?IrBXAYwXJsHU- z-_2sYT^Bko6FgB5Kl5HJX#Q{UqZ8qA~pDIyQ8}L_zy~nwoKCBw1ggzyA**+VV9j)KzpOyagW9a+;I=ql6W^{ z04#45)v$RHuQc+1`QCS|HJ8In_3B(blb(HujyHI@zzh$&%?>$22NBfa)k{G)|KVHT zsKPCQ-joE+Ec_;ir7qcv5T!D>!}f zb;Yk-2$k?k%v>|vr__r4g2&%j7JF)5a<^oAo$>ehD7X6cvb#+S+9=Pz^zI%J))mhF z$Rgh6-+rrKRCqnf01s|;`wFfp6Z&k3n!FnQt{fuaiw~GlC~1Ygrd_{VuKpYf+Iqw- zjJH3!z#zLGcDGrT5}O$oC&1U>1FUD9xo#ss+sEm{uEMEpy|{u0IjJwv7Etb#1B}TJ ze^t~Cg3IxoSgK2M#UyaRqGurv*7@vQI%MnIn)$K+m^GVAEy?a}yGC8IJ1{Ex_(wk?NqwXcoc z8a$nvRZbP@`?}*S2W~yAyTn(CzsObLZ!66bHvi)D$*A6?gi8pE!Ot;qjx2GH^+rtW zh==b__%&--4h{mjWqCjnx~lS8I8Ry!KK#yUlCP`uGc@d~;i?VJL-w>pt>#|*GwTD; zC)2xB2q~7OajP))?2kYIZ?@b{qb1?lPm#J#Njq%|$U~EPeFvT?%ZaO3zwKqWiVT9Z z4bBJJP*)#PyYVKJ!aKqAq;u3P1j!Nw0BnBcYtw8JKF>{Pz3-h)qtS~mQR;SvfaVIX zg4O2dzMxm|eJido%|EVE+09}H&hPFI`5U2{57l*X92;Rt)~Zwy$5%vwq^g(qsJod^$2>iWKN;T^X2z2k}-1okPHA2jsgSBq6JYsXx!cUy$(*s z9NNBma~$UO6xxa0y|Gne%fNcUSCL-2-Ms_-ogWDiFgl#0`u@0z=`yi?dBtG!tvcWB zlebc7`k#MTcc%>2rC#?As>B*rrU?1AxdG9`_kHLxYI}+Ngh+kOpH?NmlFfA8k-NVB z{U@QVUmL6gW)Qr@!uLBpG@d z0pAH(<uy%owgt(dI{yPnD0KfG45KU9*n7sE#-O+b^IVb<}Q6U##Df(m@M42tjlq zLD<;xQ@(gftCGjhZhjqn*MJi*laC9w8X-UTp?E}{Ai5?!79ZU9*n@_6&4}H#p-)q} zvz=XH_(#1j$8P4m)ehOmYSw0;NTkH$H$JAsi0cx;D7~#a1!3Et1HrYgT(;c@Cz^?n zN~gU}q1C){O}?L-KDMhk+5X_B*?deM{gF0UcC{P1G#|Fu8uePq+o34d;#r$r>U`%X zU$>JRu|UprpV5)p;#i0=H>K&G{~t=BwZPig4T#RC%iH7;;QMm86nNQn5Yul^wl^i&tIN zdsfkI7SvtQ@!;yZ9Cq=F_`t&PX=M@d;i{~hUBo^f4$6YMpX)jOS9hOt@BQ7XuxtDywxh-GEUb-FaBqHgye>2Wc)zX7tl9JnKuSsmgeo=+>qAk#NsU6H2?P7LTk(?$swl zbYrLVKttTT#oVI|-H)ujIyt!VSBY7Ym~g6x`D{eK^u}tjY14VND6EPVE{r$6|CYCT zHO*$|`e;~u_2}~--TAwDbnB-|j|zO>Uu$l^U9nt|I???9bJvLvMT3 z7Q-$Z{n=N%_5ON*;y4!tTluo8n`S>vl|woDuf9rvvR6AlOg8XRXSICI6kQj-@n}XL zQ&o!lfm7jO&`s4+pG+z6(5UNfG06CZKi*w*n->vTIofA`nv`^CY>&e&V*VpKKd_~& zav7a2yStCP4uv;05B2F;IgfwmXCBYQ?kod8BoQZ}uGrK>xBB^U>(6EvR5+QN1u-E{ z%$sjt5C}~cjuga7wJY|gv%IqW8MP!v$cNwX^s_5_bu3yol!hP&^)Fx6rznyqpot8i zMiyz`frNA8$6x$d$uli>%&!Hv!#!%-X9lXMq(`fu^quCTpM2@{RO0aS;h-kU;qKes zh_6CZcx!Kdy)fhgB!br-ZFlJ8SZSS~Z<<6hjbSF#4Jkfz!r&oO?HDFedg5UGJwkg6HD3s-f_p76rjzIzBl z#M+weDL7jmQTlWU(EHb4Z$+@(oBi^#j_zN7Ln6EQV(o%cNTMs@l#TW;JciIGwSa7F zvO<;Cy$CS|)!nw8dY%@<6nNCvoEpZzc3CS`;G3e*5T^tzG>)oS0=K01_m9z3 z?z$4$t~SGYsRS1xSr2Q8pUmcnyuoYW8|=02?jOD$g%`-WMSEhhefEhGfKtnn)fr}S zan2Hlma;sr2-~^|t9d{A%P+oT^(;2oSTkz6MOZ)7dcLxhxp1(yMbH%4i8#C!k^Q7Pe*8zLNit^u>22kwT=txeh8V*GJsjA|ZWLCj5S1!cWG%uI_L8)YReda$~xk7OlHQd$c!* zGn%|8^C3s%5mw(gy3slE#BVVpv=iIE3`cfZOxYBo!ZAP=ni-UQOlRoEKm0(y<`^sO zuk#yY1U<71mVUsj?|lg!J9Kk_!7r)C6Q?^!dk6pSZO zo{5~9sPFFn6TLNkL&rA=Nh@1lXqGDB+SK*RnQOlF(&d zd6#3fmCvz_?bx%Bos?81ME#O-oWu(SIkwC3U;X$Z53QSbjD3R3Njk#j++rLM?`YQw zDO3;)aR399+j6poHAK&rAAh3%>wL2i&l-=auo-1+6kdwUuOuUL4 z|MAu9OG>3xI8iiRIC?mKMH{$er0~CCbsK%?!{sJtC`2;~+Y7o9@P{_ici0rfPldf& zU;0;%SDUrJIJDEA>*aXSQ<6a+6CXJ@j&ojCr-LGaYkMa(uDmd8B%C8VDA7;AGda75 zv}61%jON_)~04ZG2 ztBaQHhtq;nTfxhxpohW>k!3AWWeG=@j#&87xAOyJj@R&q;jjCvf3C~rHct5N^%S?(O}zUHkfbn6)-WO1E5a7KQJnN&SGk@2N3_ggT<{FV(+ zo>&7;?5JnyG!No5iqAKWryu_Q58fzNoF%Mg$`OZPgLFYq?qXQa6L)C4NIUPkocog+ z(R9yrm=u#W0!f=`gg{lEy1e%Of9vU{3pvTn=9YSn5%<0@$HYY(+;NYZC4EpY4MQ4*2a0QQjx1U{Y%U9j|Ws}dRiW=udVBP>VVi~(;H<`AzrWX|FE8}1N#)M%f z^YcV>2x6OVm5o6JF(bDfJg0WgzC#EYtpzBW&3GBBtOIa@z3`KEco;f$Vj@$>taE#+ zJeO0hGVgvx?&J_GwveuD&WZB8Cw7APrC*_q?9?*}U`vVvP_d@WpIvM*rj0i2QA{XK zVJ>l!KK$qdbLD$JrckRR^T11XG1njEZ4z-JBP8TP^cA0aGDB(6yjkXEpLUAn zQgGOX5cfg1LW-Bns!ozp7+m-LFFYcrxB?jrtmL?&!Kjj-(ggoM$Jq zUS0f>Jm6yHyDyLa@0TA`H`U>d{rEe-Whbop0Q8gA zW3@G+o%h{FTu`weXxaT^y>YIowh89CQw|7!!h|j4c}D;Gg;LwIkPbJUYt2tc$u+e- zq%zdlsv;;%Jp6uohj^Em*JeR=j3WK@(OX~t1dM~MV#DU_GeRg#k1V;?w=AE=xgzps zm25Vf6?Mc%Kk>@rXdRa~1cb0NNccysn@C%Fu{UD+=w^U`K`}_byf*s$*Vqs%bdHyj z>8I?`9(?P0nhxGxlTW<0K3u|sadd5GHq?V#`}1qbx!ip8yGFY>#{{2)tfDmN=cBR4 zKz8Z-TkcRl@Q<4#4H;d~pyaA!2dRkTqbN@%E}hx%U$*1g&;vD>r{H0(%FCG~me8D* zB8>UulYZdOJ7XvJzEKhW@*=p~+1>#BWqtM7l0};g-_|_Oe8&QJcmA{n;wxN122Wtj z{Em5oQ+31+Q!qoJBgFy!)k324ZjPbJF;|1g58RCP)MO;T)T3N&6#OibU8I`Y9I=_6 zHdlMj6ch45P026)skd50xd0lGG1J~yo9=E!qK!1j$oQLd)L&^4WeMOf*X#Sg3|2SD zeEgM03E-4QqyHUz;1<9zc3x6j-4m<|n3ELpr`pjUSKRX*hA3 zdl&bZ?Gl!Ruz}j#dc5WR4WFD!X?6GJTaX#V3B&D;3(74aU7Hczn_4%twsQ2f8duHG-EJj@;i<1KoNE+XrE`)e)S5ZqR_Kvag6!v!iS4kfN^l2YKZmDvmBG<}$P-T;=cn_`4Wjo5f86f}RZ@FLBEyGfU9HzogJv zU2V*8^!s0sH`W}vFu7H0)p17mnu|H*{TE*61H8KQUP|Wq2;cmMVe+|e|GN>V_IpsZ z)aZGPb9c$6id)4p*X<4qg|iq);UX$j%qIMW6p~&uYlFZ zpuY*`DW{>Gj&xDe+>gFNBFzQEOcHXAHn$_njQ;Z1k>?{6oJ1qCtEpHDpZFWEy(Jp# z^17NbsEIe`4E|CezJ(59@$e_#0~)A67R*LC4+c%e{B_~DyEd$Eb`eqOfa}GvxkVxN z@w=avD<2$_WE0OIIxI77U)WGz`A3gB-?f2}mxo;!0G>xdlRQsw{3sz7AH;V>)4r2Y1OwNiHKbRg9ZKw$v)Di;)a z@sH)*Uwm)gQajedq0+!9m(AKe;p?d{LZc)k6G%hDM@T^QTetn>(=XVP-{EbKS=a7V zFVUG?xi@#3+^s(NqS6xR#Kj^hH;o=ZuFGjiI1~?&weTF%-ic5!z*8)e=dD`Po|K5? zY=Xt6$m(`LWNyNEXE z8cZPrsNzq*vy+l-XCggA%a-HTG6rA*-9RBS@C@n};Xft+QvIfL!wqIbJTC6 zYvc-sHp8flGB|`{P7MOxgbTmw(CTR!`LcHnhG0qMt_V}0` zKPrP_@S1F?DdGOQG`4BUB4uH3uHGS@sz(o^tPj5ugCjn)3tI1j=ZZA`OC&Y`vpKo+ zIhODsIU7%I&BgFRuLw$0!nAzh9OpB9EoC=Wgooy>7yRq6tY07f`B$H8;zf4VN>M#& zHE;vxN`F#=M23P3I(g28cS24OH}d;mi7KKf3?*QJty^*b{R!{Y#-O24>Rl5mXSRDG zoC_03kyQ^yKla+Auon+b9nzEKbS*;=b`f{sVRw>KkrvMjUWI$wF}07;8TxwiH#t>CF(I+E++tRiaCKxMb3(hdfm-#sfa#RlhVu$4Vh!U zm+msv5;-~rvYDOB{QT};>%Cw$S8mHt@IuehtFk5G?Q!?7AE#C3VLP?n)!LfG<6wJ6 zV&!lEy*RY+=|0g075UugM;|;I3Uhno)J0!#s+X9Tn`GVa=T3QKg7AWr=4L9xNGreBi@ha)aJf zS)d>7pw0(QAV+`aWqUf?+jDNtO(MbB$H(*S;Z0aO*BusGkSg)-3UrZ#r^dcE2s@k}{@{~!DR(M_{3u(my}%KUy3XmuqG*el z#5{4nY>c(nF7`k8ZaW;s$lIuA#q9Q*mY<(esAQt1pj5zTy=m?EH+_D`l|ly}duO$8 z<Yj8?dcgIYHtQ4!Xd4eS%_iR%l&BW_RY>* zyU?A1t8D-{h{dp9?gT+R%GSQp#MvjFy_zeF#gstE6jY2cEgu7zogbWQt(^aDoN% zIUxg_2!xxyZn-$FJ!S2})4L0QbmwvanYUalfodhcU0ox&nhKNCP2b%Ap(ifOLiwQH zT(!e?i<;7iJ)g79F&H1 z%SYe*;kQG_PbC`4%}v!)7s&XjyE#53=lJM{UlDG|$(X4^=0v;d9QhYeIq$v?%1Yp( z^S6!LY7>`OiCKYXC5rfYI{FVUl!y_RYQ7kbw54gqK2MxozKGk2eO%G@-M7+L+$KZ? zbZ}Q~MQc9#M)$Qj=7d#AGhcX5?>4T41K7N39YSB?fhp}SmwKBSkRxB@!mXnkAZkai zjsESozp-JF%G*P6$ui7qrt+iB$6tU}`sk-#elopoXN3;$(fO&3$p)t&n6ceeJr9W1 z1vxq48bS$(@bLFFGzK`0BZ!jA)3j3tt*N^jG4pH-VAI2D)V%aM%{yUCj|LqmQkOqv;MwqrELD84vp2MqgtK|5S`*?n&BwplqABDG-C4Mrxh}OQk@2sM-&MjXJiV~P=2n^O z94Y0Miw^-PH<$pZNb5X(_#YlmmCl`u5+Fu^8em9-d2V;azZ^d&es4z@HHO}u{qT#Cs)oS&ytUXB3=Q@16S5qE0CvAKOF z;Y$@UD#w{Dp5raOZHHQ>glrYw3A%0ZAla|7(`I#>(u>kOGvEK-YHh1~tAL(O5O`Q3 zh~7R4>A5J(OALIie%nFjQnLC_JS%m`@w>jgWo<}E5D>TUFlOd6-}huVUqEtqraNjn zE5m{tQaaiBvq{nAb$rkmpz71)bQ2Q00eZKPt(;IvD;GI_5phd(pM}smtUg^6+*v`QtxcGR#pHuo% z&Caf9xX2j!c0IHi=_Hu~f08o%iU)x@#Hi)NJiHi4$q)|dH+odlS%-d zsMPI|s$6fsK#8CKs~!sP&?$M?3GB>NX2U{r)q}cL@BZ_X)?2v1kY#-9tgY}Z^vt~@ zgxa`jIg4llThF?4zxS!9e-xXaPRyl9vctMgHxK{b-y&E5trvP=D=Ad}mOf}kbvkhW zrG9_F66*xSrWC6d)0sJ=%(Koya_;9E5)YfkLJ%YCvwcxUAPg-oo!p}0@#qSpKlhoh z*p2_zr~$x6wg52O%5g8n_6*V;qhB$lZFiv&?3u58uVjWL{CH$`TV0v`8N|WN;%j@* zfRgEDOG?}=yRxJR4&IOtdQmVft=V7vxFHo=EoR$mX%7raDTUf0WcY0#aAlto zf-}dBEDvFbK;XsR!q4-YlTpu`=`-Nrtfv?7P8voudm75rs~scEyKT@_>6Ba1O6WZM zJE(uv2CwQNKyB*}-+6OMma(qY=LNK3ckGt7#>&^e4Xu)BQYo3FW${_rDzi;uArO;v z?~jjV=I9gW#dKNEU_(TXj`(retuR3|cLTf3toc%r@9)3dulf~#-N9wh->r%OjCFrf zt+3P)wNO5s^0D%&eKC|>p2BiSZtvkRPQ{fCh3)Zw{{V#!(lkdrCs-VL#(8qJ4yWA` zm%AN@Zx?`|Ak=Fd9vNW=MxCpp1hVFG-Hl~5DU)aSFHg4{-$0kOd zFlRBJ!uhjcMPbV+w=gh-6gE|o;9>;F;5fazl!tyLYx$AY^^jm40JqMst{(z4XlBjx z@gMp0+jK8_t5b>9#Vy64k-|^0^q>)8KKjlV9`P2r*&B(v7V82^aoxGY6W|R`c)(D_ z?lnBu-!Ss_Pi8Y66_?azXMnnNKr&&o!q(Ic5Ci#s!*n-dXzJ?nkrDgbXm9JVHI)O< zayWpr_x|_fn{9p|icFxT907n0Em_m-M7z>**6#q9q4E3zSBpNKRJQK3)7dsO8>CX# zPJL}tMsSUp=WY&nQVyx|+?com&|F+}6<`E#@6NZS8xsQeFnRq}2_o8*>jc&AzmK#f zq`I85a?$Q40Lc6o$#oT|Qf`Dxo!$kw$R|Pr&w<0~T*&tyd3U{(S7!`sHwbp{Ty4|D|tw7rUt>JM&OQ>pWtA5KmLlIM^u$8DqJrPwx6IOAwFLDW~;< zTwe5(>Che*aFkg!lauuD?n%mKCO!)Vcs2&dLY!)mV8*B8erI3X$*Qbb^=N0y-S~%I z)y-&Kp+H}agUP|qc?dlbQ~LeNKFya2dG-8sLl37plEYZXac+md_TUzsGq8km0a8+) zvsS0m>B5@pCvGA|YBNoi(H6Ew9uMMNowjyY80%?1*zyotQ{aO(>UN!VX#y7aE&Tcs zF@JoN7LWq}a@=B#8_ZtDuv+Uj;}K0k<*O+$bQrnnuT>KOE$} z18;%@|3l?!LIWcTCP!pzLV9X_nNgF=u%Rs@n&NjqZM&C+SmGDdL-Nf@gBJB_LiD%$ zKl!Ll2aWfKz1_JYz&%9{_9jyEc2Kgiqj|qOYOjyJe*B~=F3RLqvOD0eJJ$nME1U#I z#$s|2)k(_Po^%Rxs~LUvWAb6j7Sd@3>tVU1xZ3GDCC<-&Sx@?24O({zY3b8yUPS_A zB5!x!tCO{OUM}rD;)M-4Ic%corbotq`jdjLcA3D$lerE515ZyW-+20u7He5rCrZV&3^-wzyYCRdvuBvAa#`aB zUKcROBno%z6QNEmUhssS3DCI{JtP-PU9s282F3~SwgE`lHc6a0Nd0dQF_+xur1 zXO<)_NzlG#Cf6DBZFW(SNw7M@Hkd(gJ)W1Cvu_*YpZMfkT7I)Z-P_~C{5tn8vv79- z!Y6zXQeC+)M*a?p(VV|CLX@SxQriu+O(0BK?JN(X{1Lx zL_fjvMVx-^7=N|T@P5`^4$QLJqvjg*Z@H_mR9Z5)N}lr7`0|Ah9xe&=AT#UAh8K`6 zs+VA+!gbU=l@@?BkB)|g>lA@s@sKzspvEX{E3gu`uEL7y1Z{vScXoBpe)(J-)6g>% zVM{H4&c79==>i+3d~a?s3TQNnTX?8AGATM`Ha$U+CCn5jAXH!r+ZwBGbNf+~*S zVx*gxLe9<=+v%hASHCd{&a7ecw@&7dh@f~6zbnBN-np64RxDiT%|Uy9UCnfrZEc2s z`0J0O<)PUk(caB;0;L(dAQlyI^pfq-GYvOpYsp)^4gsD#;*n=caG|jn{cR%&U);+_NK<4&k7eO}8Wxi8@hjG*Z;S9Dyf?IK! z_bEO8Z$9_NffLUFi6_%Y3#z0o8E5hUaeuYv4!^ClxA>5#AS80NowXR zAHjv8i&%%0mAz$G^emOaT>`LI6zrReJFO?7H~uP$c9`C1eO93R1_1=i=>L3aF<32k zSglOhcOYl2v8lPc?|3s~=zhPP_{TQHCal$1&wlZPy1R9F1i03iC`;7=WgBY9w`}@QdXBqyLoW`WN+LPItkZ|{Kxp`5gGRGz- z@)|0dT`jpAQDl}=xy?32x3jPoQmM>uPeQcZ<_J}KLT&93?k3j88O(wxFQa?nO-Q6P zKw+^LJ_}{&4dp!he}3i$J=5(a!ZO*My4gONdU#qCz!2&IwveW%R6OtSSu3EJU>L;+ z%5@!1;v)P#FlUfuHN`8tbh2%^R=FM|XO-09Hrz6^(ZBpK4H+sX+r!suP;lO)?*50J zcjB7~!c#?j+wU9@0uW9oos=Z-@#Ej-*?k%){%xYysfXh?@D$RS=uMygcY+eweo3!Z zK_oX@bhB@(TgbdFC4#;hIQCp!#?EIRoO|aqn~D2dI{L1!>S5yN8dh^h(_05!u2i%F z!^mM*D`2A9G4vBW~a-3-q$NlX51-y*AhE)nyx3W!Atxgr~H51$zUkEDQUH~&aj==NrWDRYAv zIPH2(A+r7Bmj?l+B}2nd!s+SXcw0#M`He6d-d)_JAiyjJl^ZwfttZ+ASzHOkFSEZU3OBvr!FI~W5d!~2;!k~!*ZKFfA zJ)g>BGy01k#=`m>cv~PuH0p#O{n$r#I>ixGmB{HeBSrjB+si|C1MGo4d*68{l6WdL zp?m_l3i5(2=BEkVf7}-Fvr`c??sy_Uz({5Q@*Y|i5C4#ykbIiZT7Lz7%k#rCFbdPL z_=U&86kr3`?u4L~ui!TnwRZHdqt;rrO`I7SC!5pS`@~NsZ%vwXep|wrlOcg}trKod zE3RD*qffpFn+-A$YT(#*GAo?W7DOrE&{@d*9y+H{+coUf43(Z$(!JXq&2HknTP6%ef|5v5_g)jG)@C(WOi>0yg5 zkO@-oZv&7wjI(p0+pXf4od)b4==HC!1~SX6yMN&oXLQo0?U944JYt0SK~~)Gh=%`g zF*C^gL`rAQ-Z)XXO1Y1Xe#dMP0f)}K!1P`TI!{*P)^zl5MsT^uQ+R4R(ovoQTep%- zDcp)p-0<;Xy$?r!=lC0h$=%=l;9G{OwobPLK64HTcHK>bHA~dhTot?!oifWSAmG%E zicrsy1$NxR7T&PEh-h-~xP0!%_c2q*YIvv1sm2V3N}S z-tfZmsfK!OM7-u5tD&A}%IRFjl~7mNd`LIutb6!kztSstcR_-TCH%vY-fBzU*U}|B zS8t*)i|@X!Y4{mW%&9XhWr}+>5|*{4^=2|>R}3ZX$JYwS zi*KD*djMV^yYOTM2T*E;SxLeyLHz8?@0c)`9G@!~6wrt%NO z^j>!J-Dai-4nfh&Df5?hXs#G%b~aP^IRi{UW)K}XPvZq3GV4Vnmw7WYuPr3Rb{)x* z@Ge|T0M&Em<<7p0KmF!{z6qC0crhbNUFDOs_UiEuZQq&_-L2D#`JTk53DNE|!buk^ z(cYh7*DN@dy`|GVg|^yDrH^bf-9JXdO@9IR_L6uHu2$Rt(V)daPr8Ve88sU0l zX7rpC%=_=^mHK#HX18RECDJT8m)m#-p5|f=Y5N1y&E9{dF$$K~5Df(Ow&0Jzb8FGd zcCE3P1*n zmcE=3IG|S2NqIY_Ah&0Sv+X9Bx$h~K*oo*S{PgG-UV1y|DFcOQ0nDMcn?l;``tGza z=Grp+2X8`=-xK3~5S@=M+(6M&_#=-eyMmPW2qEtX`)X4jAM2GpwI^xSB+7E5ke!m| z@1J!AqwunK3|r^w%gOCgYl4c5{I0V_jWaSU7dLd8f)LQ&GEre78ieCD97|St_`kf4 zEszal{-_Bvj9>iPcNb1;o%2I;l{%Y?Ib7h%0Ge7;XKT%QXR+{^bCMc|n?t|~kX4*- z>>j+7a7moVC_bj6pZKtT=1ad@RDN0eZgXImS_rV==l`wEqAf#1?@^ZmI%R6{pa*!3 zkN?WYpBx0(p*f^jyAN@TkOj4XROf+TTUB2zwRsm*v-NVBjVbl9hQiu4MD9+v`C#P` z36R9bx~NHQOpS~vSBJ}1jQF2k@Xq1l?ryp)`y8PoSf-By5Ns~&4Y@H<>F9y1EM){A zy}$j6(J1~H7R0-Kb9DzlIC$s#?|d9DsB{kAN~qj)csZBBq}gw6FsLW%L4XAq@@O5& zyMM2{&BteR;?q&G1pqc|47qcfTEg510bXqh7v8Dg||JxViB!g&U@+E_n^hp zl{NZ63VOhQ>FeK;jn)$_t=%Q`S*)e~*#l^IYx3h?V>}~5Zdz{lTi8v_a``0s98U?w z(|`H^W1x$+acQP`|7YIQf!nC8;S8m-FblVAcWVQrNL6E^RhHvwBlmo;>w@;P%=ee8 zxv-D6aVH#I$px`#!tt3&#ngJ%n$mb0J@$2iV-|<-sza#YSZ_$t-xUgVhyp2fSG&#x zGLIGh-5zGmh*fvkVJ3VMa`}h>c_mh`(|`yx!tX}+uY3S|-dkB~x6H=u6AhSA5vscXn&4nGm z|MjE$-`aTd*hwMy%%D=bg}fnA3BnBg^vN<^dg}OPTsQcWMeG?s^z*)NT_S*TQp~7% zSy)Bq2o?<--M+c~?f}E$^YI@*pD?NEk^u@U?T7_&P$dkRZNO%U>V%zswP%^IwmK(y z|5s&~nlXcCuhuLGfgP87CrudrwJ(x`CHEIKxN&IJ;Mv_t-fhfMy7H?l(5I{7w&z1| zhvf&8%MBoa9Fls^e(s%CF?*lJQFFT4&xYkhEwB}s$Sd_ac-w`%zW-DMJ=~1vB3~7! zsnyLI5oYD_V{^<*%x?Pow`gsz`K$mA&ZtD{x%|0{wMkVRy zr;>q1))}et0+ADSOk63^Om+n~`rt*utLN#(h!WYE&wRN&O@V6Rdwt~S)(SChyf zR@eraxYWW7D$KgvW%O3T(FWv0`D9wqHypkE>g#8ufb7`ZBys!c2i`ol`mTp@&Sp2A z>G0RUOZoY*0eAf^1BWu`b`>LOi3cZHE_uJzcT3{wUwg9z9}2ICZkt?SMFODgPv3oa zm7bK@d^rIDoE+Gm6Ha7G!^DU&wFf|b$;~~SQ;1ZG6!|2BTsiPYy7y82?lnPJv4jib zb%S)uw&t|Ng<_n(tVoxr{lH;cZzxX;EHlq)tY|eFAa^j1+J?`|?ZFj5a}%|TSxS>)XNw$d z0#+J;4s}SQ|I%BBr5ZS4#uN1f*DLP4>7Y{b`~CF*dr7C~)xg3wpqGpLUV^iKFUw>7qco|_gA%B`Xl!2>dd~dTg&f1i zRfUD*@mK&cE{M$WXH8pxA5jW`fvy{m)n;#J3Vg>i*WY9t*KQCvG0|B_kTTUaX=;xli{}V zmm-kVt<3jOfcnvqg~a;4C@}bDPVoC*7V4dMzS+J+`l8RoOFTFG@eh}O#h4$b(zZvW znmLATykd|8EGXTQWwYJcxBUfAo~KOMJpoJTlDHdxF(uCkc7#ph9dM{u=;40qF%`;{ zms=aa;L?1{rQsrAwtMdBmwVyH^ECsmuJ6F;TD3xAsB;3O9c-H*qsXoO+ zlh^AqCjkDR@-oHVeYFQB{@Fk3jjtQP5ZQ1SdC$yO!$Y#~=kfqQrhH?ZeL@^qiBIxb zcI5X*g-u9*+PFrtIT(p# z6aK-s<_ZYn8+L6{T8t5)ki!-M2L0O6-~aG#0r~uu9?IQb4*0q2ta=F%&#C2YMt|l7 zdz}?!2_b5bO<~%*f(v(`=k?u}-%;kWKCe!;WPD{seU&2(ZGb!UD#Zad-#xH&B4c|jn-clotv%0S*+I;*E zf7?40kQV?g2fHU#wjwGB!glVeSFR$06J6#Fcfa?&q{w$bKq`3+pyDfEFxK&fP-FuK zo-W{?*9MzYIaK_@Ff?*FNM#Ve$5YM`&$WKqt`elg)#^nQ4Z&vY{}@j*Yo;GvcsWZ z=G~#0C_Ry~>l$#NIsv3#H>ShjYUJ2Sgz~kv9sRpk-?gtRz2P9*1jyWOu2@xfHDP}+ zn_2~Mz{MgT{~z;5(`l3d9}&@yOi-YfST{CjPH*6@AQIs<3vu%C(f90s0T!lmCpK3` z+ho=*KLVLKe{KLZk!hK6^QSK?Soj4|kkTW|T!T7z;e+va z4}CJM&d}g+);z633Zk6xUU^*|6B4h7UB4Xv{g3GjFTLWmB2HjU%=j+%=JwI0amFRUB2tYp9+TwD5hE_&6jUhjM^k*l(-v<3g=UJ1MVfV~N zB!RBlPmaDCLpj{?^`n%)J&7Fs#A~lc)jWf9=^uV_>v=PPM0q+fO}z8)OS&gO#hEq4 zvW@Tzeo6&#yMAu;&tCZF9qzlD?PozK#3XCKCo*<`nh(VxVum! zw)DYQ7H8=&_Z~*?&b!@dC0Zto9{aaFbAyQNK|3B`Y`N>8NeBut3gMGB3j|7Y>soCjfZ}R;)9j6)IHEzG$CLno6`LWc#iXDaU-WUt`j8{K3~fN{{=-19)e3 zO?TlrL+izc1#wec5H#d@yOJjDyTAXA?`uh!=Q(>kuTH`h9dawdEASaMeh=VZX8bG0 z8->}yNEN#D5^XC%K-jU|R4uV)V*KuvMGQ*=863j6s_BztM|lBq|TLLoxw@VRmMuH8mj2^4aiUx>=+% ztYu_r{7#a0!)|&GFN&yzTENknMXSvuglYQ!ntc=AHiS8Z(8XH$in>vnXa4}a zf02u!md{8Dyv$Bx+Wrg3rYf}MoWj!n`H6OD8awuN2fJ)G0N(;7a3yaESS`71LiW|s zue=6_!RbwfZompy(LU_xLA>kz*B;HzQR3}ICct@Qi@=R_*-M@kxAN(j)3?izo}7pO z>b=lmmQ7sjf!x2JwvNonCp%&)r`toZ+?LcX2_5e@oGr74XORO zN-N#f4kJfA8qfe*p2Pnc2yOu1Y%&M^5)z8QVrOL9xMfN^U@cxvN-HAA_bzrAPa zS#^;HX3bJwxIc8u@ppZ?hrdK-NSzyW4QuPDq*N_l;izyl%H08s=+TEhvg=PifOBA9 zeTA?svc+@g`v-81b{8 zdrZIt$tjS9^;EMJ3TXgiF55y7UUSYr=R7wUvvYR$*JKED=7-HtNba8^-snb~QIIEQWnmg4K?5P%~) zj6VHJuD~~Ae#YB8f-d^p>Vy-m*XjfdkaF?x{z)ast&d$>q0LXYSBIUst>%}_b87H> ztMAW&a@#H_nW=KlYoy9;tDeg<+pv~Qdu02t(GEAv82bfwybqTSNyr6ctXQWambFMPNDJ8i&f;-Wu-9joP8OHuSC@f=IR`OQEy0tg9u*{m|aed=nTDd6dH~=KT_ZI__uyg+|8yM%SuZNk{~W@|HYB?G(aJxWHN@|=9wPD zy@wXYcvwtV{mqHKEL>5h8X0_iV_qk-D7$Zj7=Fql?`~WZF&b&Q;63Q zfHZw+@EMSd2D8hiCrdC4me;-i_y_JlO%rn%7#<~q4?l02aSnPBX5Yj`uvcY#sT}}f zb@L~5pZa@(^r{>;CrjTS+`bz%!!5?S&WMpp{r0(g485ZEaPKm0_+BZHP#K%nA@DtK zh&-or)QI#lv5=w_AoQ^gQ(#lXegj}k*88=KbKs%$rIi?>u|saThS`+UOOlMG6 z){3M3X{AZh6_6%IqDbtC-kcT3KN77*pHa6P57RGs?S!8|*GZ;&7@ISCe%asW9=gQ# z0wj?+2s+}JJFGWzfjWvCB_J&l1whoMkz6jkjSUFlFKYHmL(Sg`c})-jJxd1NB4utr zu8S>Ulc#}p+XGJ_&YPcFB=+|F06Ky6gp2jE*l8o+B< zowo}^UGs-L!}1&DZZnghSKqkN$zy-NMCt>iLeFgmaKe!-M~snM2v3quf%V`B{3_aA zx*!4|@!hqp5KaoWufx8q4z(DvxE{>rj*1qSCUL%SL+*vRYW(ORuFuJSgh*InpyV8f zio5BpgSWo;ZIJkZ8RIgCcI%asMsc<$AmAuoHDDmSC8j`M6OPP2i#?rcFFc@U$cqdW z{Nr*70-S>`x%=a6ZWSvn9jnS_xfY(j+?`p5U8y{plrog3_aN5F;Agk;d)w8n>@U1I zU&3(xuq~k17Fia3DQj&XjzteJz*7oVSIJQS%ih{yCpeJU^ts7(M^1pY1%%2;_HRF2 zNmpz)JrlElGPrJ5TqLGuMw~P?Pe=~nZ3`rwC$_YXXNU>TQF$R1%PZuhwt?|fiS)ld zE9WUFE(?J_cy7IeClnc)w_kJjpk+Bu=%eFFzBxQBp$k~-1c9m0dXvMSfJ>5drwQ;i z8tr!{9V|5Q`Z4H#>pkk6wd;uO|IE`JNxO+cH|-Te3l+cebmB)oS?&Pej>Nzq+D`_#8nQr)wHK0gGiVc zGh&e$6a9AZ{CnSE*u|bMbmxGApeTjseZyz_sdjQz5)p4t1G?RhqycxXmg03K5Nf-WhI|q88Bg7_!2JGPf zcn&=lqY+k)P=V8ONxr#W+4S(yIM=CCI+I6f(a9*3Ewqd#CNy(W-U}%QZ z?HY*tq32T;zT+PB07noE-_H#HD1zN~v0SG3yk@oA-<5liG@=@{6o_JFNZyDNw)ej~ z_WTho9hYZ_MlaiCKh=)fJUY72^f-lIS&o~GGidgR%4;3O>}hdqH|u$Vh20zqv^Q+L zB{n2IVfa3DyZPt<4c9Z&FBW5WncjalboEWB2?vlm7JXvRUG!=)9K865h1<=ERaQ;2 zw}lXX;|^^3ePsk0Scs#_<#N2AD>i!b!}2!R!MSdC)hC@SB3I#%)9klBLXVwX<@RWa zV^5179kfCAiIi?nK)=olWxXruld=s4|M`Rb*n{zheu}CTm9>b01~~g&faYEw%Bo4_ zoyL?t3h<5vP`d1JWL+sXKZixIn<6zH&RCcs^h=HX*++Ug5dpsjWV~@z0`escWFa6R ze9w9SXJYAtAIxT(SJ;QTjjA+%;!J(gteS;w9CjJ!uwQ|O8yesHCfFd%;P%eEhr!I2 zqJkbXt)C61CGkz~?W^ZGzr>r=`hiTxV`h zLdX+PGzWd-ON<}>&_ht&xTOdUv>gLTw!|!I5pLCxK$VoB$&O)SPo~`M{4NNv%BmHB zSbh#)X<0>{O>)Zfw52o)oinhel6@E`aOVY{v2jybEjlMNsdZq++4Zb3f;M@vP1wKw zI_}zOU+g@nl6fK0r$zW*08@J1cL{|BLTQ{B2(uqT$34IG|SKd>O zcmVZ$K)!S%Y7WVQ@hI5snU)mOkn~c^EVyVqx`fp0$E(!Um!gMI<5|E@k)}1L2%XQ~ z+R?q{$Q-$p0|HrIrZd)fST}G{@hFzeXm*^+i^bZaESqXKlOVi?k^x9&Z*%``o!q0| zl6Oy37A#XAQ=1FIuW!Ev5`w0bf>g$%Al*YQ;R-e-j#J_0_a0F5Syi}_hQ*;hu{yz8 zHduCP3m)|k1=UyyQ4Y5+A)EcV=bOV}Jp74gX$Ju-t-0*xPT5stH~hqh9unGYjYt4C z#I_DF9T#WNKs^QO0ME(nK(W@z5udYveD^vhjoh=Bj@b>q^uf+)@wSy0#$N<^QG326e2NI{2C!*n6{j7 zmY-Vb;76Z9PrQ&oU{@>{>@@r%>Cp;;&6*WeV{!?n*Mk(Dv3$m$y=?Z7Unnf@9L?#l zzWvSDTgyZ3^RMcL3hBP9}p7 zJ^S|j;I342%IP3zDZ+I#GVDBo5m_Sc12PZLs%yK}v!@vX4_A0u6^>|whRmeE7;+9t zh^ay2Z7(LmZaL!KKlsCU-itPU0S@vrAC;@~$RdwLsp`EYa>GA*`86Z5N(P^Ul759O z=FX{Sb=_ERPD{+PxlCO3yN+5)>&xXJJuJ_=_HFzO|!ApN@uD6H7vS;S#8ffx9Y|ZV_{_%J5^TY9U z422_=^0O4#i}dy8d*7(sVuxll6cEh#1m6NM7B6~}w636LrniuDbUyrWD^1FDc@#p4 zANLDuW9-vz#)I~iG>JR{0E|7Ym?xhvdyCL>4Um-^6fcEh1s}X2oVS#GdptRE}oNe&KNVX$zv+tqt7h zjzNLfrS=;PrxpLH(EXyQX6L&!RBJ|VVtBH+9PyvJxMMU9*koB!BHsGgqmH=7c`1t> z;^AWlBo=^3ov*G@rviA~xzv#NQ``3agO|P_a(qxI!;6as445u>*!kOX{#v;x~!+I-xFxcFXrh~ z9bqeUBsx$e0#a+D>0D(FzI$*_S}JC;i>QFH4G5Q3dQWix&E=)xkA3u$W1H6@ZCt^w z0R_FwX4g!`nQC0vwj)u^*A&1QfP8oUp23s%dW|U7*OfVfC-C`b@AQ(DIT5$njl2b_ z`mR}QN*UX;3@?%U?5$tL1Q6B1PHrU?ykfIBpB1E8=v#CYPUA>A2IFw}doL2@c^0*t zg~wwhcFCXqPj?sa584_lFGbG&#W}s1&IDjbO-SM(D>LY;DA-#++lt9Gtxa&pXmry6 zeDVsD7z=|&69T?G0(XrMS`+Bj&l)a`qX|UB- z{fVSaXt#SrbMplAX#pfhg%+84Cf}czt2xSn#LO*l&Oc`O^0339XPo4Qds^+i@DpAG z)f*V*P1bnb2z~@$-P)=JYxu>4F0WQI%1+0?+O`o>i!(D`t_JVU@4(q&L?BRbOok5a zrD_G`(8Kjg$W}378sDf^d@_6M6K_<&iwPP^s#{o^sYEDpBuqF}jp8<_r}gOVe~bYA zzdynab=8+Uu}Ls}WU!n)H8zm)`3$TDqTtoB<^i1qTKKL97bXz^dYOQHj$tZt$?;2t z>g2gy!XT+rlhpXE!M}g@XSY*rv&jQRU!s9?5djrng`F;M``F5_rW?ATI2N96t&@4W zOV3V`5H}xxb2kcO$hF6vRI-}}MCdq!cl%o7(oUTi>$)8NDlDINTXpT|@wh_yGEfMx z0%YRNcN0D+g1y~t9H&Zl6z=~}7AEppTkI(KRe)Hp*VNZ5Yr0xx3s{&9i zu<4k?t^_|=i_N{paXX^J}na5nlp}DO-AULj$Cug{&587*Daf;!g7|7ESI_R!QE)^?VR?5 zpMPHG_E>h3`f#`fyM{02Pict}`)%uIivVlzoQPvYgn1v zoBNi)Mr$V%it@O2O?%{TRf?a7H}BJDrm&!mU(i>{%1gM-00|FdIeCpdPNK;Imb=nPq$oUHAt(&~Z6V)fV5<1k-?R)t z%7_No6@@g>rNt)OKqsNU316G6k7rs}aU%dyeVJRt@P|M5As%m*XEtTdHg5+cbPM-P z2xDs)+7K*SRo87DF6(dmse70xx6zo~95-;o4UW_bJwd)2UY7U2>dt{@IRiEVp?aHI zlbPfj!76Qv^TJ{vD+vx4s6c)1>wd>#D%Yap@$rz=(Mh3pH3l=>!7o1VO;K2As3^#l zIuV3K^Iqu8l2ix3JGcuL73xs?*AaaJuF{?HxCL76PKcG;cSaY$l}rugVsVoN=cJW$ zW|a_K28cg=bry@%xlb@@b6t%``gkh^WVXJ75pKW2p5y?@GFO~_fc++wz@Hb%LJ=e- zQ*14Ohay(<>7FUmna%UsZ1NNo($&X-s%!V{QUyTw^LT^4Di=u_$jpJFe zKWt97FIb4sO3h&@$ADvrH;0K8nF(6m0;qIeVZc~TqM>{!qQ&`o*~+Fr4lXB_ABAUq zE;(a7X<>9X^z6GNdUD{2S+C5=?aOz8EuDv%A~>sd_*UGSY+9f9L3PTNwK9VxJMCHt zfKdzbF!PdJm2Dj2N?;qATNAM%TVxFIfI ziJtv4UItnMvOQ<3h2<0=Dcr4Y|D}f`40L9gI9xdK_&RO)W4o94Jp1il(KwLq10mOh z+K&onen=}qnS{gY&LdR59F-a>@MLlYJNyc^8*@9}CR<|Wh3AU`L0IuRLWjEiV5C+| zyLI9zMyz6i@QEY_ag5K4kxH>*e9SlI@H@WjO|I}(iVL1w)@Dyue*_sJjJ4aWIaD08 zbhOd^0nr=%l(V)}gf26s9=!Y^lId@ zYJYL?)nRhUt*Z=>=_0{paxnOY4*(-@vK{Hi-kq8L@LN9o0NG`FTwg?!jO+qC5(|Gv zlmst$)JZDxU^Lmj`;{+x7j!pZlZ1jSmumw^Cz`YL@{texi(Ldcb7qx!l4n!#5`YJC zygLJACX`EVzdk`|=Nip5dZO&Ipi!#q_*4YSZ6JXXsNPJ4BX2rp^ABpU*um?9OD3g1Q*wLY=JO z6S|*A`O-Vn)y6!Q#^GroBNOPLJWQ%R-=EjRN8>iv0exFn(^C%!t0rEynxvSk{vv>I zMIh$)Kh4rozXdbRSp_gG$3W1}U#Bi{^IflcL^^LLTFoD^{Hh?w;kWIvQFV5v$HfGo ztI!ns$yZs3%S?oj&n=#Yb{uI5^% z{WeK9Gm9)y=+Efk@HJK$S6%G;$XFgBVH;Y*e}%^Wd{nwjIe|wev?I&0Q7N^<`7Jjm zhU+w&BO85kVBmavePq4v^MrNWnhYn`z3JpEZcT@*yo8j*E&@?cI?UCiI@!jY>hF@)Ll262)Id^0?4% z!w2gO@{`e-sN;p|2L>qR8PPiW>I{h1=g4*_)42bWZ(u-v_2)21qT-do;7UR?C=9yR z!4*G9RniTAV;y!C~{$UHCPxLwQcQZrmjx1%Obig0g65VwHV& z@KN^B0WePhLTcGDqz=0jGgPoUPHAGzUQ9(_W8|r@nP+gr5|IZiRG)e^k$MkhQH{6Pu^}zPH8rUIm z&7`Pv4BT-qw(Dg#S&^gpxWebUBZ`W$@bhC6Wkz#=s{7!Jn->5@u?qjJ3?e}ZU_l(N z>%`@$2DL$(>1QVgz*V#D<3E1KJRbbco4njk^^Gl}Otu ziJVK5+Ouj6R;7k(u&0i=r>4DcZTCxziVYT*D5Qb8w@a z0f%p=gOr<jH|Jt45HvEazNmy&DdKi>%vyc59DI-txS=meZ4>0?3G%8Ryp?;t_OQ*^op#i;itUt5UinW2>n7FzUN@t0`~&9A_*+0J$wNM zw)3SyMQr$o4<10LW$rVR$vlqq=|#)9lR{(hXcVbh(8Nf*B3~Z-^2@w(+Hu(;W$j%z zH>^21zYaf&J?Jm3FrMtzVT5GrMnV9L1GhBNEKv|PO~<>#$r!xj#Wra+)U`cB{lz;3 z!p7j*X?HmG6Oh?y384ET2}C!fpnIF$0m&t(CBU_s2`sHZL?= z8}Q)TS#98H`=rxVkQR;Wt=HQ>c?vL-y(6BBy#_^+PJtV$Hz zC4NjCbYb{2tm&PwFnM^LZ2>{=brX9W={sp-kB$>f-7+#7k_X|*kGwYHc?Gy$XU$9; z)_}s#@cK843{~h^0VS5b7NU`p;M0k^BjKIOA4b?vx$8AtjBoet)t=|4e(vL=0AHvYxVkSSW6B12C-ks z*@_$79lSla!$p**A3hfncEwOSR*-yi`2`x?9*52=?tMjs2 zkW3n{u3Cv9uHA*^iMyujw^Xaz!#mhGMliga3y0P=P8qns2IhO|iBd%jObG!rmkAJ> z{-MZrrLA1;gC$Xpi%uTxFRd#$2bbCmKK25dF}*Q`*g|7bb$EUtQ&5k^@#BIAYtqu2 z?zc^`=tQhUmmp9v3mD$tOwj6Vmtz?K1h8_rZFoP1%v^Y#PIKK>tb#%hkFCPlL#iMa z&{R%BHS;gHo8&RGoX=+_GZ|9f^F3IEiZl5%cbB3yh1Vbz>N!TS`5zyPH3FT|>t!j)#Mm ztPA_j&?l#{g;whFD}E0hueYsLMD~8sQ-e=^uimZHVl2~*P@&h2i5!*(EKP6U{y}^S zk9|NDIXO7F;BNw!!^muYX;P-#b$Ht!n5|2hwWxI zek=tA2vj-!0p9aB{ZU08+ripZk(KMOx#e z8L*S^{_yS?+pdvgs{yG5E_ZejYUPAJ^@V(P(n8y;!Zxz(J-eguoipN7-VawI#s?_e zlYDCMk!K%WCQL{Jg{KCY*7@Xbkcv7AmPWwsE~By2_^NNfSiBhiwY&5wI_R~1UKN+U z2rR?fZ`XGYn@g9hM=)->b%St^TTA`P_uhl$w1R5}sv8G#B+QA{Yq<_QndAH4{A%fT z;Br051{m7+m!mcO%}=AX1&g&7ZxdaXG@d^E+UFnfxzkb0<6-NxU1tJX`ON#k{#HN- ziWKfnsuY)v3FfpG|L+*rI92(BdD< zd+oTa(9?$pPWko^;8yOHmS?wc^zChDZ*co-ETtPkUZ~9BEQ#!A!iNn8%^Q?AkA&dD z{(99hY?Ceo(OtDD8Dw=o%l}*$12$iA;zS*mllcN#+4#nC1NU-0EErZ+p+2;KN~tQN zm5M=EVO>-`u=nw!yxH(J6BjLXU^zw+p9-#5NYITJ3SJ*=e~QjtfM3F~?b!AMA@&Rm zYLnb}BBeDl3CSu67$khx+y6CiSA@MwgNfX@EL6SWBvu1v0$iDVaYG77JAE4GXdryB zxyBmgGN=+coI3y5SEtx{=O7%vhT|=K4jo7z+Ly4uF!)fogYCJlCmD#YSd@Lb|HGLT zjj4`UYG${GBWZ2!3_h|L&>6`wa4BB{qMTV5vZaqkb)0V5{`ODrE_OJPuuk3+*5IGK zOWw?!);d%R6sU{ydbXfDdMJKjBos#N9xn*S^8UKAI=FD63eV{o_2gR~v|!>pSex{S zG2u9Y>ucFjY@}2pZv{!H1F$ZxkH7rkqJ+X>$V2n3>MaMfLEsPH>RV5+TJuOFk;e!U z7XBlihiOzMdO3H2T^^3FnC|8ko*_Wo-JH`6ot$0ba#8HT554#%dRdKk)B2QPn!YDH z(oV><_jEC^*P+>fk>vLMv2XFh1@x@I{X#2ZLTV}?rWQoVF2vzCe^nG8@)bx=Fv8u! zz2wBs*#))8oMJ}|e&rc!ZJu02i+!6@z`Ds!mn=D6Fhl1h%vQGIiJu1}r`%=3kAF75 zYKru%6R_joxp?Svr9AursDg3iD(;jeQq<@1CN708)q%o$40eKy)Xwn9i=Iwsb_Bv6 z%0G+`N^(iBRE^-y;On0+R=Im&dCvyDLr_N?Gd+c2cBb)JSh~;^rd9ibO~8Lzk3l|E z*S9}6XLnL(PnS%H8zg|GL%QZJPlEtubOxxROX4l8&K#D812?qtx+WDCJ5Pa(&2cA& zb#PyK6>zch`Ya^ZqjjpKo*(?`^Dp~WQ#H%S_@hM9X&<8@2+GR!H1>C9l=}3!#daGu zCrfg{Em?UL55Mn2V4Uw4n<%a+7VJK)2~d9b7g5`M0U_@c;WukK7So&^2H5_(?iJK&s@S486R>3$Ptu z!HI}_Z7J&B;OhskjwU%{YY;_a{CQ6Ak=sv3)M-s`;Um+Nr|rbS79{9tcN>*csGFZa zRqqrt;>=JCqq=$NuCrSevN|Qbm@VQvln(4cR9u zl5+bM?yYzI;76Y$GQE%2fbYuYppZP@d?N-Cd@*ef6Z>gsbnvCahjSGO|4|_eAij42SacuiElhMtM^7%t!@X;Jm1cVZD{B*Y3^-igP`5nS#CUdI?pa(B-jwcuZ59= zeg%;afS{H@21^iZJXv>-@duj`SgleK1dp|(D=082SLd~^3c=1>ktob!2A*Oh8X|_=>AWKVAIJhluzVwr*EUGIpcBS)%>T04LO_usggP(timQU_lU)u@r#iQ+hJNSQ}ef@|S7T0x^ z3ww9a(uETRc4V_c8h+=8AEx3e;Vq8|?0la=jeh0@yY;>WWP(;Lko`5&2;+T&-+B4o za=bnWQ#UyhVb5%Gn^S9sW{$~v@V=M$b>9`F#*Z-5k9wT|InVIdCEhMew%EeB$b-L# z0&2GVA#T1_#0mx%qU#E3kmm{bp~K&uhx-*Yq(svqwj1oKP(UVd2l@+dBwI?d%;c#0 z2)$aD6F?Pb2u@pSxJt%eQbFzP&4a>U=os+!aek&uy6nwpwP&u7jCH{Cs$$qr|Bbnc z3M)WF`Nl!gv%?8@+{U_Dzf>L2o)~=XHv|0>ynSSu?KSB6E-mcHnLHUpw_pBfCLTo| zU{j#OsMHZ)X#?DI`Qt>I**R3pBO}~&@#;K$>zUVVp4?5>wHXtO$aJM?&DwXIl(@pV z_n;2F*t_<@3vegPHdYmWD=eq7!ZA+tHZ!!jr)jP-TW_ujmg|EnnC7~g)78@`1a*^Q z=@*Xuk_Ve4*pkOl4*~uTHAA4E@Tw?u9hI_@eTvU1k_`U(^Lhu`nNfE=qCyyeYO(}K zV3yr{N5MmlVZzR4?e!GZ5^Sy(dr5HD24QEH{o~(8FV~}FlXEiR0^;$99-1~B@qEvM zogdgn#T66d8mSuSxX&6(6ESZ&v&@@+6472Bf%R8qvIY;&U?NEZIaF6*1;W%6LrY99ox5KFzkx<(} zD9p`fU?T7?P4(wpRXBw?InR3_K)?==fQT~F8Z8Xhb`AnR>EM9bt_$+%f6QDuib6*8 zv4`vwL7(OTH3cy@r}EhpKYaS&_h+xdl4WBrLHQA$At6>JH{bQ}I$xZKJlor$gk0wM z^5UMdxE#aGf9e2G0dYjSTMfVCi{A8sd%0hOmlMhOXyzB~U za_9MG?w55i`0AHlAM3GIxH{Bo9cprp^SyvIGBmZ>C$B9$ON9SxA2$kqB|7&ySUjJt zM=r2#T*roz8n8W@$wu5FBPtcJbo=Bl{au(M6swS!0_$WNv6?)^k9}(7FTu~A9$m`R zlz~=C$7+bjgMauO7>V$R#6)2DEu?-seDcPpbc3JD`JKlcjQL8QFJy+#6H^X{|DLom zRE1g-T2BD18uZ4AhvrwS4FBMVWnm5Ey(2|i(fZ;Yw_yV(-e52!2E7hx|!>gL$60ab)?JN-tqE1mE2>ls6ve3JR!${<-Lv;UB^&&szjsf&B5M!oqwy$n;G9i^ zM%D`}yIns1I06UyepeRZc=!)Sj~3u=&PXF#T%lka6zjD{!~nJp&liO z&QdhqX#qnrR~%@eNAh^{x9$|%>ox04#;Tuuy!4iPpF!Cbv+DI?iuIsnVdtZ`2bPAs zmtSE&P{h-54JA(#7$s*#qV$*&h{yt(yDuK@4Bz^kx8cz*sPx{Uw*=qAC2giSdq{NR z6->Vj9EGK+e+NFm{IOiF!V}Vp0x>yd#xWJ(qUAJ5&E`9l*G%V%gn)I~$JuehB>Q0FLAZ5hiR-F}`q)9Db#@pYsqI$4)|k<La8+`f4?n!GW0R-zIg_($Vf>ctE#xqSXvO)1;>Bco0!<<50&7GP^Bp;~n zSGAJu3(&E4zwzzJsopJB*|zyl{nvL7%LWFfA}~tw!M}a(O_`v(`KhDLxYK>Ns&^Wf z_2g9#BFS_;ua4gGA5`{c4UGB_j`fK{n<*(xs(`@9+N>P!clb*6vB9T3haJI7&!G7( zZP4lT_Uj)_E3a%P8Dj($Y`@HXu?mVDEIk{DHSXF2CF}cYJmuf|!n@W4tk+ByfH7$J zWdC|+x(-1UK1>|~`CMW^6 z#1clTiJGG~NanhIB8TmM1j>*lnpD`r&UxqJ0{a)c{WtQus#b-zqxzK|_=qK& z8!dzq#xwvDat#BbO&=tTd%XkkESto0uh5wt43OP(AElOJ z+Qq{e6pC3l#~OI`9~yk@^UA24^;U<(=Hfu5 z*aUjEl@VUD9Wim^ad`U&C@bV^w6Y|1{`u{% zLywytIg!HXXl}>LUCOB$XOjVJ1N~rCthd1NH4%7{j>j`1vsI@h&{^l@v{#kgMACBZ1@+4C@_&O2uCmi(`$n_pOcXS(CX&2Ik0D7Q`eZ-c*EyE>ut`)Er)X9 zE(>pm=_)nGvY2>5e^SYwk1v>#s!K-ph5erogwIP&h-w39m1rIU7O-m+IrQ7X55Ax; z_5l<%6)|1vleo65x?(PL_)#ET*Z>L0_KUC>PwJ#4!O7gYA?3i*0}v8_^if_ApzTDJ z#TEjPLZB#k^Y}gSr3zHbvUoJeT@WEbQqxB|J%@Z_HGJh`50?=jZ=7>4Ez+k%gLR1X zsbxv>#9>&QYv9m5N}Az!+$oo9;5xD{qynKlzK~E#IKsV1CT(-vEvYB}mX^x;oSt;a zv$W9~7DOLD;QK3SC-0hZ14dGI-3@j^3QC-;Bm&L!|{T$CW}Km1$o zdk+{)r1HEQ{K8Kcs#C+7*@Nz2m|Zzwt~me{J3(`S+`5y;d?BCqn*#KQeTxIJH=s}J z&BRp{bc+L6`R12s+FBW#_G}{g0)V#L@cQ;L;K$iwRQSpYfy}|C5-AHfo_Ep3k6K-KH3sfwUl3^KJogHoBRo`+%V5gkr^e9C`oXgU1Nk)H@<9<1LP3 zSa=~Lg4)1k$gwu}Km18`HnANoLg5%t*%(3z!IFb#`eb}MECK)nTa(pjD{suZgMYVr zq@NRnz&1RmN-QUm7rYpGBZaX219PH!z>7d0Tw299j~|Lup(tgIjDTy1`5GoE=*AZt zdkNV5#n?xjZNi(wU-;4oTrZzD-O?A$O(AWflifqI0#4LB%cr#~V`+kX`+0N=&!{V_ zEX6SqY|JYBj#tH!A$3TES}>IR!kuHm}_A5y(IHmxlDZ$ut=-raDEFY z!S&cbMk^Os8c7{)Wlit1@;LnbpV6C4U`9?DQOh-0w>sE5r1Wz5xk5V;T}s34`88?E zNSeac`3fgHbT5b%c9BQhpGRly33C^6z_;~yKo1Cl0^z7tZGc;rW99v?Lg{>jp1V0i zHip6ewKqmYfItxg%4I|fBeBrB%+4FaUB+RuzMivk@ULF57JcF1@|4`}vK^#N56K=m z<=erpfBFM%;>b)OOb>GtKp|`j@gpCC+M8u?hR5>+=jMaK7kH@85oilB@4QCm!w=Z6 z>xGCm(yTuzVzCH;!?Cop@h&6TzgH|~uzHpeCEN!nveEEMMOT-WnT>M-h${o2)H#%X zJ)alj?Uy~8tnJ#W4yPJ5nW|@+TpEw?tEeTl$)ZQXoXh~=J>dqQ`of*IEl3^S+nO;T z-Us9b2|#n&0qEWP8Iy@jcmEUZ3Q~C*VV=F)riSlZ8O1oKNU8_F_1v3MHE(#12rrA4CKk3x zpD!T_PyU|E!JoIW%A1e8iFzs~7#DXwPN_9Lg~;AnhJ6t@8`?bzi+F>5g;>OaHmXFe zh!07Kvp@B!nQe_=a@{iP%f?Y)*!8W?VOzi`#1}w^Fbk?@f9c+$7lpbW5O9dX}lK8sW1Iw!_ZDkG}N!VsU`Hm6T7S(?Yz4<3qwLOI-q`<|8Ne zybDO#ohD*Y!q8bFTfm~4ccJsZa^2cd7f?>fhriv%(n6;d`{??^eIvirVLbfbK2oaF#>^4zDoYj)+8svZuc^AtY)M|k=5_d|ciS~AY@xmen|Jb<72e*t z>dlFwQ^kX)B&UY&U(w zGqsy&U`tpoCngi${F-ioI243}@RP-QLiFb8WWPDR@Wd2_CgpJ*T_t_^OWR=?0Y^5*`XQ6mf(_JEJyw_ z_f@&y%a^G~o&juF*#_)n8D_g34u0-4U)zYlm;ef@HdT3EJ4DLUUOMzk)dq!twoC0) zC_7AfiJAt(7tlx2jBD6(vMq~<4ySYvXp1WhfD65AgHdYkPg!Y<8P%I$O%U?BMehINo085Sg;Ci$tDSP-gciUV5xRB_j;@(-G&Q!5Lv-Tz zEtWZiFjH}MYEAh~v(!C1+oY$2-GHN!IGrx+&%QB<2#dd9httKKsSSy!AjGvd#%_np zn=7SH*q^;Of_40r;Kn%PmXnLVtSQP@!c7|8eoS7QBdIf(*2?Nx1s&r&qK5u6dONdU|Hw$@~&#$t2)Q zTB0>L8oRufrb({1@=~tyR$i*ok=ibom$J)yxooQy^$kS12u11%SuR9Gxe*}|o5&zW z2_yx*k8vdCocC&o>4%_jZ1YH04yg zJu8r}mm}jHt%N@gJny5EOHc^&62NZ&|7iR6)t@{e?C8&~PO7ksXh3M~Ri{U}51)TA zxC+fI7=O}*sH2|8J>01?$E&b_yVPoT(WR;t{q6D- znNG``2h^|DhXj1Z9S*?)C*MKktsYd5{kRG25O7&i=M` z%b=_AbUaNP_vmlk`vgsg$#EXF>E#IQ_2Xas(agGF35nK5NVqrtyp8rLk#K48nsVvQ z{#A2hcX_{%@l6HrL8!h4c_c0$yRWj)f*f6*qY69)X>sNc&9_fa=q@rJdKjx$=S7Gl zAQU;UN!QF%1rSEzvmb91=&skih5^S4@X^5GrS|tqdr3T$Ow&)k_(o!+v|mF-Qi32U z4|MKt>0&R;GA3nQlFgz|StF=fDX))4xZDm$Gw<-!PQN#hPL6Pte zp(>$EHJT1l5GLRMso}a;IOTXBnAZ-{+H{`ZGKi4)3q@Rf!u$3A`_JCkvjKIgvzt1i zA1*TRtfFqp&cN82$ZfIlJq$ug?(h>|eqhrD>a+9k#PCF9%2uY7SdWW$d_kR3kr zkiV#;S5>to-5CMH5-)M5$Q>B;@b;fd(A~zb8qkV#p(A`Je(yKLKpkqU20^@hU--aU zNaC|!FacisxTsGGCj@*$=O;K+)&1LF`{^LLJ+4NmS>xUo_eK7ilO4sxDi_B-5vWK` z%TBj#(T5vC2NP+6WP@-%+CTPP%n#JVomZQdz=aaZe56Z?PO1a^40{?jYY_>Jej=gG z>jI)zXaU!6+mC-}*&m?gA_ z9xuFc_K4Re2*M1`cAYsp%0b5{V1#d7Pxd;Db(S<6>d6H%k$H+9soqi7W6>-;!IHNY z-$~r3jUTlE=(Y+Wc}d3CTypR__P;13^H2O%QUQh zUxEpE4-`$JJQSZ;r%0MB7j%7Ld46u666E?;L!6I)@*n&BvP?*1ZXvmNrOw0Em52L{ zS`Kx??hOAVCueW^@;4*Qtfg&VZj`rbi%9ou=S2L4e?$L#mh%lcF>Lh=3ABi0m4!}L z;Zj(|>-qL?c`Roe1xPW7bACDjkMHgO{RvdxY(=OV9-hq7e)tkPm{X{@3GBdI?s=o) z>g&dD^bcQt9)oW8cy({NF@f(N`24ayvxCmtli?E0Yu;dK`N?O>1s!H)0fhkQ5b693 zY7X@6SN`qO3=MhmakDHaR4bVD2@>JM!H~W!!d*wes)MQ543!)2g?LYEoQ>T3Fult25 zTm*HmaRPAj2c2L0#DXw~wV!8zSJZp-CqMp!pEe6hA%TV<7bJM7I~Pk&EWG{jAH^-O zf%b(gO0TG3+)@C3;g-anJ2Cv||F%!ZwD!+|pO0p0`5c{v-2Z}!bnV5r+)jr1A&39c zrySVZI3Iu-wTt9sK>Mwucot36;LDWWxa5H&W6?lkvnz}K_o#(`EA6zSGzpc1G z^932^H8vqRr1*Cxi9MiO=|~wjz%=;`K+GxpnNFKd4S$mQkVN+K7phXq zho3QBua$PQH1|s{ZWHiI?RK}rm*4rew`s@z$Q*UQZU&EmZm~0a1gaClMD!Db29Tfs z`u^hQ95iN7DfQvUzrNG;0j!kVy+(fcwNEV`Y(pt;$VRB@kw}2Oefb}ML)@2-D|So8 z%aKd{;PAa){Z?%GCCP$OM4wRL8{{%m!hp#T5q$g)eo99W=jIO!vYgL(U$8Uhtb~OH#i0QNQd211X>z4CHI5!DxZyeyUh7WdSrPvbrrSiw;$q8 zI_`)y-R{FYp><(1g;}TasJ`F$5C2V*Kp*_&x={~Q2$3%; z6Wcg@nG1-G5FP*U^T>`2_F2)W?AurW%TF%?O|bk^!31#ndkIza@qhjWF;xMf!0pLr zYkBfse!T?lk9@NSlnp^Pe&0xgq5H0*w=r@O>bJl6GZ0CN1%2lV?hXx^m^N*}44x0q zxeh*i!Dn9v?&~Cvn}Maz&WU)GaM`VjxzR%X+nd)wc%l%3!3(A;(Atws>`V#A?8ZKA zysSU`N`vb^98@6N89g-Gjy~-_VUXgVGH1i0G1JSLQ0_&>BqnQW8XYqL3ZV5 zb>Onq?FyYvLqa?*sqyhwzXAyFJ3R+CgAvS2JszfG+w7V=3f)%?Cc?po-}ghikN@aD zEh2NNl=}#nonY|YTnGbs8~>>oO^NgKdPk6BHEO5VvC>az3wRoF_~ifj?q-mUL-Typ zUU3X<$o4NGE_Np70%2oIyW0Z8phmd;_-o!vx?Qkwb2_0q)MS_7-aUz!1W!^Fv&7Sr zX}sGC2Ui}D_=4niQJsthCu@}Z%Gau;0yOb%>*ZhmUtcKGa{+73nd_n`A)Y%MhuPV< zsl2z(*$j?EVLuQt{r2lW`o*ix(fA5h;CvbyGtX|WQjnw2E9wk$jX+;DA#O7$T)gR* zTMa!<4p9(nq7&O8E%ejJKlznrt`$8!es;#syrODUG?Yx>eqvZsh>H~ zp&(iId%kAY+VOamU4#GRzyD@e0piOa0_a8eDrq~-I~E9?Y4_pJnZzKYD6{DlIc~!J zGZ&WJxPI^ZShruD48(oB!RyDD%@@Xjbo1@eD8Z81%s3u;SY!uls~ZXpH;A#k?)~=F zeD&iW{H=(aq>M$FSnz@UB4DB4wh3lhC1K0Xb^M7;jP%^<;3{+5idP&6KF{{Ie*BeR zU=12PRK%dfshbRa*x}>b$OYGU`B*{v7QfI?UKb+c$u;_+O0Z-=`w#yHCT@`a(UDrf zSI_JBf9ccan2#sc(JvF%4Vg4apQxW>mwe$iXdTwjl?1$TFz&$v`UYyMnh19XGQIq+ z5C2@bo#lNd?CzOEY8?R}>Ivy$urM563i9D^esP^!AYva`Pwx7**70FtyG)9<-D5{) z>@K>Eet@=L`}hxkM{Essy-dM+7l=s<+;L<1m4SSq!ns#4jn$(kXd@p4-0dO4w#DncaN+dp}n0m@`vT zHBSJ42F=+n+P%yu8WE*0j+_lvy}(;}ipc%#cmIKJ9?KheV=O`AP{>2QD8myw;be%t z{rJ~VOK&RY1m5nmY<=&4{M>^;L2#DqorYc~1}2ut5R-f9e1d?kx2G28>L>r+rxr0j zZGsRrD8>a3H9Caz@R}Vpr3I!q_Ybwzcx(0r>Lt0D4LtM>9C zbvuv(AjovaUcT%1khT+@)y{ml%l;l?2^U(aa_>1}%)LFz}i&_8!Vaq{JU^8LZ$prGau zu*M>D$ZP^RfTw16jCKsC^XE7AFAq9kN&&CiAI4O_MLO5dN!bbbxGej262JHFf3w8M zF6np2Q`9>=wl~l6lch}SM?S{C`)f=4Dm(Jk8Rw7gPmyp*O{&>e7b!Px&kz6AulYm8 zIKcZ^_G43=1QohIY}ah%!AHEzO81UxK#;P=ETHAFfC#8X+w~8nFd|*_wl6w^Hzq+Fnx8L$lfa~{-+re?kJdVi6pZPKK zNmYQTs-MLLTW-#A2TP60-u|_}@VU!_?^Jg6sI!8rX1gVup2ox!cF?`I%$#pjG+1VS zD`*;LuR|pwh*?Q0p(e54$D^qtU`MU< z$OUjcF|@yZ_B+bJoPAlz&)`$YI}Kolxynmb@<4;L!zb;@8^C7hd?RfegjAWnf3kp@ zxqU`{&Be}O#0s?T-N!%n6R2_L7uFBEDd?h)f9h|uXPHo;Vj3;UF%X5}pZ_gvXrJ~> zp46_+3H4R^N%c;4*<-zC`MFrk$yhAnr?bFYjbN#5#v}=x3!a;J621s|$RSlRuX}JIPADTb-)VpF1 z;88DSLaAv{fTmC@0Fx~c@}&iTND|LyZ54gvrzJRoTl3*Oh6QmIz_8i#XcyAXf92NYX zN+6XOP3;p4O2zF-=`77m1<(RZjp>V!CbmOppO&IMYJiy~%0J)!h27VdGuWagp1xB) zh~I=pJx|N84tjpZ7aI_;L^R5k?tKCv)>LB?%%gX{a#ww=l|k;Y3%>tHzw5J7zm*En z!45rnN#z}Yj5NQ7i&g@qB!K0Q(8Bd)8f4(edAFAOiW4r>avD$GXxLzX@9+O8$e!wt z|Hbc(!N2&}_?L+-j+53#6jB+r8+BIodi#4AbZ_b!rHz1XAIH(Ffqo6z_0OH~*^PJW zlnC@>mFJhjGzUvL89o-%d;2cqbolmfGGsxq(NcCQe zo9DUQthsJpRp|8U02~h}zzULn>F9D$GXxOa#jSVZ1u(eLO7G+8rxiQjoGYSOxMAw~ zUN^{_`SItz@`F$3jkH4ow+1=LvF0pEK_Tp31SYg)e0e;VC0QpiSeQAE1={;2V9=oD57w2~G ztJe#B|Fk>w=7MQgeIFSF&wBOkuhTJRz_V;52TuSI>47`!@uZCH1~2uQBw=KbT=NG8OO#;I?TLit6dg z_usxLR&CTrgyEiaYa8dh1xzt#8{XbN$X{R+0U+sZmdgf!RKo%(i7olCAeRU-12BYw zBA#t&*e73z4N!k+$cDZMrVQ+eccWkbh4R$3hT7=aQhU2|RG%M-u08-WUTv50?VCR) z#%T&*QZu>CyR)!kW$VbjeJWi6&I!#wAg7UeH1)sw@$a_3Q;RoNzRRY!zwvjzz?o9H z=@!>@6L1c81i;IUx|0}YjvuL1>A{IW_h|e5zYfRgjenhI5(Vg&y8-``!}boQkegjd zQ0<4y*QWR(x)StT=O6wv1#BcEb8nG;44h7*cww*2XaCH1v6EKKX68KT4JzKA34*H+ zExy~^1ae}8&1%&|b zJiz&r8Z^1{^V{-sA_*JZJRewTm3@uRb8;A9iync!kpU0TLVNp znnS(){hvQwR0lGXmu;r=bdyPzd#?a8_~jSu?t#eyilNY+dFdA8<8OS$r1G%7%|T}F zobgyNwtZ_5xtmhfM4MTGMW$c?XUimD3F3iyDaOT z^Wx8bP6yNTTW}@VBR!YsSX$`*KyZKSTUr5WL32mh$!pR>issbo73Fv3^W$Io%6BSX zBsp?AS6q8Ib1NJ!cvyjNoNrI$AOA>rV5!NGBuOdPXR2`#aT6Zn;P$1=F;?~V2Ywc+4f17P zm{WIZ_gn3ibp&==K7Q>-2c#)*?a|RiZ^-U4qsV}7=2v%#mDfe0-i+PXK`S^$_r0sj z6?guwybUm29Ws+vx$LoZK^J{uo+lvSf;udIHuOY{Kl2g&6W?q|XYADtB&8WG!>)SH zsaBwl{bzsb2jAEwJ|uGFtK90ssugNR+?{6`i)z%v1ooNOzf)NA zm{*a}>EIF%1!i!IOU{Aoz~DQomMPBV64I)-grT{>)F1DrCa}Q+&O91yDBz8f4Q_`s zajm7(V`QKJjC)Si#_NU2XsX3@Blls@$wI76zsM$q=8ID=^(~+2yHUBEmU2BwA-do8S&qpnCw$7NKP`Ne7_+`ZYDjwUxBU7sH&i z(Si24qas3X#Dprozg_tS&>nU3AZJ~-VV zD}8`1J)gj&bGa_tB-+`1=_o0?{%Z@Eo`|-6c-@mQASuV@@?x9p?$6z3e+phNWLw?& za)ph|PhJ%3?U-J6Shq?&ysED$1DDw zV1Y7jN%3{rbe$_jSd)rtjsOlT`)OFbIQLqxFJ*y&i&#aU5%S>_5|saQ{HR)Q~=zHF6z=sWh-ws4)`mm1DZtIqi^Z?N%sE?60m>{49WbY*Z`)Md1rHAQpCz;y4Kf0>#PwfPp2!T^UZ>g6 zyUBe7r~{ST*NJ3ffG>H-TD_sN0-3xZquJEC7m^7n@SO+fq zYWd|FuWM+5PQc8HG_aDM1nDGzl)}Axva-&nQ_jQC z+$6WUHV4<{u%y+_Inue1IIlte-0rr;8rr^Yj~+pYbD9c_l@(Pm6%R%=2dVs7_`gyu zkh;*g8>l;(P}-bNyZIpRc)8fEOG`#`nl(W*t0pImD#}8abitDoE<#kGSp<&WT{r6w z#KFknKohq<>YP*^jVczwEJ;Z86`Y-%m0NpWC=5DV5U{j`;0%ddCWT%IpkBZSB6Uh* z_c9QB)GBBpimfH{4_##p4hP>>Yt*K?xkN}xu*NG3@G z^RP8050V|w^Joz-(!H#EwIEnWx0LK>8M)I78jrz`PkS;gNk(8dU^F5Wl%A=)E*s1( z)A$-H$VCC>W*On=n5|PbEyVDYE7Jt*bpUM1)*$l^eimQP(p*PMqrD2>zJgB0f>EV# zgL&>TRmhUkYqOqy-i~uYuFukSQBI6a>a_PqWgs;VrcD@YU%~9}*jTux)+_@Z+3T<* zSP(H+Yk{%da8)Pv>N*ekg?nhrQF^Hu`0_S6iqcOaqiuujB~dhNYG*r zya+DE!IT!dFUkN?uUFMv{}%<=@6jjA*H%B_D2ui)=%%fcb*%9 z1H_ntsvIwOf!}%?6bw#{VjF^%)ato_p7Q0E3dZ;t9_!b@e#||um#XQ^;Kn0l$h^!U z5!S=uUJ}nIwZEOYTc?Z1b~=-)_`Vo7$XOeUa3gVQ%h}$uTQ^UMZ-;H8G*6=RlHq`B=wn90M*xC0!?IVRGGl&PfLYiU{lEi3U*87zC*7JYk)x8MsrJrCRQw z5y99Y?{A6{vf6i|J?N&m$DX|(h$1>h%sLcHKRzM9Lli3589?&EvFE{iiRN6?=D6+_TQ(I_9{io4O;_6saYEbt`dB~*ha~?wI;79b z`%>&|@zFdNjxAOv7*!bf9TJA|9Zh%vmC3t4_V5q!mu7QF#E?KUvq54%Jc z?hLN%mC9_>v1j(j6n1OFul=dsyw-zZ$@IrL4zh7x4`PV2vKDgTv4#e^ytwI1deK5{ z&R*M~8qp|9NUhtyIhGXeL8JF;LTh%SoV0@+`a$H{Y2B;Fj-Y@sATS#KaIzu}oqp88 z{ivW{dt!#$^`7iG|FDLxv;&oAS%z9n9a$+{FEUmkrbpi(EuzM`|CXtFw{sf|kv7woKUMFrl z>a(TfTgAII*8t$|Mw@LQwFJ&p&QPZ(eGMYR{?0yQ20VD=T6yNJO10hv@3O{skW9O7 zhuwDl7+C7v9^gq1&b|@);4aFvb8P>Mdj`)N*f{_P9)(6l_~PUGt(wblmkutHwC)-LKl;QPC!!&Wl7TVu==3-B4R`??gw85nm2QM@26r|KK6;V#4#4g%3Vu5