From 2e598c7f9b0fedd93d539006690ad46644b85dd0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 5 Jul 2025 16:23:51 +0200 Subject: [PATCH 01/17] remove some gunk --- src/catalog_api.cpp | 9 +++------ src/include/catalog_api.hpp | 7 +------ src/include/storage/irc_schema_entry.hpp | 2 -- src/storage/irc_schema_set.cpp | 3 +-- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 701afa34..aaf1069c 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -90,8 +90,8 @@ vector IRCAPI::GetTables(ClientContext &conte return std::move(list_tables_response.identifiers); } -vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catalog) { - vector result; +vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catalog) { + vector result; auto endpoint_builder = catalog.GetBaseUrl(); endpoint_builder.AddPathComponent(catalog.prefix); endpoint_builder.AddPathComponent("namespaces"); @@ -110,16 +110,13 @@ vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catal } auto &schemas = list_namespaces_response.namespaces; for (auto &schema : schemas) { - IRCAPISchema schema_result; - schema_result.catalog_name = catalog.GetName(); auto &value = schema.value; if (value.size() != 1) { //! FIXME: we likely want to fix this by concatenating the components with a `.` ? throw NotImplementedException("Only a namespace with a single component is supported currently, found %d", value.size()); } - schema_result.schema_name = value[0]; - result.push_back(schema_result); + result.push_back(value[0]); } return result; diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 84602d96..7ba66d9f 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -12,11 +12,6 @@ namespace duckdb { class IRCatalog; -struct IRCAPISchema { - string schema_name; - string catalog_name; -}; - class IRCAPI { public: static const string API_VERSION_1; @@ -25,7 +20,7 @@ class IRCAPI { const string &schema); static rest_api_objects::LoadTableResult GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table_name); - static vector GetSchemas(ClientContext &context, IRCatalog &catalog); + static vector GetSchemas(ClientContext &context, IRCatalog &catalog); }; } // namespace duckdb diff --git a/src/include/storage/irc_schema_entry.hpp b/src/include/storage/irc_schema_entry.hpp index 8743dc80..dbdc8a19 100644 --- a/src/include/storage/irc_schema_entry.hpp +++ b/src/include/storage/irc_schema_entry.hpp @@ -13,8 +13,6 @@ class IRCSchemaEntry : public SchemaCatalogEntry { IRCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); ~IRCSchemaEntry() override; - unique_ptr schema_data; - public: optional_ptr CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override; optional_ptr CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) override; diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index fd21d6e6..c1e0a40a 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -38,10 +38,9 @@ void IRCSchemaSet::LoadEntries(ClientContext &context) { auto schemas = IRCAPI::GetSchemas(context, ic_catalog); for (const auto &schema : schemas) { CreateSchemaInfo info; - info.schema = schema.schema_name; + info.schema = schema; info.internal = false; auto schema_entry = make_uniq(catalog, info); - schema_entry->schema_data = make_uniq(schema); CreateEntryInternal(context, std::move(schema_entry)); } } From 260f95306705fde96f41bf3807d1c6c6c346ad9d Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 5 Jul 2025 17:08:39 +0200 Subject: [PATCH 02/17] always return schema and table entries --- src/catalog_api.cpp | 1 - src/include/storage/irc_catalog.hpp | 2 -- src/include/storage/irc_schema_entry.hpp | 2 ++ src/storage/irc_catalog.cpp | 35 ------------------------ src/storage/irc_schema_set.cpp | 13 +++++++-- src/storage/irc_table_set.cpp | 9 ++++-- 6 files changed, 19 insertions(+), 43 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index aaf1069c..1dfac2cf 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -49,7 +49,6 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); auto *root = yyjson_doc_get_root(doc.get()); auto load_table_result = rest_api_objects::LoadTableResult::FromJSON(root); - catalog.SetCachedValue(url, api_result, load_table_result); return api_result; } diff --git a/src/include/storage/irc_catalog.hpp b/src/include/storage/irc_catalog.hpp index f759e778..6c3fe6dc 100644 --- a/src/include/storage/irc_catalog.hpp +++ b/src/include/storage/irc_catalog.hpp @@ -44,8 +44,6 @@ class IRCatalog : public Catalog { static unique_ptr GetIcebergSecret(ClientContext &context, const string &secret_name); void GetConfig(ClientContext &context, IcebergEndpointType &endpoint_type); IRCEndpointBuilder GetBaseUrl() const; - string OptionalGetCachedValue(const string &url); - bool SetCachedValue(const string &url, const string &value, const rest_api_objects::LoadTableResult &result); public: static unique_ptr Attach(StorageExtensionInfo *storage_info, ClientContext &context, AttachedDatabase &db, diff --git a/src/include/storage/irc_schema_entry.hpp b/src/include/storage/irc_schema_entry.hpp index dbdc8a19..c16e2c34 100644 --- a/src/include/storage/irc_schema_entry.hpp +++ b/src/include/storage/irc_schema_entry.hpp @@ -39,6 +39,8 @@ class IRCSchemaEntry : public SchemaCatalogEntry { public: ICTableSet tables; + //! Whether this schema entry is verified as existing + bool verified = false; }; } // namespace duckdb diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index f178a3d1..97b83225 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -280,41 +280,6 @@ void IRCatalog::GetConfig(ClientContext &context, IcebergEndpointType &endpoint_ } } -string IRCatalog::OptionalGetCachedValue(const string &url) { - std::lock_guard lock(metadata_cache_mutex); - auto value = metadata_cache.find(url); - if (value != metadata_cache.end()) { - auto now = system_clock::now(); - if (now < value->second->expires_at) { - return value->second->data; - } - } - return ""; -} - -bool IRCatalog::SetCachedValue(const string &url, const string &value, - const rest_api_objects::LoadTableResult &result) { - //! FIXME: shouldn't this also store the 'storage-credentials' ?? - if (!result.has_config) { - return false; - } - auto &credentials = result.config; - auto expires_at_it = credentials.find("s3.session-token-expires-at-ms"); - if (expires_at_it == credentials.end()) { - return false; - } - - auto &expires_at = expires_at_it->second; - auto epochMillis = std::stoll(expires_at); - auto expired_time = system_clock::time_point(milliseconds(epochMillis)); - auto val = make_uniq(value, expired_time); - { - std::lock_guard lock(metadata_cache_mutex); - metadata_cache[url] = std::move(val); - } - return true; -} - //===--------------------------------------------------------------------===// // Attach //===--------------------------------------------------------------------===// diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index c1e0a40a..877cd8cc 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -12,11 +12,20 @@ IRCSchemaSet::IRCSchemaSet(Catalog &catalog) : catalog(catalog) { } optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const string &name) { - LoadEntries(context); lock_guard l(entry_lock); + auto &ic_catalog = catalog.Cast(); + auto entry = entries.find(name); if (entry == entries.end()) { - return nullptr; + //! We create the entry immediately optimistically, + //! when we scan from the table we'll figure out if it exists or not. + CreateSchemaInfo info; + info.schema = name; + info.internal = false; + auto schema_entry = make_uniq(catalog, info); + CreateEntryInternal(context, std::move(schema_entry)); + entry = entries.find(name); + D_ASSERT(entry != entries.end()); } return entry->second.get(); } diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index 8f5d4497..9a478626 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -270,11 +270,14 @@ unique_ptr ICTableSet::GetTableInfo(ClientContext &context, IRCSche } optional_ptr ICTableSet::GetEntry(ClientContext &context, const EntryLookupInfo &lookup) { - LoadEntries(context); lock_guard l(entry_lock); - auto entry = entries.find(lookup.GetEntryName()); + auto &ic_catalog = catalog.Cast(); + + auto table_name = lookup.GetEntryName(); + auto entry = entries.find(table_name); if (entry == entries.end()) { - return nullptr; + auto it = entries.emplace(table_name, IcebergTableInformation(ic_catalog, schema, table_name)); + entry = it.first; } FillEntry(context, entry->second); return entry->second.GetSchemaVersion(lookup.GetAtClause()); From c1f0ded60ee7b9610b54bfad209866371e9690fc Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 5 Jul 2025 17:26:27 +0200 Subject: [PATCH 03/17] make a HEAD request to verify the existence of the namespace --- src/aws.cpp | 49 ++++++++++++++++++++ src/catalog_api.cpp | 15 ++++++ src/common/api_utils.cpp | 21 +++++++++ src/include/api_utils.hpp | 2 + src/include/aws.hpp | 1 + src/include/storage/authorization/none.hpp | 1 + src/include/storage/authorization/oauth2.hpp | 1 + src/include/storage/authorization/sigv4.hpp | 1 + src/include/storage/irc_authorization.hpp | 2 + src/include/storage/irc_table_set.hpp | 2 + src/storage/authorization/none.cpp | 5 ++ src/storage/authorization/oauth2.cpp | 5 ++ src/storage/authorization/sigv4.cpp | 37 +++++++++++++++ 13 files changed, 142 insertions(+) diff --git a/src/aws.cpp b/src/aws.cpp index b5aaf08b..252870f1 100644 --- a/src/aws.cpp +++ b/src/aws.cpp @@ -128,6 +128,55 @@ unique_ptr AWSInput::GetRequest(ClientContext &context) { return result; } +unique_ptr AWSInput::HeadRequest(ClientContext &context) { + InitAWSAPI(); + auto clientConfig = make_uniq(); + + if (!cert_path.empty()) { + clientConfig->caFile = cert_path; + } + + Aws::Http::URI uri; + Aws::Http::Scheme scheme = Aws::Http::Scheme::HTTPS; + uri.SetScheme(scheme); + uri.SetAuthority(authority); + for (auto &segment : path_segments) { + uri.AddPathSegment(segment); + } + + for (auto ¶m : query_string_parameters) { + uri.AddQueryStringParameter(param.first.c_str(), param.second.c_str()); + } + + std::shared_ptr provider; + provider = std::make_shared(key_id, secret, session_token); + auto signer = make_uniq(provider, service.c_str(), region.c_str()); + + const Aws::Http::URI uri_const = Aws::Http::URI(uri); + auto request = Aws::Http::CreateHttpRequest(uri_const, Aws::Http::HttpMethod::HTTP_HEAD, + Aws::Utils::Stream::DefaultResponseStreamFactoryMethod); + request->SetUserAgent(user_agent); + + signer->SignRequest(*request); + + std::shared_ptr MyHttpClient; + MyHttpClient = Aws::Http::CreateHttpClient(*clientConfig); + + LogAWSRequest(context, request); + std::shared_ptr res = MyHttpClient->MakeRequest(request); + Aws::Http::HttpResponseCode resCode = res->GetResponseCode(); + DUCKDB_LOG(context, IcebergLogType, + "GET %s (response %d) (signed with key_id '%s' for service '%s', in region '%s')", uri.GetURIString(), + resCode, key_id, service.c_str(), region.c_str()); + + unique_ptr result = make_uniq(HTTPStatusCode(static_cast(resCode))); + result->url = uri.GetURIString(); + Aws::StringStream resBody; + resBody << res->GetResponseBody().rdbuf(); + result->body = resBody.str(); + return result; +} + unique_ptr AWSInput::PostRequest(ClientContext &context, string post_body) { InitAWSAPI(); diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 1dfac2cf..49a716a6 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -31,6 +31,20 @@ namespace duckdb { int(response.status)); } +void VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema) { + auto url_builder = catalog.GetBaseUrl(); + url_builder.AddPathComponent(catalog.prefix); + url_builder.AddPathComponent("namespaces"); + url_builder.AddPathComponent(schema); + + auto url = url_builder.GetURL(); + auto response = catalog.auth_handler->HeadRequest(context, url_builder); + if (response->Success()) { + return; + } + ThrowException(url, *response, "HEAD"); +} + static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table) { auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); @@ -42,6 +56,7 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const auto url = url_builder.GetURL(); auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { + VerifySchemaExistence(context, catalog, schema); ThrowException(url, *response, "GET"); } diff --git a/src/common/api_utils.cpp b/src/common/api_utils.cpp index 961037ed..a3ab5c9f 100644 --- a/src/common/api_utils.cpp +++ b/src/common/api_utils.cpp @@ -97,4 +97,25 @@ unique_ptr APIUtils::GetRequest(ClientContext &context, const IRCE return http_util.Request(get_request); } +unique_ptr APIUtils::HeadRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, + const string &token) { + auto &db = DatabaseInstance::GetDatabase(context); + + HTTPHeaders headers(db); + headers.Insert("X-Iceberg-Access-Delegation", "vended-credentials"); + if (!token.empty()) { + headers.Insert("Authorization", StringUtil::Format("Bearer %s", token)); + } + + auto &http_util = HTTPUtil::Get(db); + unique_ptr params; + + string request_url = AddHttpHostIfMissing(endpoint_builder.GetURL()); + + params = http_util.InitializeParameters(context, request_url); + + HeadRequestInfo head_request(request_url, headers, *params); + return http_util.Request(head_request); +} + } // namespace duckdb diff --git a/src/include/api_utils.hpp b/src/include/api_utils.hpp index 45e2e777..56f4f27d 100644 --- a/src/include/api_utils.hpp +++ b/src/include/api_utils.hpp @@ -35,6 +35,8 @@ class APIUtils { public: static unique_ptr GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &token = ""); + static unique_ptr HeadRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, + const string &token = ""); static unique_ptr DeleteRequest(ClientContext &context, const string &url, const string &token = ""); static unique_ptr PostRequest(ClientContext &context, const string &url, const string &post_data, const unordered_map &additional_headers, diff --git a/src/include/aws.hpp b/src/include/aws.hpp index 6abf1165..d5eb7fc4 100644 --- a/src/include/aws.hpp +++ b/src/include/aws.hpp @@ -12,6 +12,7 @@ class AWSInput { public: unique_ptr GetRequest(ClientContext &context); + unique_ptr HeadRequest(ClientContext &context); unique_ptr PostRequest(ClientContext &context, string post_body); public: diff --git a/src/include/storage/authorization/none.hpp b/src/include/storage/authorization/none.hpp index cb32718a..f81bcaae 100644 --- a/src/include/storage/authorization/none.hpp +++ b/src/include/storage/authorization/none.hpp @@ -14,6 +14,7 @@ class NoneAuthorization : public IRCAuthorization { public: static unique_ptr FromAttachOptions(IcebergAttachOptions &input); unique_ptr GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder) override; + unique_ptr HeadRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder) override; unique_ptr PostRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &body) override; }; diff --git a/src/include/storage/authorization/oauth2.hpp b/src/include/storage/authorization/oauth2.hpp index 6ec53390..88f9ce00 100644 --- a/src/include/storage/authorization/oauth2.hpp +++ b/src/include/storage/authorization/oauth2.hpp @@ -16,6 +16,7 @@ class OAuth2Authorization : public IRCAuthorization { public: static unique_ptr FromAttachOptions(ClientContext &context, IcebergAttachOptions &input); unique_ptr GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder) override; + unique_ptr HeadRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder) override; unique_ptr PostRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &body) override; static string GetToken(ClientContext &context, const string &grant_type, const string &uri, const string &client_id, diff --git a/src/include/storage/authorization/sigv4.hpp b/src/include/storage/authorization/sigv4.hpp index 54d8b4d2..1de6ac99 100644 --- a/src/include/storage/authorization/sigv4.hpp +++ b/src/include/storage/authorization/sigv4.hpp @@ -15,6 +15,7 @@ class SIGV4Authorization : public IRCAuthorization { public: static unique_ptr FromAttachOptions(IcebergAttachOptions &input); unique_ptr GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder) override; + unique_ptr HeadRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder) override; unique_ptr PostRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &body) override; diff --git a/src/include/storage/irc_authorization.hpp b/src/include/storage/irc_authorization.hpp index 3b5b4499..3316542e 100644 --- a/src/include/storage/irc_authorization.hpp +++ b/src/include/storage/irc_authorization.hpp @@ -33,6 +33,8 @@ struct IRCAuthorization { public: virtual unique_ptr GetRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder) = 0; + virtual unique_ptr HeadRequest(ClientContext &context, + const IRCEndpointBuilder &endpoint_builder) = 0; virtual unique_ptr PostRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &body) = 0; diff --git a/src/include/storage/irc_table_set.hpp b/src/include/storage/irc_table_set.hpp index 0678d7a9..0539862e 100644 --- a/src/include/storage/irc_table_set.hpp +++ b/src/include/storage/irc_table_set.hpp @@ -32,6 +32,8 @@ struct IcebergTableInformation { IRCSchemaEntry &schema; string name; string table_id; + //! Whether this table entry is verified as existing + bool verified = false; rest_api_objects::LoadTableResult load_table_result; IcebergTableMetadata table_metadata; diff --git a/src/storage/authorization/none.cpp b/src/storage/authorization/none.cpp index 351ad9dc..ff80a1f3 100644 --- a/src/storage/authorization/none.cpp +++ b/src/storage/authorization/none.cpp @@ -17,6 +17,11 @@ unique_ptr NoneAuthorization::GetRequest(ClientContext &context, return APIUtils::GetRequest(context, endpoint_builder, ""); } +unique_ptr NoneAuthorization::HeadRequest(ClientContext &context, + const IRCEndpointBuilder &endpoint_builder) { + return APIUtils::HeadRequest(context, endpoint_builder, ""); +} + unique_ptr NoneAuthorization::PostRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &body) { auto url = endpoint_builder.GetURL(); diff --git a/src/storage/authorization/oauth2.cpp b/src/storage/authorization/oauth2.cpp index 8e7e4436..973bb752 100644 --- a/src/storage/authorization/oauth2.cpp +++ b/src/storage/authorization/oauth2.cpp @@ -103,6 +103,11 @@ unique_ptr OAuth2Authorization::GetRequest(ClientContext &context, return APIUtils::GetRequest(context, endpoint_builder, token); } +unique_ptr OAuth2Authorization::HeadRequest(ClientContext &context, + const IRCEndpointBuilder &endpoint_builder) { + return APIUtils::HeadRequest(context, endpoint_builder, token); +} + unique_ptr OAuth2Authorization::PostRequest(ClientContext &context, const IRCEndpointBuilder &endpoint_builder, const string &body) { diff --git a/src/storage/authorization/sigv4.cpp b/src/storage/authorization/sigv4.cpp index 656c49eb..0032e6bb 100644 --- a/src/storage/authorization/sigv4.cpp +++ b/src/storage/authorization/sigv4.cpp @@ -139,4 +139,41 @@ unique_ptr SIGV4Authorization::GetRequest(ClientContext &context, return aws_input.GetRequest(context); } +unique_ptr SIGV4Authorization::HeadRequest(ClientContext &context, + const IRCEndpointBuilder &endpoint_builder) { + AWSInput aws_input; + aws_input.cert_path = APIUtils::GetCURLCertPath(); + // Set the user Agent. + auto &config = DBConfig::GetConfig(context); + aws_input.user_agent = config.UserAgent(); + + aws_input.service = GetAwsService(endpoint_builder.GetHost()); + aws_input.region = GetAwsRegion(endpoint_builder.GetHost()); + + auto decomposed_host = DecomposeHost(endpoint_builder.GetHost()); + + aws_input.authority = decomposed_host.authority; + for (auto &component : decomposed_host.path_components) { + aws_input.path_segments.push_back(component); + } + for (auto &component : endpoint_builder.path_components) { + aws_input.path_segments.push_back(component); + } + + for (auto ¶m : endpoint_builder.GetParams()) { + aws_input.query_string_parameters.emplace_back(param); + } + + // will error if no secret can be found for AWS services + auto secret_entry = IRCatalog::GetStorageSecret(context, secret); + auto kv_secret = dynamic_cast(*secret_entry->secret); + + aws_input.key_id = kv_secret.secret_map["key_id"].GetValue(); + aws_input.secret = kv_secret.secret_map["secret"].GetValue(); + aws_input.session_token = + kv_secret.secret_map["session_token"].IsNull() ? "" : kv_secret.secret_map["session_token"].GetValue(); + + return aws_input.HeadRequest(context); +} + } // namespace duckdb From 1a7cd93764b8e025dfdd0eb2f62e58f321a3c750 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 5 Jul 2025 17:32:15 +0200 Subject: [PATCH 04/17] throw a CatalogException --- src/catalog_api.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 49a716a6..7d3f4d99 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -42,7 +42,7 @@ void VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const str if (response->Success()) { return; } - ThrowException(url, *response, "HEAD"); + throw CatalogException("Namespace by the name of '%s' does not exist", schema); } static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table) { @@ -57,7 +57,7 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { VerifySchemaExistence(context, catalog, schema); - ThrowException(url, *response, "GET"); + throw CatalogException("Table by the name of '%s' in namespace '%s' doesn't exist", table, schema); } const auto &api_result = response->body; From bbb60c15a1289e8aaaba3ddb1f00dbf4ac9f6721 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 5 Jul 2025 18:14:23 +0200 Subject: [PATCH 05/17] remember the OnEntryNotFound type used when looking up the schema, to enforce it when we find out the schema doesnt exist --- src/catalog_api.cpp | 40 +++++++++++++++++------- src/include/catalog_api.hpp | 5 +-- src/include/storage/irc_schema_entry.hpp | 12 +++++-- src/include/storage/irc_schema_set.hpp | 2 +- src/include/storage/irc_table_set.hpp | 2 +- src/storage/irc_catalog.cpp | 2 +- src/storage/irc_schema_set.cpp | 4 ++- src/storage/irc_table_set.cpp | 15 ++++++--- 8 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 7d3f4d99..55c16371 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -1,6 +1,7 @@ #include "catalog_api.hpp" #include "catalog_utils.hpp" #include "storage/irc_catalog.hpp" +#include "storage/irc_schema_entry.hpp" #include "yyjson.hpp" #include "iceberg_utils.hpp" #include "api_utils.hpp" @@ -31,7 +32,7 @@ namespace duckdb { int(response.status)); } -void VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema) { +bool VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema) { auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); url_builder.AddPathComponent("namespaces"); @@ -40,24 +41,38 @@ void VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const str auto url = url_builder.GetURL(); auto response = catalog.auth_handler->HeadRequest(context, url_builder); if (response->Success()) { - return; + return true; } - throw CatalogException("Namespace by the name of '%s' does not exist", schema); + return false; } -static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const string &schema, const string &table) { +static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, + const string &table) { + const auto &schema_name = schema.name; + auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); url_builder.AddPathComponent("namespaces"); - url_builder.AddPathComponent(schema); + url_builder.AddPathComponent(schema_name); url_builder.AddPathComponent("tables"); url_builder.AddPathComponent(table); auto url = url_builder.GetURL(); auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { - VerifySchemaExistence(context, catalog, schema); - throw CatalogException("Table by the name of '%s' in namespace '%s' doesn't exist", table, schema); + if (schema.existence_state.type == SchemaExistenceType::UNKNOWN) { + bool exists = VerifySchemaExistence(context, catalog, schema_name); + if (exists) { + schema.existence_state.type = SchemaExistenceType::PRESENT; + } else { + schema.existence_state.type = SchemaExistenceType::MISSING; + if (schema.existence_state.if_not_found == OnEntryNotFound::RETURN_NULL) { + return string(); + } + throw CatalogException("Namespace by the name of '%s' does not exist", schema_name); + } + } + return string(); } const auto &api_result = response->body; @@ -71,13 +86,16 @@ vector IRCAPI::GetCatalogs(ClientContext &context, IRCatalog &catalog) { throw NotImplementedException("ICAPI::GetCatalogs"); } -rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, - const string &table_name) { +bool IRCAPI::GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, const string &table_name, + rest_api_objects::LoadTableResult &out) { string result = GetTableMetadata(context, catalog, schema, table_name); + if (result.empty()) { + return false; + } std::unique_ptr doc(ICUtils::api_result_to_doc(result)); auto *metadata_root = yyjson_doc_get_root(doc.get()); - auto load_table_result = rest_api_objects::LoadTableResult::FromJSON(metadata_root); - return load_table_result; + out = rest_api_objects::LoadTableResult::FromJSON(metadata_root); + return true; } // TODO: handle out-of-order columns using position property diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 7ba66d9f..ec5f0e9a 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -11,6 +11,7 @@ namespace duckdb { class IRCatalog; +class IRCSchemaEntry; class IRCAPI { public: @@ -18,8 +19,8 @@ class IRCAPI { static vector GetCatalogs(ClientContext &context, IRCatalog &catalog); static vector GetTables(ClientContext &context, IRCatalog &catalog, const string &schema); - static rest_api_objects::LoadTableResult GetTable(ClientContext &context, IRCatalog &catalog, const string &schema, - const string &table_name); + static bool GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, const string &table_name, + rest_api_objects::LoadTableResult &out); static vector GetSchemas(ClientContext &context, IRCatalog &catalog); }; diff --git a/src/include/storage/irc_schema_entry.hpp b/src/include/storage/irc_schema_entry.hpp index c16e2c34..c16846e6 100644 --- a/src/include/storage/irc_schema_entry.hpp +++ b/src/include/storage/irc_schema_entry.hpp @@ -4,10 +4,19 @@ #include "catalog_api.hpp" #include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" #include "storage/irc_table_set.hpp" +#include "duckdb/common/enums/on_entry_not_found.hpp" namespace duckdb { class IRCTransaction; +enum class SchemaExistenceType : uint8_t { UNKNOWN, PRESENT, MISSING }; + +struct SchemaExistenceState { +public: + SchemaExistenceType type = SchemaExistenceType::UNKNOWN; + OnEntryNotFound if_not_found; +}; + class IRCSchemaEntry : public SchemaCatalogEntry { public: IRCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); @@ -39,8 +48,7 @@ class IRCSchemaEntry : public SchemaCatalogEntry { public: ICTableSet tables; - //! Whether this schema entry is verified as existing - bool verified = false; + SchemaExistenceState existence_state; }; } // namespace duckdb diff --git a/src/include/storage/irc_schema_set.hpp b/src/include/storage/irc_schema_set.hpp index ddb52688..e50fe2ac 100644 --- a/src/include/storage/irc_schema_set.hpp +++ b/src/include/storage/irc_schema_set.hpp @@ -12,7 +12,7 @@ class IRCSchemaSet { public: void LoadEntries(ClientContext &context); - optional_ptr GetEntry(ClientContext &context, const string &name); + optional_ptr GetEntry(ClientContext &context, const string &name, OnEntryNotFound if_not_found); void Scan(ClientContext &context, const std::function &callback); protected: diff --git a/src/include/storage/irc_table_set.hpp b/src/include/storage/irc_table_set.hpp index 0539862e..75743e89 100644 --- a/src/include/storage/irc_table_set.hpp +++ b/src/include/storage/irc_table_set.hpp @@ -55,7 +55,7 @@ class ICTableSet { public: void LoadEntries(ClientContext &context); - void FillEntry(ClientContext &context, IcebergTableInformation &table); + bool FillEntry(ClientContext &context, IcebergTableInformation &table); public: IRCSchemaEntry &schema; diff --git a/src/storage/irc_catalog.cpp b/src/storage/irc_catalog.cpp index 97b83225..0fcea92a 100644 --- a/src/storage/irc_catalog.cpp +++ b/src/storage/irc_catalog.cpp @@ -56,7 +56,7 @@ optional_ptr IRCatalog::LookupSchema(CatalogTransaction tran auto &schemas = irc_transaction.GetSchemas(); auto &schema_name = schema_lookup.GetEntryName(); - auto entry = schemas.GetEntry(transaction.GetContext(), schema_name); + auto entry = schemas.GetEntry(transaction.GetContext(), schema_name, if_not_found); if (!entry && if_not_found != OnEntryNotFound::RETURN_NULL) { throw CatalogException(schema_lookup.GetErrorContext(), "Schema with name \"%s\" not found", schema_name); } diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index 877cd8cc..29ca6c7b 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -11,7 +11,8 @@ namespace duckdb { IRCSchemaSet::IRCSchemaSet(Catalog &catalog) : catalog(catalog) { } -optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const string &name) { +optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const string &name, + OnEntryNotFound if_not_found) { lock_guard l(entry_lock); auto &ic_catalog = catalog.Cast(); @@ -23,6 +24,7 @@ optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const info.schema = name; info.internal = false; auto schema_entry = make_uniq(catalog, info); + schema_entry->existence_state.if_not_found = if_not_found; CreateEntryInternal(context, std::move(schema_entry)); entry = entries.find(name); D_ASSERT(entry != entries.end()); diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index 9a478626..b0743318 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -221,14 +221,15 @@ optional_ptr IcebergTableInformation::GetSchemaVersion(optional_pt ICTableSet::ICTableSet(IRCSchemaEntry &schema) : schema(schema), catalog(schema.ParentCatalog()) { } -void ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &table) { +bool ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &table) { if (!table.schema_versions.empty()) { - //! Already filled - return; + return true; } auto &ic_catalog = catalog.Cast(); - table.load_table_result = IRCAPI::GetTable(context, ic_catalog, schema.name, table.name); + if (!IRCAPI::GetTable(context, ic_catalog, schema, table.name, table.load_table_result)) { + return false; + } table.table_metadata = IcebergTableMetadata::FromTableMetadata(table.load_table_result.metadata); auto &schemas = table.table_metadata.schemas; @@ -237,6 +238,7 @@ void ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &tabl for (auto &table_schema : schemas) { table.CreateSchemaVersion(*table_schema.second); } + return true; } void ICTableSet::Scan(ClientContext &context, const std::function &callback) { @@ -279,7 +281,10 @@ optional_ptr ICTableSet::GetEntry(ClientContext &context, const En auto it = entries.emplace(table_name, IcebergTableInformation(ic_catalog, schema, table_name)); entry = it.first; } - FillEntry(context, entry->second); + if (!FillEntry(context, entry->second)) { + //! The namespace or table doesn't exist and the schema lookup said OnEntryNotFound::RETURN_NULL + return nullptr; + } return entry->second.GetSchemaVersion(lookup.GetAtClause()); } From 64538055842c5887260f8bbf77293cb11dcb8bb4 Mon Sep 17 00:00:00 2001 From: Tishj Date: Sat, 5 Jul 2025 18:44:38 +0200 Subject: [PATCH 06/17] fix some issues with USE and GetTables --- src/include/storage/irc_table_set.hpp | 4 ++-- src/storage/irc_table_set.cpp | 16 +++++++++++++--- .../local/irc/iceberg_catalog_eager_refresh.test | 6 ------ test/sql/local/irc/iceberg_catalog_read.test | 4 ---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/include/storage/irc_table_set.hpp b/src/include/storage/irc_table_set.hpp index 75743e89..cc68a538 100644 --- a/src/include/storage/irc_table_set.hpp +++ b/src/include/storage/irc_table_set.hpp @@ -32,8 +32,6 @@ struct IcebergTableInformation { IRCSchemaEntry &schema; string name; string table_id; - //! Whether this table entry is verified as existing - bool verified = false; rest_api_objects::LoadTableResult load_table_result; IcebergTableMetadata table_metadata; @@ -61,6 +59,8 @@ class ICTableSet { IRCSchemaEntry &schema; Catalog &catalog; case_insensitive_map_t entries; + //! Whether a listing is done for this transaction + bool listed = false; private: mutex entry_lock; diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index b0743318..f1951aac 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -204,7 +204,9 @@ optional_ptr IcebergTableInformation::CreateSchemaVersion(IcebergT } optional_ptr IcebergTableInformation::GetSchemaVersion(optional_ptr at) { - D_ASSERT(!schema_versions.empty()); + if (schema_versions.empty()) { + return nullptr; + } auto snapshot_lookup = IcebergSnapshotLookup::FromAtClause(at); int32_t schema_id; @@ -248,12 +250,16 @@ void ICTableSet::Scan(ClientContext &context, const std::function ICTableSet::GetTableInfo(ClientContext &context, IRCSchemaEntry &schema, diff --git a/test/sql/local/irc/iceberg_catalog_eager_refresh.test b/test/sql/local/irc/iceberg_catalog_eager_refresh.test index 1389b9f7..fe396f08 100644 --- a/test/sql/local/irc/iceberg_catalog_eager_refresh.test +++ b/test/sql/local/irc/iceberg_catalog_eager_refresh.test @@ -58,8 +58,6 @@ select * from my_datalake.default.table_more_deletes order by all; query II SELECT request.url, response.reason FROM duckdb_logs_parsed('HTTP') WHERE request.type='GET' AND (request.url).starts_with('http://127.0.0.1:8181/v1/') order by timestamp ---- -http://127.0.0.1:8181/v1/namespaces OK -http://127.0.0.1:8181/v1/namespaces/default/tables OK http://127.0.0.1:8181/v1/namespaces/default/tables/table_unpartitioned OK http://127.0.0.1:8181/v1/namespaces/default/tables/table_more_deletes OK @@ -91,9 +89,5 @@ select * from my_datalake.default.table_more_deletes order by all; query II SELECT request.url, response.reason FROM duckdb_logs_parsed('HTTP') WHERE request.type='GET' AND (request.url).starts_with('http://127.0.0.1:8181/v1/') order by timestamp ---- -http://127.0.0.1:8181/v1/namespaces OK -http://127.0.0.1:8181/v1/namespaces/default/tables OK http://127.0.0.1:8181/v1/namespaces/default/tables/table_more_deletes OK -http://127.0.0.1:8181/v1/namespaces OK -http://127.0.0.1:8181/v1/namespaces/default/tables OK http://127.0.0.1:8181/v1/namespaces/default/tables/table_more_deletes OK diff --git a/test/sql/local/irc/iceberg_catalog_read.test b/test/sql/local/irc/iceberg_catalog_read.test index 716ac727..3f10361d 100644 --- a/test/sql/local/irc/iceberg_catalog_read.test +++ b/test/sql/local/irc/iceberg_catalog_read.test @@ -88,11 +88,7 @@ select * from my_datalake.default.table_more_deletes order by all; query II SELECT request.url, response.reason FROM duckdb_logs_parsed('HTTP') WHERE request.type='GET' AND (request.url).starts_with('http://127.0.0.1:8181/v1/') order by timestamp ---- -http://127.0.0.1:8181/v1/namespaces OK -http://127.0.0.1:8181/v1/namespaces/default/tables OK http://127.0.0.1:8181/v1/namespaces/default/tables/table_unpartitioned OK -http://127.0.0.1:8181/v1/namespaces OK -http://127.0.0.1:8181/v1/namespaces/default/tables OK http://127.0.0.1:8181/v1/namespaces/default/tables/table_more_deletes OK query I From d135e44ddaf2086b49aa858e64051fd13081e80a Mon Sep 17 00:00:00 2001 From: Tishj Date: Mon, 7 Jul 2025 10:35:36 +0200 Subject: [PATCH 07/17] attempt to check schemas at the end of the query - does not work because the callback gets issued AFTER the transaction is destroyed --- CMakeLists.txt | 1 + src/catalog_api.cpp | 37 ++++++++++++++--------- src/include/catalog_api.hpp | 5 +-- src/include/storage/irc_context_state.hpp | 27 +++++++++++++++++ src/include/storage/irc_schema_set.hpp | 1 + src/include/storage/irc_table_set.hpp | 2 +- src/include/storage/irc_transaction.hpp | 1 + src/storage/irc_context_state.cpp | 29 ++++++++++++++++++ src/storage/irc_schema_set.cpp | 18 +++++++++++ src/storage/irc_table_set.cpp | 15 ++++++--- src/storage/irc_transaction.cpp | 25 +++++++++------ test/sql/local/irc/test_use.test | 13 +++++--- 12 files changed, 137 insertions(+), 37 deletions(-) create mode 100644 src/include/storage/irc_context_state.hpp create mode 100644 src/storage/irc_context_state.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e215c4a..f18506b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,7 @@ set(EXTENSION_SOURCES src/storage/irc_table_set.cpp src/storage/irc_transaction.cpp src/storage/irc_authorization.cpp + src/storage/irc_context_state.cpp src/storage/irc_transaction_manager.cpp src/storage/table_update/iceberg_add_snapshot.cpp ) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 55c16371..3e042554 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -32,7 +32,7 @@ namespace duckdb { int(response.status)); } -bool VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema) { +bool IRCAPI::VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema) { auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); url_builder.AddPathComponent("namespaces"); @@ -61,7 +61,7 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, IRCSc auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { if (schema.existence_state.type == SchemaExistenceType::UNKNOWN) { - bool exists = VerifySchemaExistence(context, catalog, schema_name); + bool exists = IRCAPI::VerifySchemaExistence(context, catalog, schema_name); if (exists) { schema.existence_state.type = SchemaExistenceType::PRESENT; } else { @@ -75,11 +75,7 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, IRCSc return string(); } - const auto &api_result = response->body; - std::unique_ptr doc(ICUtils::api_result_to_doc(api_result)); - auto *root = yyjson_doc_get_root(doc.get()); - auto load_table_result = rest_api_objects::LoadTableResult::FromJSON(root); - return api_result; + return response->body; } vector IRCAPI::GetCatalogs(ClientContext &context, IRCatalog &catalog) { @@ -98,18 +94,30 @@ bool IRCAPI::GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry return true; } -// TODO: handle out-of-order columns using position property -vector IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, - const string &schema) { +bool IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, + vector &out) { + auto schema_name = schema.name; + auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); url_builder.AddPathComponent("namespaces"); - url_builder.AddPathComponent(schema); + url_builder.AddPathComponent(schema_name); url_builder.AddPathComponent("tables"); auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { - auto url = url_builder.GetURL(); - ThrowException(url, *response, "GET"); + if (schema.existence_state.type == SchemaExistenceType::UNKNOWN) { + bool exists = VerifySchemaExistence(context, catalog, schema_name); + if (exists) { + schema.existence_state.type = SchemaExistenceType::PRESENT; + } else { + schema.existence_state.type = SchemaExistenceType::MISSING; + if (schema.existence_state.if_not_found == OnEntryNotFound::RETURN_NULL) { + return false; + } + throw CatalogException("Namespace by the name of '%s' does not exist", schema_name); + } + } + return false; } std::unique_ptr doc(ICUtils::api_result_to_doc(response->body)); @@ -119,7 +127,8 @@ vector IRCAPI::GetTables(ClientContext &conte if (!list_tables_response.has_identifiers) { throw NotImplementedException("List of 'identifiers' is missing, missing support for Iceberg V1"); } - return std::move(list_tables_response.identifiers); + out = std::move(list_tables_response.identifiers); + return true; } vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catalog) { diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index ec5f0e9a..0cba1bf8 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -17,11 +17,12 @@ class IRCAPI { public: static const string API_VERSION_1; static vector GetCatalogs(ClientContext &context, IRCatalog &catalog); - static vector GetTables(ClientContext &context, IRCatalog &catalog, - const string &schema); + static bool GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, + vector &out); static bool GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, const string &table_name, rest_api_objects::LoadTableResult &out); static vector GetSchemas(ClientContext &context, IRCatalog &catalog); + static bool VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema); }; } // namespace duckdb diff --git a/src/include/storage/irc_context_state.hpp b/src/include/storage/irc_context_state.hpp new file mode 100644 index 00000000..13758006 --- /dev/null +++ b/src/include/storage/irc_context_state.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "duckdb/main/client_context_state.hpp" + +namespace duckdb { + +class IRCSchemaEntry; + +class IRCContextState : public ClientContextState { +public: + IRCContextState() { + } + +public: + void QueryEnd(ClientContext &context) override; + void QueryEnd(ClientContext &context, optional_ptr error) override { + QueryEnd(context); + } + +public: + void RegisterSchema(IRCSchemaEntry &schema); + +public: + vector> schemas; +}; + +} // namespace duckdb diff --git a/src/include/storage/irc_schema_set.hpp b/src/include/storage/irc_schema_set.hpp index e50fe2ac..87cbe912 100644 --- a/src/include/storage/irc_schema_set.hpp +++ b/src/include/storage/irc_schema_set.hpp @@ -14,6 +14,7 @@ class IRCSchemaSet { void LoadEntries(ClientContext &context); optional_ptr GetEntry(ClientContext &context, const string &name, OnEntryNotFound if_not_found); void Scan(ClientContext &context, const std::function &callback); + void VerifySchemas(ClientContext &context); protected: optional_ptr CreateEntryInternal(ClientContext &context, unique_ptr entry); diff --git a/src/include/storage/irc_table_set.hpp b/src/include/storage/irc_table_set.hpp index cc68a538..53a39b59 100644 --- a/src/include/storage/irc_table_set.hpp +++ b/src/include/storage/irc_table_set.hpp @@ -52,7 +52,7 @@ class ICTableSet { void Scan(ClientContext &context, const std::function &callback); public: - void LoadEntries(ClientContext &context); + bool LoadEntries(ClientContext &context); bool FillEntry(ClientContext &context, IcebergTableInformation &table); public: diff --git a/src/include/storage/irc_transaction.hpp b/src/include/storage/irc_transaction.hpp index 649ad7eb..2be4446d 100644 --- a/src/include/storage/irc_transaction.hpp +++ b/src/include/storage/irc_transaction.hpp @@ -31,6 +31,7 @@ class IRCTransaction : public Transaction { private: void CleanupFiles(); + void CommitToRESTCatalog(Connection &connection, ClientContext &context); private: DatabaseInstance &db; diff --git a/src/storage/irc_context_state.cpp b/src/storage/irc_context_state.cpp new file mode 100644 index 00000000..e4afe2c7 --- /dev/null +++ b/src/storage/irc_context_state.cpp @@ -0,0 +1,29 @@ +#include "storage/irc_context_state.hpp" + +#include "storage/irc_schema_entry.hpp" +#include "storage/irc_catalog.hpp" +#include "duckdb/common/enums/on_entry_not_found.hpp" +#include "catalog_api.hpp" + +namespace duckdb { + +void IRCContextState::QueryEnd(ClientContext &context) { + for (auto &it : schemas) { + auto &schema = it.get(); + if (schema.existence_state.type == SchemaExistenceType::UNKNOWN && + schema.existence_state.if_not_found == OnEntryNotFound::THROW_EXCEPTION) { + auto &ic_catalog = schema.catalog.Cast(); + auto schema_exists = IRCAPI::VerifySchemaExistence(context, ic_catalog, schema.name); + if (!schema_exists) { + throw CatalogException("Namespace by the name of '%s' does not exist in catalog '%s'", + ic_catalog.GetName()); + } + } + } +} + +void IRCContextState::RegisterSchema(IRCSchemaEntry &schema) { + schemas.push_back(schema); +} + +} // namespace duckdb diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index 29ca6c7b..29bad1b1 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -5,12 +5,27 @@ #include "storage/irc_catalog.hpp" #include "storage/irc_schema_set.hpp" #include "storage/irc_transaction.hpp" +#include "storage/irc_context_state.hpp" namespace duckdb { IRCSchemaSet::IRCSchemaSet(Catalog &catalog) : catalog(catalog) { } +void IRCSchemaSet::VerifySchemas(ClientContext &context) { + auto &ic_catalog = catalog.Cast(); + for (auto &entry : entries) { + auto &schema = entry.second->Cast(); + if (schema.existence_state.type != SchemaExistenceType::UNKNOWN) { + continue; + } + if (!IRCAPI::VerifySchemaExistence(context, ic_catalog, schema.name)) { + throw CatalogException("Iceberg namespace by the name of '%s' does not exist", schema.name); + } + schema.existence_state.type = SchemaExistenceType::PRESENT; + } +} + optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const string &name, OnEntryNotFound if_not_found) { lock_guard l(entry_lock); @@ -18,12 +33,15 @@ optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const auto entry = entries.find(name); if (entry == entries.end()) { + auto context_state = context.registered_state->GetOrCreate("iceberg"); + //! We create the entry immediately optimistically, //! when we scan from the table we'll figure out if it exists or not. CreateSchemaInfo info; info.schema = name; info.internal = false; auto schema_entry = make_uniq(catalog, info); + context_state->RegisterSchema(*schema_entry); schema_entry->existence_state.if_not_found = if_not_found; CreateEntryInternal(context, std::move(schema_entry)); entry = entries.find(name); diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index f1951aac..f9c594a1 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -245,7 +245,9 @@ bool ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &tabl void ICTableSet::Scan(ClientContext &context, const std::function &callback) { lock_guard l(entry_lock); - LoadEntries(context); + if (!LoadEntries(context)) { + return; + } for (auto &entry : entries) { auto &table_info = entry.second; FillEntry(context, table_info); @@ -257,15 +259,17 @@ void ICTableSet::Scan(ClientContext &context, const std::function(); - // TODO: handle out-of-order columns using position property - auto tables = IRCAPI::GetTables(context, ic_catalog, schema.name); + vector tables; + if (!IRCAPI::GetTables(context, ic_catalog, schema, tables)) { + return false; + } for (auto &table : tables) { auto entry_it = entries.find(table.name); @@ -274,6 +278,7 @@ void ICTableSet::LoadEntries(ClientContext &context) { } } listed = true; + return true; } unique_ptr ICTableSet::GetTableInfo(ClientContext &context, IRCSchemaEntry &schema, diff --git a/src/storage/irc_transaction.cpp b/src/storage/irc_transaction.cpp index 75e1eacd..c6d6bd80 100644 --- a/src/storage/irc_transaction.cpp +++ b/src/storage/irc_transaction.cpp @@ -191,16 +191,13 @@ rest_api_objects::CommitTransactionRequest IRCTransaction::GetTransactionRequest return transaction; } -void IRCTransaction::Commit() { +void IRCTransaction::CommitToRESTCatalog(Connection &connection, ClientContext &context) { if (dirty_tables.empty()) { return; } - Connection temp_con(db); - temp_con.BeginTransaction(); - auto &context = temp_con.context; try { - auto transaction = GetTransactionRequest(*context); + auto transaction = GetTransactionRequest(context); auto &authentication = *catalog.auth_handler; if (catalog.supported_urls.find("POST /v1/{prefix}/transactions/commit") != catalog.supported_urls.end()) { // commit all transactions at once @@ -218,7 +215,7 @@ void IRCTransaction::Commit() { url_builder.AddPathComponent("transactions"); url_builder.AddPathComponent("commit"); - auto response = authentication.PostRequest(*context, url_builder, transaction_json); + auto response = authentication.PostRequest(context, url_builder, transaction_json); if (response->status != HTTPStatusCode::OK_200) { throw InvalidConfigurationException( "Request to '%s' returned a non-200 status code (%s), with reason: %s, body: %s", @@ -241,7 +238,7 @@ void IRCTransaction::Commit() { auto transaction_json = ConstructTableUpdateJSON(table_change); - auto response = authentication.PostRequest(*context, url_builder, transaction_json); + auto response = authentication.PostRequest(context, url_builder, transaction_json); if (response->status != HTTPStatusCode::OK_200) { throw InvalidConfigurationException( "Request to '%s' returned a non-200 status code (%s), with reason: %s, body: %s", @@ -249,14 +246,22 @@ void IRCTransaction::Commit() { } } } - DropSecrets(*context); + DropSecrets(context); } catch (std::exception &ex) { ErrorData error(ex); CleanupFiles(); - DropSecrets(*context); - temp_con.Rollback(); + DropSecrets(context); + connection.Rollback(); error.Throw("Failed to commit Iceberg transaction: "); } +} + +void IRCTransaction::Commit() { + Connection temp_con(db); + temp_con.BeginTransaction(); + auto &context = temp_con.context; + + CommitToRESTCatalog(temp_con, *context); temp_con.Rollback(); } diff --git a/test/sql/local/irc/test_use.test b/test/sql/local/irc/test_use.test index f48e6eef..8a8d4940 100644 --- a/test/sql/local/irc/test_use.test +++ b/test/sql/local/irc/test_use.test @@ -35,20 +35,23 @@ ATTACH '' AS my_datalake ( ENDPOINT 'http://127.0.0.1:8181' ); -statement ok +statement error use my_datalake.default; +---- +Request returned HTTP 500 for HTTP HEAD to 'http://127.0.0.1:8181/v1/namespaces/default' -# need catalog and schema name for use. + +# schema will default to 'main', which doesnt exist +# FIXME: with our optimistic approach, we can't error here statement error use my_datalake; ---- -Catalog Error +Request returned HTTP 500 for HTTP HEAD to 'http://127.0.0.1:8181/v1/namespaces/default' -# even after showing all tables. use statement needs catalog + schema. statement ok show all tables; statement error use my_datalake; ---- -Catalog Error \ No newline at end of file +Catalog Error From b153e0e5b44d3075d2cfcdac4c30e41ca5190345 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 8 Jul 2025 16:54:50 +0200 Subject: [PATCH 08/17] use a HEAD request per schema/table instead of listing the namespaces/tables --- CMakeLists.txt | 1 - duckdb | 2 +- src/catalog_api.cpp | 64 +++++++++---------- src/include/catalog_api.hpp | 6 +- src/include/storage/irc_context_state.hpp | 27 -------- src/include/storage/irc_schema_entry.hpp | 9 --- src/include/storage/irc_schema_set.hpp | 1 - src/include/storage/irc_table_set.hpp | 2 +- src/storage/irc_context_state.cpp | 29 --------- src/storage/irc_schema_set.cpp | 28 ++------ src/storage/irc_table_set.cpp | 17 ++--- .../local/irc/test_minimal_head_requests.test | 8 +-- test/sql/local/irc/test_use.test | 11 ++-- 13 files changed, 58 insertions(+), 147 deletions(-) delete mode 100644 src/include/storage/irc_context_state.hpp delete mode 100644 src/storage/irc_context_state.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f18506b4..5e215c4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,7 +60,6 @@ set(EXTENSION_SOURCES src/storage/irc_table_set.cpp src/storage/irc_transaction.cpp src/storage/irc_authorization.cpp - src/storage/irc_context_state.cpp src/storage/irc_transaction_manager.cpp src/storage/table_update/iceberg_add_snapshot.cpp ) diff --git a/duckdb b/duckdb index 49247310..95923354 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 49247310e79ba3cf39d21bbeccb27ea9b39425e1 +Subproject commit 959233546009eda23a6854d94f78ec1bc8e3d231 diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 3e042554..ca2d1a07 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -38,6 +38,28 @@ bool IRCAPI::VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, c url_builder.AddPathComponent("namespaces"); url_builder.AddPathComponent(schema); + auto url = url_builder.GetURL(); + try { + auto response = catalog.auth_handler->HeadRequest(context, url_builder); + if (response->Success()) { + return true; + } + return false; + } catch (std::exception &ex) { + //! For some reason this API likes to throw, instead of just returning a response with an error + return false; + } +} + +bool IRCAPI::VerifyTableExistence(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, + const string &table) { + auto url_builder = catalog.GetBaseUrl(); + url_builder.AddPathComponent(catalog.prefix); + url_builder.AddPathComponent("namespaces"); + url_builder.AddPathComponent(schema.name); + url_builder.AddPathComponent("tables"); + url_builder.AddPathComponent(table); + auto url = url_builder.GetURL(); auto response = catalog.auth_handler->HeadRequest(context, url_builder); if (response->Success()) { @@ -60,19 +82,8 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, IRCSc auto url = url_builder.GetURL(); auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { - if (schema.existence_state.type == SchemaExistenceType::UNKNOWN) { - bool exists = IRCAPI::VerifySchemaExistence(context, catalog, schema_name); - if (exists) { - schema.existence_state.type = SchemaExistenceType::PRESENT; - } else { - schema.existence_state.type = SchemaExistenceType::MISSING; - if (schema.existence_state.if_not_found == OnEntryNotFound::RETURN_NULL) { - return string(); - } - throw CatalogException("Namespace by the name of '%s' does not exist", schema_name); - } - } - return string(); + auto url = url_builder.GetURL(); + ThrowException(url, *response, "GET"); } return response->body; @@ -82,16 +93,12 @@ vector IRCAPI::GetCatalogs(ClientContext &context, IRCatalog &catalog) { throw NotImplementedException("ICAPI::GetCatalogs"); } -bool IRCAPI::GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, const string &table_name, - rest_api_objects::LoadTableResult &out) { - string result = GetTableMetadata(context, catalog, schema, table_name); - if (result.empty()) { - return false; - } +rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, + const string &table_name) { + auto result = GetTableMetadata(context, catalog, schema, table_name); std::unique_ptr doc(ICUtils::api_result_to_doc(result)); auto *metadata_root = yyjson_doc_get_root(doc.get()); - out = rest_api_objects::LoadTableResult::FromJSON(metadata_root); - return true; + return rest_api_objects::LoadTableResult::FromJSON(metadata_root); } bool IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, @@ -105,19 +112,8 @@ bool IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntr url_builder.AddPathComponent("tables"); auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { - if (schema.existence_state.type == SchemaExistenceType::UNKNOWN) { - bool exists = VerifySchemaExistence(context, catalog, schema_name); - if (exists) { - schema.existence_state.type = SchemaExistenceType::PRESENT; - } else { - schema.existence_state.type = SchemaExistenceType::MISSING; - if (schema.existence_state.if_not_found == OnEntryNotFound::RETURN_NULL) { - return false; - } - throw CatalogException("Namespace by the name of '%s' does not exist", schema_name); - } - } - return false; + auto url = url_builder.GetURL(); + ThrowException(url, *response, "GET"); } std::unique_ptr doc(ICUtils::api_result_to_doc(response->body)); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 0cba1bf8..02bc067c 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -19,10 +19,12 @@ class IRCAPI { static vector GetCatalogs(ClientContext &context, IRCatalog &catalog); static bool GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, vector &out); - static bool GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, const string &table_name, - rest_api_objects::LoadTableResult &out); + static rest_api_objects::LoadTableResult GetTable(ClientContext &context, IRCatalog &catalog, + IRCSchemaEntry &schema, const string &table_name); static vector GetSchemas(ClientContext &context, IRCatalog &catalog); static bool VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema); + static bool VerifyTableExistence(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, + const string &table); }; } // namespace duckdb diff --git a/src/include/storage/irc_context_state.hpp b/src/include/storage/irc_context_state.hpp deleted file mode 100644 index 13758006..00000000 --- a/src/include/storage/irc_context_state.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include "duckdb/main/client_context_state.hpp" - -namespace duckdb { - -class IRCSchemaEntry; - -class IRCContextState : public ClientContextState { -public: - IRCContextState() { - } - -public: - void QueryEnd(ClientContext &context) override; - void QueryEnd(ClientContext &context, optional_ptr error) override { - QueryEnd(context); - } - -public: - void RegisterSchema(IRCSchemaEntry &schema); - -public: - vector> schemas; -}; - -} // namespace duckdb diff --git a/src/include/storage/irc_schema_entry.hpp b/src/include/storage/irc_schema_entry.hpp index c16846e6..5c0270c5 100644 --- a/src/include/storage/irc_schema_entry.hpp +++ b/src/include/storage/irc_schema_entry.hpp @@ -9,14 +9,6 @@ namespace duckdb { class IRCTransaction; -enum class SchemaExistenceType : uint8_t { UNKNOWN, PRESENT, MISSING }; - -struct SchemaExistenceState { -public: - SchemaExistenceType type = SchemaExistenceType::UNKNOWN; - OnEntryNotFound if_not_found; -}; - class IRCSchemaEntry : public SchemaCatalogEntry { public: IRCSchemaEntry(Catalog &catalog, CreateSchemaInfo &info); @@ -48,7 +40,6 @@ class IRCSchemaEntry : public SchemaCatalogEntry { public: ICTableSet tables; - SchemaExistenceState existence_state; }; } // namespace duckdb diff --git a/src/include/storage/irc_schema_set.hpp b/src/include/storage/irc_schema_set.hpp index 87cbe912..e50fe2ac 100644 --- a/src/include/storage/irc_schema_set.hpp +++ b/src/include/storage/irc_schema_set.hpp @@ -14,7 +14,6 @@ class IRCSchemaSet { void LoadEntries(ClientContext &context); optional_ptr GetEntry(ClientContext &context, const string &name, OnEntryNotFound if_not_found); void Scan(ClientContext &context, const std::function &callback); - void VerifySchemas(ClientContext &context); protected: optional_ptr CreateEntryInternal(ClientContext &context, unique_ptr entry); diff --git a/src/include/storage/irc_table_set.hpp b/src/include/storage/irc_table_set.hpp index 53a39b59..235f4485 100644 --- a/src/include/storage/irc_table_set.hpp +++ b/src/include/storage/irc_table_set.hpp @@ -53,7 +53,7 @@ class ICTableSet { public: bool LoadEntries(ClientContext &context); - bool FillEntry(ClientContext &context, IcebergTableInformation &table); + void FillEntry(ClientContext &context, IcebergTableInformation &table); public: IRCSchemaEntry &schema; diff --git a/src/storage/irc_context_state.cpp b/src/storage/irc_context_state.cpp deleted file mode 100644 index e4afe2c7..00000000 --- a/src/storage/irc_context_state.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "storage/irc_context_state.hpp" - -#include "storage/irc_schema_entry.hpp" -#include "storage/irc_catalog.hpp" -#include "duckdb/common/enums/on_entry_not_found.hpp" -#include "catalog_api.hpp" - -namespace duckdb { - -void IRCContextState::QueryEnd(ClientContext &context) { - for (auto &it : schemas) { - auto &schema = it.get(); - if (schema.existence_state.type == SchemaExistenceType::UNKNOWN && - schema.existence_state.if_not_found == OnEntryNotFound::THROW_EXCEPTION) { - auto &ic_catalog = schema.catalog.Cast(); - auto schema_exists = IRCAPI::VerifySchemaExistence(context, ic_catalog, schema.name); - if (!schema_exists) { - throw CatalogException("Namespace by the name of '%s' does not exist in catalog '%s'", - ic_catalog.GetName()); - } - } - } -} - -void IRCContextState::RegisterSchema(IRCSchemaEntry &schema) { - schemas.push_back(schema); -} - -} // namespace duckdb diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index 29bad1b1..5799086c 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -5,27 +5,12 @@ #include "storage/irc_catalog.hpp" #include "storage/irc_schema_set.hpp" #include "storage/irc_transaction.hpp" -#include "storage/irc_context_state.hpp" namespace duckdb { IRCSchemaSet::IRCSchemaSet(Catalog &catalog) : catalog(catalog) { } -void IRCSchemaSet::VerifySchemas(ClientContext &context) { - auto &ic_catalog = catalog.Cast(); - for (auto &entry : entries) { - auto &schema = entry.second->Cast(); - if (schema.existence_state.type != SchemaExistenceType::UNKNOWN) { - continue; - } - if (!IRCAPI::VerifySchemaExistence(context, ic_catalog, schema.name)) { - throw CatalogException("Iceberg namespace by the name of '%s' does not exist", schema.name); - } - schema.existence_state.type = SchemaExistenceType::PRESENT; - } -} - optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const string &name, OnEntryNotFound if_not_found) { lock_guard l(entry_lock); @@ -33,16 +18,17 @@ optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const auto entry = entries.find(name); if (entry == entries.end()) { - auto context_state = context.registered_state->GetOrCreate("iceberg"); - - //! We create the entry immediately optimistically, - //! when we scan from the table we'll figure out if it exists or not. CreateSchemaInfo info; + if (!IRCAPI::VerifySchemaExistence(context, ic_catalog, name)) { + if (if_not_found == OnEntryNotFound::RETURN_NULL) { + return nullptr; + } else { + throw CatalogException("Iceberg namespace by the name of '%s' does not exist", name); + } + } info.schema = name; info.internal = false; auto schema_entry = make_uniq(catalog, info); - context_state->RegisterSchema(*schema_entry); - schema_entry->existence_state.if_not_found = if_not_found; CreateEntryInternal(context, std::move(schema_entry)); entry = entries.find(name); D_ASSERT(entry != entries.end()); diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index f9c594a1..5e3c8046 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -223,15 +223,13 @@ optional_ptr IcebergTableInformation::GetSchemaVersion(optional_pt ICTableSet::ICTableSet(IRCSchemaEntry &schema) : schema(schema), catalog(schema.ParentCatalog()) { } -bool ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &table) { +void ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &table) { if (!table.schema_versions.empty()) { - return true; + return; } auto &ic_catalog = catalog.Cast(); - if (!IRCAPI::GetTable(context, ic_catalog, schema, table.name, table.load_table_result)) { - return false; - } + table.load_table_result = IRCAPI::GetTable(context, ic_catalog, schema, table.name); table.table_metadata = IcebergTableMetadata::FromTableMetadata(table.load_table_result.metadata); auto &schemas = table.table_metadata.schemas; @@ -240,7 +238,6 @@ bool ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &tabl for (auto &table_schema : schemas) { table.CreateSchemaVersion(*table_schema.second); } - return true; } void ICTableSet::Scan(ClientContext &context, const std::function &callback) { @@ -293,13 +290,13 @@ optional_ptr ICTableSet::GetEntry(ClientContext &context, const En auto table_name = lookup.GetEntryName(); auto entry = entries.find(table_name); if (entry == entries.end()) { + if (!IRCAPI::VerifyTableExistence(context, ic_catalog, schema, table_name)) { + return nullptr; + } auto it = entries.emplace(table_name, IcebergTableInformation(ic_catalog, schema, table_name)); entry = it.first; } - if (!FillEntry(context, entry->second)) { - //! The namespace or table doesn't exist and the schema lookup said OnEntryNotFound::RETURN_NULL - return nullptr; - } + FillEntry(context, entry->second); return entry->second.GetSchemaVersion(lookup.GetAtClause()); } diff --git a/test/sql/local/irc/test_minimal_head_requests.test b/test/sql/local/irc/test_minimal_head_requests.test index 00215f1c..2698f0de 100644 --- a/test/sql/local/irc/test_minimal_head_requests.test +++ b/test/sql/local/irc/test_minimal_head_requests.test @@ -54,11 +54,11 @@ select * from my_datalake.default.table_unpartitioned order by all; 2023-03-11 11 k 2023-03-12 12 l -# 0 head requests to all files +# 2 head requests to scan all files (1 for the schema, 1 for the table) query I select count(*) from duckdb_logs_parsed('HTTP') where request.type = 'HEAD'; ---- -0 +2 # only 13 get requests query I nosort get_count_1 @@ -85,11 +85,11 @@ select * from my_datalake.default.table_unpartitioned order by all; 2023-03-12 12 l -# after reading the same table, still no head requests +# after reading the same table, only 2 more head requests (because of new transactions) query I select count(*) from duckdb_logs_parsed('HTTP') where request.type = 'HEAD'; ---- -0 +4 # same amount of get calls query I nosort get_count_1 diff --git a/test/sql/local/irc/test_use.test b/test/sql/local/irc/test_use.test index 8a8d4940..f5c3ae55 100644 --- a/test/sql/local/irc/test_use.test +++ b/test/sql/local/irc/test_use.test @@ -35,23 +35,20 @@ ATTACH '' AS my_datalake ( ENDPOINT 'http://127.0.0.1:8181' ); -statement error +statement ok use my_datalake.default; ----- -Request returned HTTP 500 for HTTP HEAD to 'http://127.0.0.1:8181/v1/namespaces/default' - # schema will default to 'main', which doesnt exist -# FIXME: with our optimistic approach, we can't error here statement error use my_datalake; ---- -Request returned HTTP 500 for HTTP HEAD to 'http://127.0.0.1:8181/v1/namespaces/default' +Catalog Error: SET schema: No catalog + schema named "my_datalake" found. statement ok show all tables; +# 'main' still doesn't exist statement error use my_datalake; ---- -Catalog Error +Catalog Error: SET schema: No catalog + schema named "my_datalake" found. From 20fb022910ecfd1db981a4e72520f86ffd694a34 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 8 Jul 2025 17:01:27 +0200 Subject: [PATCH 09/17] cleaning --- src/catalog_api.cpp | 3 +-- src/include/catalog_api.hpp | 2 +- src/include/storage/irc_schema_set.hpp | 2 ++ src/include/storage/irc_table_set.hpp | 2 +- src/storage/irc_schema_set.cpp | 3 ++- src/storage/irc_table_set.cpp | 14 ++++---------- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index ca2d1a07..d6789321 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -101,7 +101,7 @@ rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCat return rest_api_objects::LoadTableResult::FromJSON(metadata_root); } -bool IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, +void IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, vector &out) { auto schema_name = schema.name; @@ -124,7 +124,6 @@ bool IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntr throw NotImplementedException("List of 'identifiers' is missing, missing support for Iceberg V1"); } out = std::move(list_tables_response.identifiers); - return true; } vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catalog) { diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 02bc067c..b35921c5 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -17,7 +17,7 @@ class IRCAPI { public: static const string API_VERSION_1; static vector GetCatalogs(ClientContext &context, IRCatalog &catalog); - static bool GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, + static void GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, vector &out); static rest_api_objects::LoadTableResult GetTable(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, const string &table_name); diff --git a/src/include/storage/irc_schema_set.hpp b/src/include/storage/irc_schema_set.hpp index e50fe2ac..dca8c65d 100644 --- a/src/include/storage/irc_schema_set.hpp +++ b/src/include/storage/irc_schema_set.hpp @@ -21,6 +21,8 @@ class IRCSchemaSet { public: Catalog &catalog; case_insensitive_map_t> entries; + //! Whether a listing has been done for the catalog + bool listed = false; private: mutex entry_lock; diff --git a/src/include/storage/irc_table_set.hpp b/src/include/storage/irc_table_set.hpp index 235f4485..0fbb35e7 100644 --- a/src/include/storage/irc_table_set.hpp +++ b/src/include/storage/irc_table_set.hpp @@ -52,7 +52,7 @@ class ICTableSet { void Scan(ClientContext &context, const std::function &callback); public: - bool LoadEntries(ClientContext &context); + void LoadEntries(ClientContext &context); void FillEntry(ClientContext &context, IcebergTableInformation &table); public: diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index 5799086c..d8d11ff5 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -45,7 +45,7 @@ void IRCSchemaSet::Scan(ClientContext &context, const std::function(catalog, info); CreateEntryInternal(context, std::move(schema_entry)); } + listed = true; } optional_ptr IRCSchemaSet::CreateEntryInternal(ClientContext &context, unique_ptr entry) { diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index 5e3c8046..8b7525c8 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -242,9 +242,7 @@ void ICTableSet::FillEntry(ClientContext &context, IcebergTableInformation &tabl void ICTableSet::Scan(ClientContext &context, const std::function &callback) { lock_guard l(entry_lock); - if (!LoadEntries(context)) { - return; - } + LoadEntries(context); for (auto &entry : entries) { auto &table_info = entry.second; FillEntry(context, table_info); @@ -256,17 +254,14 @@ void ICTableSet::Scan(ClientContext &context, const std::function(); vector tables; - if (!IRCAPI::GetTables(context, ic_catalog, schema, tables)) { - return false; - } + IRCAPI::GetTables(context, ic_catalog, schema, tables); for (auto &table : tables) { auto entry_it = entries.find(table.name); @@ -275,7 +270,6 @@ bool ICTableSet::LoadEntries(ClientContext &context) { } } listed = true; - return true; } unique_ptr ICTableSet::GetTableInfo(ClientContext &context, IRCSchemaEntry &schema, From 22fa1b4a359a7e01de597766b0cfad5c02edbe91 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 8 Jul 2025 17:14:11 +0200 Subject: [PATCH 10/17] remove some dead code --- src/storage/irc_table_set.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index 8b7525c8..440901c6 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -204,9 +204,6 @@ optional_ptr IcebergTableInformation::CreateSchemaVersion(IcebergT } optional_ptr IcebergTableInformation::GetSchemaVersion(optional_ptr at) { - if (schema_versions.empty()) { - return nullptr; - } auto snapshot_lookup = IcebergSnapshotLookup::FromAtClause(at); int32_t schema_id; @@ -247,9 +244,6 @@ void ICTableSet::Scan(ClientContext &context, const std::function Date: Tue, 2 Sep 2025 12:30:06 +0200 Subject: [PATCH 11/17] fix up problems --- src/catalog_api.cpp | 7 +++++-- src/storage/irc_schema_set.cpp | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index c203bdaf..dfa04418 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -40,6 +40,7 @@ string IRCAPI::GetSchemaName(const vector &items) { //! Used for the path parameters string IRCAPI::GetEncodedSchemaName(const vector &items) { + D_ASSERT(!items.empty()); static const string unit_separator = "%1F"; return StringUtil::Join(items, unit_separator); } @@ -84,10 +85,12 @@ bool IRCAPI::VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, c bool IRCAPI::VerifyTableExistence(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, const string &table) { + auto schema_name = GetEncodedSchemaName(schema.namespace_items); + auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); url_builder.AddPathComponent("namespaces"); - url_builder.AddPathComponent(schema.name); + url_builder.AddPathComponent(schema_name); url_builder.AddPathComponent("tables"); url_builder.AddPathComponent(table); @@ -134,7 +137,7 @@ rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCat void IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, vector &out) { - auto schema_name = schema.name; + auto schema_name = GetEncodedSchemaName(schema.namespace_items); auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index 04e26e03..a7f1b068 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -29,6 +29,7 @@ optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const info.schema = name; info.internal = false; auto schema_entry = make_uniq(catalog, info); + schema_entry->namespace_items = {name}; CreateEntryInternal(context, std::move(schema_entry)); entry = entries.find(name); D_ASSERT(entry != entries.end()); From fdb85c9b15ccba0338b60f749727f16c2b824513 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Sep 2025 12:38:19 +0200 Subject: [PATCH 12/17] better nested namespace support --- src/catalog_api.cpp | 7 +++++-- src/include/catalog_api.hpp | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index dfa04418..8bcd72e5 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -17,7 +17,7 @@ using namespace duckdb_yyjson; namespace duckdb { -vector IRCAPI::ParseSchemaName(string &namespace_name) { +vector IRCAPI::ParseSchemaName(const string &namespace_name) { idx_t start = 0; idx_t end = namespace_name.find(".", start); vector ret; @@ -65,10 +65,13 @@ string IRCAPI::GetEncodedSchemaName(const vector &items) { } bool IRCAPI::VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema) { + auto namespace_items = ParseSchemaName(schema); + auto schema_name = GetEncodedSchemaName(namespace_items); + auto url_builder = catalog.GetBaseUrl(); url_builder.AddPathComponent(catalog.prefix); url_builder.AddPathComponent("namespaces"); - url_builder.AddPathComponent(schema); + url_builder.AddPathComponent(schema_name); auto url = url_builder.GetURL(); try { diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index c86a1724..e8eda64c 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -30,7 +30,7 @@ class IRCAPI { static bool VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema); static bool VerifyTableExistence(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, const string &table); - static vector ParseSchemaName(string &namespace_name); + static vector ParseSchemaName(const string &namespace_name); static string GetSchemaName(const vector &items); static string GetEncodedSchemaName(const vector &items); static void GetTables(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, From 955f15f1c2672834f9c01d59e6048437d48b79d0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Sep 2025 13:00:03 +0200 Subject: [PATCH 13/17] cleanup --- src/catalog_api.cpp | 11 +--- src/include/catalog_api.hpp | 6 +- src/storage/irc_table_set.cpp | 110 +--------------------------------- 3 files changed, 6 insertions(+), 121 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 8bcd72e5..b2328169 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -119,17 +119,12 @@ static string GetTableMetadata(ClientContext &context, IRCatalog &catalog, const auto url = url_builder.GetURL(); auto response = catalog.auth_handler->GetRequest(context, url_builder); if (!response->Success()) { - auto url = url_builder.GetURL(); ThrowException(url, *response, "GET"); } return response->body; } -vector IRCAPI::GetCatalogs(ClientContext &context, IRCatalog &catalog) { - throw NotImplementedException("ICAPI::GetCatalogs"); -} - rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, const string &table_name) { auto result = GetTableMetadata(context, catalog, schema, table_name); @@ -138,8 +133,8 @@ rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCat return rest_api_objects::LoadTableResult::FromJSON(metadata_root); } -void IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, - vector &out) { +vector IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, + IRCSchemaEntry &schema) { auto schema_name = GetEncodedSchemaName(schema.namespace_items); auto url_builder = catalog.GetBaseUrl(); @@ -160,7 +155,7 @@ void IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntr if (!list_tables_response.has_identifiers) { throw NotImplementedException("List of 'identifiers' is missing, missing support for Iceberg V1"); } - out = std::move(list_tables_response.identifiers); + return std::move(list_tables_response.identifiers); } vector IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catalog, const vector &parent) { diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index e8eda64c..c41e4e81 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -23,10 +23,8 @@ struct IRCAPISchema { class IRCAPI { public: static const string API_VERSION_1; - static vector GetCatalogs(ClientContext &context, IRCatalog &catalog); - static void GetTables(ClientContext &context, IRCatalog &catalog, IRCSchemaEntry &schema, - vector &out); - static vector GetSchemas(ClientContext &context, IRCatalog &catalog); + static vector GetTables(ClientContext &context, IRCatalog &catalog, + IRCSchemaEntry &schema); static bool VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema); static bool VerifyTableExistence(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, const string &table); diff --git a/src/storage/irc_table_set.cpp b/src/storage/irc_table_set.cpp index 3c670331..862a9a1b 100644 --- a/src/storage/irc_table_set.cpp +++ b/src/storage/irc_table_set.cpp @@ -24,113 +24,6 @@ namespace duckdb { -// IcebergTableInformation::IcebergTableInformation(IRCatalog &catalog, IRCSchemaEntry &schema, const string &name) -// : catalog(catalog), schema(schema), name(name) { -// table_id = "uuid-" + schema.name + "-" + name; -//} - -// void IcebergTableInformation::AddSnapshot(IRCTransaction &transaction, vector &&data_files) { -// if (!transaction_data) { -// auto context = transaction.context.lock(); -// transaction_data = make_uniq(*context, *this); -// } - -// transaction_data->AddSnapshot(IcebergSnapshotOperationType::APPEND, std::move(data_files)); -//} - -// static void ParseConfigOptions(const case_insensitive_map_t &config, case_insensitive_map_t &options) -// { -// //! Set of recognized config parameters and the duckdb secret option that matches it. -// static const case_insensitive_map_t config_to_option = {{"s3.access-key-id", "key_id"}, -// {"s3.secret-access-key", "secret"}, -// {"s3.session-token", "session_token"}, -// {"s3.region", "region"}, -// {"region", "region"}, -// {"client.region", "region"}, -// {"s3.endpoint", "endpoint"}}; - -// if (config.empty()) { -// return; -// } -// for (auto &entry : config) { -// auto it = config_to_option.find(entry.first); -// if (it != config_to_option.end()) { -// options[it->second] = entry.second; -// } -// } - -// auto it = config.find("s3.path-style-access"); -// if (it != config.end()) { -// bool path_style; -// if (it->second == "true") { -// path_style = true; -// } else if (it->second == "false") { -// path_style = false; -// } else { -// throw InvalidInputException("Unexpected value ('%s') for 's3.path-style-access' in 'config' property", -// it->second); -// } - -// options["use_ssl"] = Value(!path_style); -// if (path_style) { -// options["url_style"] = "path"; -// } -// } - -// auto endpoint_it = options.find("endpoint"); -// if (endpoint_it == options.end()) { -// return; -// } -// auto endpoint = endpoint_it->second.ToString(); -// if (StringUtil::StartsWith(endpoint, "http://")) { -// endpoint = endpoint.substr(7, string::npos); -// } -// if (StringUtil::StartsWith(endpoint, "https://")) { -// endpoint = endpoint.substr(8, string::npos); -// } -// if (StringUtil::EndsWith(endpoint, "/")) { -// endpoint = endpoint.substr(0, endpoint.size() - 1); -// } -// endpoint_it->second = endpoint; -//} - -// const string &IcebergTableInformation::BaseFilePath() const { -// return load_table_result.metadata.location; -//} - -// optional_ptr IcebergTableInformation::CreateSchemaVersion(IcebergTableSchema &table_schema) { -// CreateTableInfo info; -// info.table = name; -// for (auto &col : table_schema.columns) { -// info.columns.AddColumn(ColumnDefinition(col->name, col->type)); -// } - -// auto table_entry = make_uniq(*this, catalog, schema, info); -// if (!table_entry->internal) { -// table_entry->internal = schema.internal; -// } -// auto result = table_entry.get(); -// if (result->name.empty()) { -// throw InternalException("ICTableSet::CreateEntry called with empty name"); -// } -// schema_versions.emplace(table_schema.schema_id, std::move(table_entry)); -// return result; -//} - -// optional_ptr IcebergTableInformation::GetSchemaVersion(optional_ptr at) { -// auto snapshot_lookup = IcebergSnapshotLookup::FromAtClause(at); - -// int32_t schema_id; -// if (snapshot_lookup.IsLatest()) { -// schema_id = table_metadata.current_schema_id; -// } else { -// auto snapshot = table_metadata.GetSnapshot(snapshot_lookup); -// D_ASSERT(snapshot); -// schema_id = snapshot->schema_id; -// } -// return schema_versions[schema_id].get(); -//} - ICTableSet::ICTableSet(IRCSchemaEntry &schema) : schema(schema), catalog(schema.ParentCatalog()) { } @@ -168,8 +61,7 @@ void ICTableSet::LoadEntries(ClientContext &context) { } auto &ic_catalog = catalog.Cast(); - vector tables; - IRCAPI::GetTables(context, ic_catalog, schema, tables); + auto tables = IRCAPI::GetTables(context, ic_catalog, schema); for (auto &table : tables) { auto entry_it = entries.find(table.name); From ce08c6c8cfb1bde8fa58d06dcf92e74d55e2ca2b Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Sep 2025 13:04:52 +0200 Subject: [PATCH 14/17] cleaning.. --- src/include/catalog_api.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index c41e4e81..3675f48e 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -31,8 +31,6 @@ class IRCAPI { static vector ParseSchemaName(const string &namespace_name); static string GetSchemaName(const vector &items); static string GetEncodedSchemaName(const vector &items); - static void GetTables(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, - vector &out); static rest_api_objects::LoadTableResult GetTable(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, const string &table_name); static vector GetSchemas(ClientContext &context, IRCatalog &catalog, const vector &parent); From dc1ff168e12adf5cca302a141a375d540d7404e0 Mon Sep 17 00:00:00 2001 From: Tishj Date: Tue, 2 Sep 2025 13:06:58 +0200 Subject: [PATCH 15/17] add back const --- src/catalog_api.cpp | 2 +- src/include/catalog_api.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index b2328169..5cfb41f2 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -134,7 +134,7 @@ rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCat } vector IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog, - IRCSchemaEntry &schema) { + const IRCSchemaEntry &schema) { auto schema_name = GetEncodedSchemaName(schema.namespace_items); auto url_builder = catalog.GetBaseUrl(); diff --git a/src/include/catalog_api.hpp b/src/include/catalog_api.hpp index 3675f48e..4e0af997 100644 --- a/src/include/catalog_api.hpp +++ b/src/include/catalog_api.hpp @@ -24,7 +24,7 @@ class IRCAPI { public: static const string API_VERSION_1; static vector GetTables(ClientContext &context, IRCatalog &catalog, - IRCSchemaEntry &schema); + const IRCSchemaEntry &schema); static bool VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, const string &schema); static bool VerifyTableExistence(ClientContext &context, IRCatalog &catalog, const IRCSchemaEntry &schema, const string &table); From b31f8549cfbb63bb1881b907e26dde1126d5d223 Mon Sep 17 00:00:00 2001 From: Tishj Date: Wed, 3 Sep 2025 12:52:37 +0200 Subject: [PATCH 16/17] check for 204 - success, no content --- src/catalog_api.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/catalog_api.cpp b/src/catalog_api.cpp index 5cfb41f2..d6dd0aae 100644 --- a/src/catalog_api.cpp +++ b/src/catalog_api.cpp @@ -76,7 +76,7 @@ bool IRCAPI::VerifySchemaExistence(ClientContext &context, IRCatalog &catalog, c auto url = url_builder.GetURL(); try { auto response = catalog.auth_handler->HeadRequest(context, url_builder); - if (response->Success()) { + if (response->Success() || response->status == HTTPStatusCode::NoContent_204) { return true; } return false; @@ -99,7 +99,7 @@ bool IRCAPI::VerifyTableExistence(ClientContext &context, IRCatalog &catalog, co auto url = url_builder.GetURL(); auto response = catalog.auth_handler->HeadRequest(context, url_builder); - if (response->Success()) { + if (response->Success() || response->status == HTTPStatusCode::NoContent_204) { return true; } return false; From 07b5a41b6e4b922be0099d6544bf2470f8138446 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Wed, 3 Sep 2025 18:53:38 +0200 Subject: [PATCH 17/17] parse schema name when getting the entry --- src/storage/irc_schema_set.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/irc_schema_set.cpp b/src/storage/irc_schema_set.cpp index a7f1b068..f916e308 100644 --- a/src/storage/irc_schema_set.cpp +++ b/src/storage/irc_schema_set.cpp @@ -29,7 +29,7 @@ optional_ptr IRCSchemaSet::GetEntry(ClientContext &context, const info.schema = name; info.internal = false; auto schema_entry = make_uniq(catalog, info); - schema_entry->namespace_items = {name}; + schema_entry->namespace_items = IRCAPI::ParseSchemaName(name); CreateEntryInternal(context, std::move(schema_entry)); entry = entries.find(name); D_ASSERT(entry != entries.end());