Skip to content

Commit d87c4bb

Browse files
TmonsterTmonster
authored
Merge pull request #439 from Tmonster/add_pagination_to_get_tables
Add pagination to get tables and to GetSchemas
2 parents 2138055 + 56cb054 commit d87c4bb

File tree

2 files changed

+116
-50
lines changed

2 files changed

+116
-50
lines changed

‎src/catalog_api.cpp‎

Lines changed: 79 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -100,66 +100,95 @@ rest_api_objects::LoadTableResult IRCAPI::GetTable(ClientContext &context, IRCat
100100
vector<rest_api_objects::TableIdentifier> IRCAPI::GetTables(ClientContext &context, IRCatalog &catalog,
101101
const IRCSchemaEntry &schema) {
102102
auto schema_name = GetEncodedSchemaName(schema.namespace_items);
103+
vector<rest_api_objects::TableIdentifier> all_identifiers;
104+
string page_token;
105+
106+
do {
107+
auto url_builder = catalog.GetBaseUrl();
108+
url_builder.AddPathComponent(catalog.prefix);
109+
url_builder.AddPathComponent("namespaces");
110+
url_builder.AddPathComponent(schema_name);
111+
url_builder.AddPathComponent("tables");
112+
if (!page_token.empty()) {
113+
url_builder.SetParam("pageToken", page_token);
114+
}
115+
auto response = catalog.auth_handler->GetRequest(context, url_builder);
116+
if (!response->Success()) {
117+
auto url = url_builder.GetURL();
118+
ThrowException(url, *response, "GET");
119+
}
103120

104-
auto url_builder = catalog.GetBaseUrl();
105-
url_builder.AddPathComponent(catalog.prefix);
106-
url_builder.AddPathComponent("namespaces");
107-
url_builder.AddPathComponent(schema_name);
108-
url_builder.AddPathComponent("tables");
109-
auto response = catalog.auth_handler->GetRequest(context, url_builder);
110-
if (!response->Success()) {
111-
auto url = url_builder.GetURL();
112-
ThrowException(url, *response, "GET");
113-
}
121+
std::unique_ptr<yyjson_doc, YyjsonDocDeleter> doc(ICUtils::api_result_to_doc(response->body));
122+
auto *root = yyjson_doc_get_root(doc.get());
123+
auto list_tables_response = rest_api_objects::ListTablesResponse::FromJSON(root);
114124

115-
std::unique_ptr<yyjson_doc, YyjsonDocDeleter> doc(ICUtils::api_result_to_doc(response->body));
116-
auto *root = yyjson_doc_get_root(doc.get());
117-
auto list_tables_response = rest_api_objects::ListTablesResponse::FromJSON(root);
125+
if (!list_tables_response.has_identifiers) {
126+
throw NotImplementedException("List of 'identifiers' is missing, missing support for Iceberg V1");
127+
}
118128

119-
if (!list_tables_response.has_identifiers) {
120-
throw NotImplementedException("List of 'identifiers' is missing, missing support for Iceberg V1");
121-
}
122-
return std::move(list_tables_response.identifiers);
129+
all_identifiers.insert(all_identifiers.end(), std::make_move_iterator(list_tables_response.identifiers.begin()),
130+
std::make_move_iterator(list_tables_response.identifiers.end()));
131+
132+
if (list_tables_response.has_next_page_token) {
133+
page_token = list_tables_response.next_page_token.value;
134+
} else {
135+
page_token.clear();
136+
}
137+
} while (!page_token.empty());
138+
139+
return all_identifiers;
123140
}
124141

125142
vector<IRCAPISchema> IRCAPI::GetSchemas(ClientContext &context, IRCatalog &catalog, const vector<string> &parent) {
126143
vector<IRCAPISchema> result;
127-
auto endpoint_builder = catalog.GetBaseUrl();
128-
endpoint_builder.AddPathComponent(catalog.prefix);
129-
endpoint_builder.AddPathComponent("namespaces");
130-
if (!parent.empty()) {
131-
auto parent_name = GetSchemaName(parent);
132-
endpoint_builder.SetParam("parent", parent_name);
133-
}
134-
auto response = catalog.auth_handler->GetRequest(context, endpoint_builder);
135-
if (!response->Success()) {
136-
auto url = endpoint_builder.GetURL();
137-
ThrowException(url, *response, "GET");
138-
}
144+
string page_token = "";
145+
do {
146+
auto endpoint_builder = catalog.GetBaseUrl();
147+
endpoint_builder.AddPathComponent(catalog.prefix);
148+
endpoint_builder.AddPathComponent("namespaces");
149+
if (!parent.empty()) {
150+
auto parent_name = GetSchemaName(parent);
151+
endpoint_builder.SetParam("parent", parent_name);
152+
}
153+
if (!page_token.empty()) {
154+
endpoint_builder.SetParam("pageToken", page_token);
155+
}
156+
auto response = catalog.auth_handler->GetRequest(context, endpoint_builder);
157+
if (!response->Success()) {
158+
auto url = endpoint_builder.GetURL();
159+
ThrowException(url, *response, "GET");
160+
}
139161

140-
std::unique_ptr<yyjson_doc, YyjsonDocDeleter> doc(ICUtils::api_result_to_doc(response->body));
141-
auto *root = yyjson_doc_get_root(doc.get());
142-
auto list_namespaces_response = rest_api_objects::ListNamespacesResponse::FromJSON(root);
143-
if (!list_namespaces_response.has_namespaces) {
144-
//! FIXME: old code expected 'namespaces' to always be present, but it's not a required property
145-
return result;
146-
}
147-
auto &schemas = list_namespaces_response.namespaces;
148-
for (auto &schema : schemas) {
149-
IRCAPISchema schema_result;
150-
schema_result.catalog_name = catalog.GetName();
151-
schema_result.items = std::move(schema.value);
152-
153-
if (catalog.attach_options.support_nested_namespaces) {
154-
auto new_parent = parent;
155-
new_parent.push_back(schema_result.items.back());
156-
auto nested_namespaces = GetSchemas(context, catalog, new_parent);
157-
result.insert(result.end(), std::make_move_iterator(nested_namespaces.begin()),
158-
std::make_move_iterator(nested_namespaces.end()));
162+
std::unique_ptr<yyjson_doc, YyjsonDocDeleter> doc(ICUtils::api_result_to_doc(response->body));
163+
auto *root = yyjson_doc_get_root(doc.get());
164+
auto list_namespaces_response = rest_api_objects::ListNamespacesResponse::FromJSON(root);
165+
if (!list_namespaces_response.has_namespaces) {
166+
//! FIXME: old code expected 'namespaces' to always be present, but it's not a required property
167+
return result;
168+
}
169+
auto &schemas = list_namespaces_response.namespaces;
170+
for (auto &schema : schemas) {
171+
IRCAPISchema schema_result;
172+
schema_result.catalog_name = catalog.GetName();
173+
schema_result.items = std::move(schema.value);
174+
175+
if (catalog.attach_options.support_nested_namespaces) {
176+
auto new_parent = parent;
177+
new_parent.push_back(schema_result.items.back());
178+
auto nested_namespaces = GetSchemas(context, catalog, new_parent);
179+
result.insert(result.end(), std::make_move_iterator(nested_namespaces.begin()),
180+
std::make_move_iterator(nested_namespaces.end()));
181+
}
182+
result.push_back(schema_result);
159183
}
160184

161-
result.push_back(schema_result);
162-
}
185+
if (list_namespaces_response.has_next_page_token) {
186+
page_token = list_namespaces_response.next_page_token.value;
187+
} else {
188+
page_token.clear();
189+
}
190+
} while (!page_token.empty());
191+
163192
return result;
164193
}
165194

‎test/sql/cloud/r2_catalog/test_r2_pagination.test‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# name: test/sql/cloud/r2_catalog/test_r2_pagination.test
2+
# description: test integration with iceberg catalog read
3+
# group: [r2_catalog]
4+
5+
require-env ICEBERG_AWS_REMOTE_AVAILABLE
6+
7+
require-env ICEBERG_REMOTE_INSERT_READY
8+
9+
require-env R2_TOKEN
10+
11+
require avro
12+
13+
require parquet
14+
15+
require iceberg
16+
17+
require httpfs
18+
19+
require aws
20+
21+
statement ok
22+
CREATE SECRET r2_secret (
23+
TYPE ICEBERG,
24+
TOKEN '${R2_TOKEN}'
25+
);
26+
27+
statement ok
28+
attach '6b17833f308abc1e1cc343c552b51f51_iceberg-catalog-test' AS my_datalake (
29+
TYPE ICEBERG,
30+
ENDPOINT 'https://catalog.cloudflarestorage.com/6b17833f308abc1e1cc343c552b51f51/iceberg-catalog-test',
31+
support_nested_namespaces true
32+
);
33+
34+
query I
35+
select count(*) from (show all tables);
36+
----
37+
101

0 commit comments

Comments
 (0)