From b60c6c788e27b82eed5240459456951046ae746a Mon Sep 17 00:00:00 2001 From: Nadav Steindler <32031989+nadavsteindler@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:45:22 +0200 Subject: [PATCH] Feature: Repository Search by Substring (#8417) * fuzzy repo search * fuzzy repo search * fuzzy repo search * fuzzy repo search * fuzzy repo search * fuzzy repo search * moar tests * moar tests * api * use new param * page query param name * change name * Update swagger.yml * clean code * moar tests * Apply suggestions from code review Co-authored-by: Ariel Shaqed (Scolnicov) * review fixes * yaml fmt * yaml fmt * code review --------- Co-authored-by: Ariel Shaqed (Scolnicov) --- api/swagger.yml | 90 ++++++----- clients/java/api/openapi.yaml | 17 ++ clients/java/docs/RepositoriesApi.md | 5 +- .../lakefs/clients/sdk/RepositoriesApi.java | 37 +++-- .../clients/sdk/RepositoriesApiTest.java | 2 + clients/python/docs/RepositoriesApi.md | 6 +- .../python/lakefs_sdk/api/repositories_api.py | 20 ++- clients/rust/docs/RepositoriesApi.md | 3 +- clients/rust/src/apis/repositories_api.rs | 5 +- cmd/lakefs/cmd/run.go | 2 +- docs/assets/js/swagger.yml | 90 ++++++----- pkg/api/controller.go | 9 +- pkg/catalog/catalog.go | 9 +- pkg/catalog/catalog_test.go | 153 +++++++++++++++--- pkg/gateway/operations/listbuckets.go | 2 +- webui/src/lib/api/index.js | 8 +- webui/src/pages/repositories/index.jsx | 22 +-- 17 files changed, 331 insertions(+), 149 deletions(-) diff --git a/api/swagger.yml b/api/swagger.yml index e4e16d94b0f..01cc2f46177 100644 --- a/api/swagger.yml +++ b/api/swagger.yml @@ -70,6 +70,13 @@ components: description: delimiter used to group common prefixes by schema: type: string + + SearchString: + in: query + name: search + description: string for searching relevant entries + schema: + type: string IfNoneMatch: in: header @@ -695,7 +702,7 @@ components: default: false hidden: type: boolean - description: When set, branch will not show up when listing branches by default. *EXPERIMENTAL* + description: When set, branch will not show up when listing branches by default. *EXPERIMENTAL* default: false TagCreation: @@ -742,7 +749,7 @@ components: error: type: string refs: - $ref: "#/components/schemas/RefsDump" + $ref: "#/components/schemas/RefsDump" RepositoryRestoreStatus: type: object @@ -1472,7 +1479,7 @@ components: properties: type: type: string - enum: [ common_prefix, object ] + enum: [common_prefix, object] description: Path type, can either be 'common_prefix' or 'object' path: type: string @@ -1705,7 +1712,7 @@ components: - installation_id - reports - ExternalPrincipalList: + ExternalPrincipalList: type: object required: - pagination @@ -1752,7 +1759,7 @@ components: properties: status: type: string - enum: [ open, closed, merged ] + enum: [open, closed, merged] title: type: string description: @@ -1760,36 +1767,36 @@ components: PullRequest: allOf: - - $ref: '#/components/schemas/PullRequestBasic' - - required: + - $ref: '#/components/schemas/PullRequestBasic' + - required: - status - title - description - - type: object - required: - - id - - creation_date - - author - - source_branch - - destination_branch - properties: - id: - type: string - creation_date: - type: string - format: date-time - author: - type: string - source_branch: - type: string - destination_branch: - type: string - merged_commit_id: - type: string - description: the commit id of merged PRs - closed_date: - type: string - format: date-time + - type: object + required: + - id + - creation_date + - author + - source_branch + - destination_branch + properties: + id: + type: string + creation_date: + type: string + format: date-time + author: + type: string + source_branch: + type: string + destination_branch: + type: string + merged_commit_id: + type: string + description: the commit id of merged PRs + closed_date: + type: string + format: date-time PullRequestsList: type: object @@ -2668,11 +2675,11 @@ paths: - experimental operationId: createUserExternalPrincipal summary: attach external principal to user - requestBody: + requestBody: required: false - content: - application/json: - schema: + content: + application/json: + schema: $ref: "#/components/schemas/ExternalPrincipalCreation" responses: 201: @@ -2753,7 +2760,7 @@ paths: - external - experimental operationId: getExternalPrincipal - summary: describe external principal by id + summary: describe external principal by id responses: 200: description: external principal @@ -2899,7 +2906,7 @@ paths: description: too many requests default: $ref: "#/components/responses/ServerError" - + /repositories: get: tags: @@ -2908,6 +2915,7 @@ paths: - $ref: "#/components/parameters/PaginationPrefix" - $ref: "#/components/parameters/PaginationAfter" - $ref: "#/components/parameters/PaginationAmount" + - $ref: "#/components/parameters/SearchString" operationId: listRepositories summary: list repositories responses: @@ -4564,10 +4572,10 @@ paths: application/json: schema: $ref: "#/components/schemas/StagingMetadata" - + parameters: - $ref: "#/components/parameters/IfNoneMatch" - + responses: 200: # This actually violates HTTP, which requires returning 201 if a new object was @@ -5755,7 +5763,7 @@ paths: name: status schema: type: string - enum: [ open, closed, all ] + enum: [open, closed, all] default: all description: filter pull requests by status responses: diff --git a/clients/java/api/openapi.yaml b/clients/java/api/openapi.yaml index 01b85b7b499..c2bfede9523 100644 --- a/clients/java/api/openapi.yaml +++ b/clients/java/api/openapi.yaml @@ -1965,6 +1965,14 @@ paths: minimum: -1 type: integer style: form + - description: string for searching relevant entries + explode: true + in: query + name: search + required: false + schema: + type: string + style: form responses: "200": content: @@ -7387,6 +7395,15 @@ components: schema: type: string style: form + SearchString: + description: string for searching relevant entries + explode: true + in: query + name: search + required: false + schema: + type: string + style: form IfNoneMatch: description: Set to "*" to atomically allow the upload only if the key has no object yet. Other values are not supported. diff --git a/clients/java/docs/RepositoriesApi.md b/clients/java/docs/RepositoriesApi.md index 664fcc4ef43..6312770cd96 100644 --- a/clients/java/docs/RepositoriesApi.md +++ b/clients/java/docs/RepositoriesApi.md @@ -868,7 +868,7 @@ public class Example { # **listRepositories** -> RepositoryList listRepositories().prefix(prefix).after(after).amount(amount).execute(); +> RepositoryList listRepositories().prefix(prefix).after(after).amount(amount).search(search).execute(); list repositories @@ -918,11 +918,13 @@ public class Example { String prefix = "prefix_example"; // String | return items prefixed with this value String after = "after_example"; // String | return items after this value Integer amount = 100; // Integer | how many items to return + String search = "search_example"; // String | string for searching relevant entries try { RepositoryList result = apiInstance.listRepositories() .prefix(prefix) .after(after) .amount(amount) + .search(search) .execute(); System.out.println(result); } catch (ApiException e) { @@ -943,6 +945,7 @@ public class Example { | **prefix** | **String**| return items prefixed with this value | [optional] | | **after** | **String**| return items after this value | [optional] | | **amount** | **Integer**| how many items to return | [optional] [default to 100] | +| **search** | **String**| string for searching relevant entries | [optional] | ### Return type diff --git a/clients/java/src/main/java/io/lakefs/clients/sdk/RepositoriesApi.java b/clients/java/src/main/java/io/lakefs/clients/sdk/RepositoriesApi.java index 2b45a54a744..03277973622 100644 --- a/clients/java/src/main/java/io/lakefs/clients/sdk/RepositoriesApi.java +++ b/clients/java/src/main/java/io/lakefs/clients/sdk/RepositoriesApi.java @@ -1678,7 +1678,7 @@ public okhttp3.Call executeAsync(final ApiCallback> _callbac public APIgetRepositoryMetadataRequest getRepositoryMetadata(String repository) { return new APIgetRepositoryMetadataRequest(repository); } - private okhttp3.Call listRepositoriesCall(String prefix, String after, Integer amount, final ApiCallback _callback) throws ApiException { + private okhttp3.Call listRepositoriesCall(String prefix, String after, Integer amount, String search, final ApiCallback _callback) throws ApiException { String basePath = null; // Operation Servers String[] localBasePaths = new String[] { }; @@ -1715,6 +1715,10 @@ private okhttp3.Call listRepositoriesCall(String prefix, String after, Integer a localVarQueryParams.addAll(localVarApiClient.parameterToPair("amount", amount)); } + if (search != null) { + localVarQueryParams.addAll(localVarApiClient.parameterToPair("search", search)); + } + final String[] localVarAccepts = { "application/json" }; @@ -1735,21 +1739,21 @@ private okhttp3.Call listRepositoriesCall(String prefix, String after, Integer a } @SuppressWarnings("rawtypes") - private okhttp3.Call listRepositoriesValidateBeforeCall(String prefix, String after, Integer amount, final ApiCallback _callback) throws ApiException { - return listRepositoriesCall(prefix, after, amount, _callback); + private okhttp3.Call listRepositoriesValidateBeforeCall(String prefix, String after, Integer amount, String search, final ApiCallback _callback) throws ApiException { + return listRepositoriesCall(prefix, after, amount, search, _callback); } - private ApiResponse listRepositoriesWithHttpInfo(String prefix, String after, Integer amount) throws ApiException { - okhttp3.Call localVarCall = listRepositoriesValidateBeforeCall(prefix, after, amount, null); + private ApiResponse listRepositoriesWithHttpInfo(String prefix, String after, Integer amount, String search) throws ApiException { + okhttp3.Call localVarCall = listRepositoriesValidateBeforeCall(prefix, after, amount, search, null); Type localVarReturnType = new TypeToken(){}.getType(); return localVarApiClient.execute(localVarCall, localVarReturnType); } - private okhttp3.Call listRepositoriesAsync(String prefix, String after, Integer amount, final ApiCallback _callback) throws ApiException { + private okhttp3.Call listRepositoriesAsync(String prefix, String after, Integer amount, String search, final ApiCallback _callback) throws ApiException { - okhttp3.Call localVarCall = listRepositoriesValidateBeforeCall(prefix, after, amount, _callback); + okhttp3.Call localVarCall = listRepositoriesValidateBeforeCall(prefix, after, amount, search, _callback); Type localVarReturnType = new TypeToken(){}.getType(); localVarApiClient.executeAsync(localVarCall, localVarReturnType, _callback); return localVarCall; @@ -1759,6 +1763,7 @@ public class APIlistRepositoriesRequest { private String prefix; private String after; private Integer amount; + private String search; private APIlistRepositoriesRequest() { } @@ -1793,6 +1798,16 @@ public APIlistRepositoriesRequest amount(Integer amount) { return this; } + /** + * Set search + * @param search string for searching relevant entries (optional) + * @return APIlistRepositoriesRequest + */ + public APIlistRepositoriesRequest search(String search) { + this.search = search; + return this; + } + /** * Build call for listRepositories * @param _callback ApiCallback API callback @@ -1808,7 +1823,7 @@ public APIlistRepositoriesRequest amount(Integer amount) { */ public okhttp3.Call buildCall(final ApiCallback _callback) throws ApiException { - return listRepositoriesCall(prefix, after, amount, _callback); + return listRepositoriesCall(prefix, after, amount, search, _callback); } /** @@ -1825,7 +1840,7 @@ public okhttp3.Call buildCall(final ApiCallback _callback) throws ApiException { */ public RepositoryList execute() throws ApiException { - ApiResponse localVarResp = listRepositoriesWithHttpInfo(prefix, after, amount); + ApiResponse localVarResp = listRepositoriesWithHttpInfo(prefix, after, amount, search); return localVarResp.getData(); } @@ -1843,7 +1858,7 @@ public RepositoryList execute() throws ApiException { */ public ApiResponse executeWithHttpInfo() throws ApiException { - return listRepositoriesWithHttpInfo(prefix, after, amount); + return listRepositoriesWithHttpInfo(prefix, after, amount, search); } /** @@ -1861,7 +1876,7 @@ public ApiResponse executeWithHttpInfo() throws ApiException { */ public okhttp3.Call executeAsync(final ApiCallback _callback) throws ApiException { - return listRepositoriesAsync(prefix, after, amount, _callback); + return listRepositoriesAsync(prefix, after, amount, search, _callback); } } diff --git a/clients/java/src/test/java/io/lakefs/clients/sdk/RepositoriesApiTest.java b/clients/java/src/test/java/io/lakefs/clients/sdk/RepositoriesApiTest.java index 836119642ef..81882064f68 100644 --- a/clients/java/src/test/java/io/lakefs/clients/sdk/RepositoriesApiTest.java +++ b/clients/java/src/test/java/io/lakefs/clients/sdk/RepositoriesApiTest.java @@ -170,10 +170,12 @@ public void listRepositoriesTest() throws ApiException { String prefix = null; String after = null; Integer amount = null; + String search = null; RepositoryList response = api.listRepositories() .prefix(prefix) .after(after) .amount(amount) + .search(search) .execute(); // TODO: test validations } diff --git a/clients/python/docs/RepositoriesApi.md b/clients/python/docs/RepositoriesApi.md index 6db6216fc8d..2d640e50392 100644 --- a/clients/python/docs/RepositoriesApi.md +++ b/clients/python/docs/RepositoriesApi.md @@ -1013,7 +1013,7 @@ Name | Type | Description | Notes [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) # **list_repositories** -> RepositoryList list_repositories(prefix=prefix, after=after, amount=amount) +> RepositoryList list_repositories(prefix=prefix, after=after, amount=amount, search=search) list repositories @@ -1080,10 +1080,11 @@ with lakefs_sdk.ApiClient(configuration) as api_client: prefix = 'prefix_example' # str | return items prefixed with this value (optional) after = 'after_example' # str | return items after this value (optional) amount = 100 # int | how many items to return (optional) (default to 100) + search = 'search_example' # str | string for searching relevant entries (optional) try: # list repositories - api_response = api_instance.list_repositories(prefix=prefix, after=after, amount=amount) + api_response = api_instance.list_repositories(prefix=prefix, after=after, amount=amount, search=search) print("The response of RepositoriesApi->list_repositories:\n") pprint(api_response) except Exception as e: @@ -1100,6 +1101,7 @@ Name | Type | Description | Notes **prefix** | **str**| return items prefixed with this value | [optional] **after** | **str**| return items after this value | [optional] **amount** | **int**| how many items to return | [optional] [default to 100] + **search** | **str**| string for searching relevant entries | [optional] ### Return type diff --git a/clients/python/lakefs_sdk/api/repositories_api.py b/clients/python/lakefs_sdk/api/repositories_api.py index f31d2c1d818..8f717b953bb 100644 --- a/clients/python/lakefs_sdk/api/repositories_api.py +++ b/clients/python/lakefs_sdk/api/repositories_api.py @@ -1344,13 +1344,13 @@ def get_repository_metadata_with_http_info(self, repository : StrictStr, **kwarg _request_auth=_params.get('_request_auth')) @validate_arguments - def list_repositories(self, prefix : Annotated[Optional[StrictStr], Field(description="return items prefixed with this value")] = None, after : Annotated[Optional[StrictStr], Field(description="return items after this value")] = None, amount : Annotated[Optional[conint(strict=True, le=1000, ge=-1)], Field(description="how many items to return")] = None, **kwargs) -> RepositoryList: # noqa: E501 + def list_repositories(self, prefix : Annotated[Optional[StrictStr], Field(description="return items prefixed with this value")] = None, after : Annotated[Optional[StrictStr], Field(description="return items after this value")] = None, amount : Annotated[Optional[conint(strict=True, le=1000, ge=-1)], Field(description="how many items to return")] = None, search : Annotated[Optional[StrictStr], Field(description="string for searching relevant entries")] = None, **kwargs) -> RepositoryList: # noqa: E501 """list repositories # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True - >>> thread = api.list_repositories(prefix, after, amount, async_req=True) + >>> thread = api.list_repositories(prefix, after, amount, search, async_req=True) >>> result = thread.get() :param prefix: return items prefixed with this value @@ -1359,6 +1359,8 @@ def list_repositories(self, prefix : Annotated[Optional[StrictStr], Field(descri :type after: str :param amount: how many items to return :type amount: int + :param search: string for searching relevant entries + :type search: str :param async_req: Whether to execute the request asynchronously. :type async_req: bool, optional :param _request_timeout: timeout setting for this request. If one @@ -1373,16 +1375,16 @@ def list_repositories(self, prefix : Annotated[Optional[StrictStr], Field(descri kwargs['_return_http_data_only'] = True if '_preload_content' in kwargs: raise ValueError("Error! Please call the list_repositories_with_http_info method with `_preload_content` instead and obtain raw data from ApiResponse.raw_data") - return self.list_repositories_with_http_info(prefix, after, amount, **kwargs) # noqa: E501 + return self.list_repositories_with_http_info(prefix, after, amount, search, **kwargs) # noqa: E501 @validate_arguments - def list_repositories_with_http_info(self, prefix : Annotated[Optional[StrictStr], Field(description="return items prefixed with this value")] = None, after : Annotated[Optional[StrictStr], Field(description="return items after this value")] = None, amount : Annotated[Optional[conint(strict=True, le=1000, ge=-1)], Field(description="how many items to return")] = None, **kwargs) -> ApiResponse: # noqa: E501 + def list_repositories_with_http_info(self, prefix : Annotated[Optional[StrictStr], Field(description="return items prefixed with this value")] = None, after : Annotated[Optional[StrictStr], Field(description="return items after this value")] = None, amount : Annotated[Optional[conint(strict=True, le=1000, ge=-1)], Field(description="how many items to return")] = None, search : Annotated[Optional[StrictStr], Field(description="string for searching relevant entries")] = None, **kwargs) -> ApiResponse: # noqa: E501 """list repositories # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True - >>> thread = api.list_repositories_with_http_info(prefix, after, amount, async_req=True) + >>> thread = api.list_repositories_with_http_info(prefix, after, amount, search, async_req=True) >>> result = thread.get() :param prefix: return items prefixed with this value @@ -1391,6 +1393,8 @@ def list_repositories_with_http_info(self, prefix : Annotated[Optional[StrictStr :type after: str :param amount: how many items to return :type amount: int + :param search: string for searching relevant entries + :type search: str :param async_req: Whether to execute the request asynchronously. :type async_req: bool, optional :param _preload_content: if False, the ApiResponse.data will @@ -1421,7 +1425,8 @@ def list_repositories_with_http_info(self, prefix : Annotated[Optional[StrictStr _all_params = [ 'prefix', 'after', - 'amount' + 'amount', + 'search' ] _all_params.extend( [ @@ -1461,6 +1466,9 @@ def list_repositories_with_http_info(self, prefix : Annotated[Optional[StrictStr if _params.get('amount') is not None: # noqa: E501 _query_params.append(('amount', _params['amount'])) + if _params.get('search') is not None: # noqa: E501 + _query_params.append(('search', _params['search'])) + # process the header parameters _header_params = dict(_params.get('_headers', {})) # process the form parameters diff --git a/clients/rust/docs/RepositoriesApi.md b/clients/rust/docs/RepositoriesApi.md index 01567f9637a..3ac3c845855 100644 --- a/clients/rust/docs/RepositoriesApi.md +++ b/clients/rust/docs/RepositoriesApi.md @@ -278,7 +278,7 @@ Name | Type | Description | Required | Notes ## list_repositories -> models::RepositoryList list_repositories(prefix, after, amount) +> models::RepositoryList list_repositories(prefix, after, amount, search) list repositories ### Parameters @@ -289,6 +289,7 @@ Name | Type | Description | Required | Notes **prefix** | Option<**String**> | return items prefixed with this value | | **after** | Option<**String**> | return items after this value | | **amount** | Option<**i32**> | how many items to return | |[default to 100] +**search** | Option<**String**> | string for searching relevant entries | | ### Return type diff --git a/clients/rust/src/apis/repositories_api.rs b/clients/rust/src/apis/repositories_api.rs index 87217f1235c..8f895e2408f 100644 --- a/clients/rust/src/apis/repositories_api.rs +++ b/clients/rust/src/apis/repositories_api.rs @@ -483,7 +483,7 @@ pub async fn get_repository_metadata(configuration: &configuration::Configuratio } } -pub async fn list_repositories(configuration: &configuration::Configuration, prefix: Option<&str>, after: Option<&str>, amount: Option) -> Result> { +pub async fn list_repositories(configuration: &configuration::Configuration, prefix: Option<&str>, after: Option<&str>, amount: Option, search: Option<&str>) -> Result> { let local_var_configuration = configuration; let local_var_client = &local_var_configuration.client; @@ -500,6 +500,9 @@ pub async fn list_repositories(configuration: &configuration::Configuration, pre if let Some(ref local_var_str) = amount { local_var_req_builder = local_var_req_builder.query(&[("amount", &local_var_str.to_string())]); } + if let Some(ref local_var_str) = search { + local_var_req_builder = local_var_req_builder.query(&[("search", &local_var_str.to_string())]); + } if let Some(ref local_var_user_agent) = local_var_configuration.user_agent { local_var_req_builder = local_var_req_builder.header(reqwest::header::USER_AGENT, local_var_user_agent.clone()); } diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go index 9a3263adb2a..7e1e832f01a 100644 --- a/cmd/lakefs/cmd/run.go +++ b/cmd/lakefs/cmd/run.go @@ -437,7 +437,7 @@ func checkRepos(ctx context.Context, logger logging.Logger, authMetadataManager for hasMore { var err error var repos []*catalog.Repository - repos, hasMore, err = c.ListRepositories(ctx, -1, "", next) + repos, hasMore, err = c.ListRepositories(ctx, -1, "", "", next) if err != nil { logger.WithError(err).Fatal("Checking existing repositories failed") } diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index e4e16d94b0f..01cc2f46177 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -70,6 +70,13 @@ components: description: delimiter used to group common prefixes by schema: type: string + + SearchString: + in: query + name: search + description: string for searching relevant entries + schema: + type: string IfNoneMatch: in: header @@ -695,7 +702,7 @@ components: default: false hidden: type: boolean - description: When set, branch will not show up when listing branches by default. *EXPERIMENTAL* + description: When set, branch will not show up when listing branches by default. *EXPERIMENTAL* default: false TagCreation: @@ -742,7 +749,7 @@ components: error: type: string refs: - $ref: "#/components/schemas/RefsDump" + $ref: "#/components/schemas/RefsDump" RepositoryRestoreStatus: type: object @@ -1472,7 +1479,7 @@ components: properties: type: type: string - enum: [ common_prefix, object ] + enum: [common_prefix, object] description: Path type, can either be 'common_prefix' or 'object' path: type: string @@ -1705,7 +1712,7 @@ components: - installation_id - reports - ExternalPrincipalList: + ExternalPrincipalList: type: object required: - pagination @@ -1752,7 +1759,7 @@ components: properties: status: type: string - enum: [ open, closed, merged ] + enum: [open, closed, merged] title: type: string description: @@ -1760,36 +1767,36 @@ components: PullRequest: allOf: - - $ref: '#/components/schemas/PullRequestBasic' - - required: + - $ref: '#/components/schemas/PullRequestBasic' + - required: - status - title - description - - type: object - required: - - id - - creation_date - - author - - source_branch - - destination_branch - properties: - id: - type: string - creation_date: - type: string - format: date-time - author: - type: string - source_branch: - type: string - destination_branch: - type: string - merged_commit_id: - type: string - description: the commit id of merged PRs - closed_date: - type: string - format: date-time + - type: object + required: + - id + - creation_date + - author + - source_branch + - destination_branch + properties: + id: + type: string + creation_date: + type: string + format: date-time + author: + type: string + source_branch: + type: string + destination_branch: + type: string + merged_commit_id: + type: string + description: the commit id of merged PRs + closed_date: + type: string + format: date-time PullRequestsList: type: object @@ -2668,11 +2675,11 @@ paths: - experimental operationId: createUserExternalPrincipal summary: attach external principal to user - requestBody: + requestBody: required: false - content: - application/json: - schema: + content: + application/json: + schema: $ref: "#/components/schemas/ExternalPrincipalCreation" responses: 201: @@ -2753,7 +2760,7 @@ paths: - external - experimental operationId: getExternalPrincipal - summary: describe external principal by id + summary: describe external principal by id responses: 200: description: external principal @@ -2899,7 +2906,7 @@ paths: description: too many requests default: $ref: "#/components/responses/ServerError" - + /repositories: get: tags: @@ -2908,6 +2915,7 @@ paths: - $ref: "#/components/parameters/PaginationPrefix" - $ref: "#/components/parameters/PaginationAfter" - $ref: "#/components/parameters/PaginationAmount" + - $ref: "#/components/parameters/SearchString" operationId: listRepositories summary: list repositories responses: @@ -4564,10 +4572,10 @@ paths: application/json: schema: $ref: "#/components/schemas/StagingMetadata" - + parameters: - $ref: "#/components/parameters/IfNoneMatch" - + responses: 200: # This actually violates HTTP, which requires returning 201 if a new object was @@ -5755,7 +5763,7 @@ paths: name: status schema: type: string - enum: [ open, closed, all ] + enum: [open, closed, all] default: all description: filter pull requests by status responses: diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 89133c4d407..3d12b009eaf 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -1894,7 +1894,7 @@ func (c *Controller) ListRepositories(w http.ResponseWriter, r *http.Request, pa ctx := r.Context() c.LogAction(ctx, "list_repos", r, "", "", "") - repos, hasMore, err := c.Catalog.ListRepositories(ctx, paginationAmount(params.Amount), paginationPrefix(params.Prefix), paginationAfter(params.After)) + repos, hasMore, err := c.Catalog.ListRepositories(ctx, paginationAmount(params.Amount), paginationPrefix(params.Prefix), search(params.Search), paginationAfter(params.After)) if c.handleAPIError(ctx, w, r, err) { return } @@ -5471,6 +5471,13 @@ func paginationDelimiter(v *apigen.PaginationDelimiter) string { return string(*v) } +func search(v *apigen.SearchString) string { + if v == nil { + return "" + } + return string(*v) +} + func paginationAmount(v *apigen.PaginationAmount) int { if v == nil { return DefaultPerPage diff --git a/pkg/catalog/catalog.go b/pkg/catalog/catalog.go index 99cb8d14ed5..ee405570c8d 100644 --- a/pkg/catalog/catalog.go +++ b/pkg/catalog/catalog.go @@ -584,8 +584,9 @@ func (c *Catalog) DeleteRepositoryMetadata(ctx context.Context, repository strin } // ListRepositories list repository information, the bool returned is true when more repositories can be listed. -// In this case, pass the last repository name as 'after' on the next call to ListRepositories -func (c *Catalog) ListRepositories(ctx context.Context, limit int, prefix, after string) ([]*Repository, bool, error) { +// In this case, pass the last repository name as 'after' on the next call to ListRepositories. Results can be +// filtered by specifying a prefix or, more generally, a searchString. +func (c *Catalog) ListRepositories(ctx context.Context, limit int, prefix, searchString, after string) ([]*Repository, bool, error) { // normalize limit if limit < 0 || limit > ListRepositoriesLimitMax { limit = ListRepositoriesLimitMax @@ -614,7 +615,9 @@ func (c *Catalog) ListRepositories(ctx context.Context, limit int, prefix, after if !strings.HasPrefix(string(record.RepositoryID), prefix) { break } - + if !strings.Contains(string(record.RepositoryID), searchString) { + continue + } if record.RepositoryID == afterRepositoryID { continue } diff --git a/pkg/catalog/catalog_test.go b/pkg/catalog/catalog_test.go index 6023cad2352..11877a52556 100644 --- a/pkg/catalog/catalog_test.go +++ b/pkg/catalog/catalog_test.go @@ -72,13 +72,18 @@ func TestCatalog_ListRepositories(t *testing.T) { // prepare data tests now := time.Now() gravelerData := []*graveler.RepositoryRecord{ - {RepositoryID: "repo1", Repository: &graveler.Repository{StorageNamespace: "storage1", CreationDate: now, DefaultBranchID: "main1"}}, - {RepositoryID: "repo2", Repository: &graveler.Repository{StorageNamespace: "storage2", CreationDate: now, DefaultBranchID: "main2"}}, - {RepositoryID: "repo3", Repository: &graveler.Repository{StorageNamespace: "storage3", CreationDate: now, DefaultBranchID: "main3"}}, + {RepositoryID: "re", Repository: &graveler.Repository{StorageNamespace: "storage1", CreationDate: now, DefaultBranchID: "main1"}}, + {RepositoryID: "repo1", Repository: &graveler.Repository{StorageNamespace: "storage2", CreationDate: now, DefaultBranchID: "main2"}}, + {RepositoryID: "repo2", Repository: &graveler.Repository{StorageNamespace: "storage3", CreationDate: now, DefaultBranchID: "main3"}}, + {RepositoryID: "repo22", Repository: &graveler.Repository{StorageNamespace: "storage4", CreationDate: now, DefaultBranchID: "main4"}}, + {RepositoryID: "repo23", Repository: &graveler.Repository{StorageNamespace: "storage5", CreationDate: now, DefaultBranchID: "main5"}}, + {RepositoryID: "repo3", Repository: &graveler.Repository{StorageNamespace: "storage6", CreationDate: now, DefaultBranchID: "main6"}}, } type args struct { - limit int - after string + limit int + after string + prefix string + searchString string } tests := []struct { name string @@ -90,37 +95,75 @@ func TestCatalog_ListRepositories(t *testing.T) { { name: "all", args: args{ - limit: -1, - after: "", + limit: -1, + after: "", + prefix: "", + searchString: "", }, want: []*catalog.Repository{ - {Name: "repo1", StorageNamespace: "storage1", DefaultBranch: "main1", CreationDate: now}, - {Name: "repo2", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, - {Name: "repo3", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, + {Name: "re", StorageNamespace: "storage1", DefaultBranch: "main1", CreationDate: now}, + {Name: "repo1", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, + {Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, + {Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, + {Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, + {Name: "repo3", StorageNamespace: "storage6", DefaultBranch: "main6", CreationDate: now}, }, wantHasMore: false, wantErr: false, }, { - name: "first", + name: "firstCatalog", args: args{ - limit: 1, - after: "", + limit: 1, + after: "", + prefix: "", + searchString: "", }, want: []*catalog.Repository{ - {Name: "repo1", StorageNamespace: "storage1", DefaultBranch: "main1", CreationDate: now}, + {Name: "re", StorageNamespace: "storage1", DefaultBranch: "main1", CreationDate: now}, }, wantHasMore: true, wantErr: false, }, { - name: "second", + name: "secondCatalog", + args: args{ + limit: 1, + after: "re", + prefix: "", + searchString: "", + }, + want: []*catalog.Repository{ + {Name: "repo1", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, + }, + wantHasMore: true, + wantErr: false, + }, + { + name: "thirdCatalog", + args: args{ + limit: 2, + after: "repo1", + prefix: "", + searchString: "", + }, + want: []*catalog.Repository{ + {Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, + {Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, + }, + wantHasMore: true, + wantErr: false, + }, + { + name: "prefix", args: args{ - limit: 1, - after: "repo1", + limit: 1, + after: "", + prefix: "repo", + searchString: "", }, want: []*catalog.Repository{ - {Name: "repo2", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, + {Name: "repo1", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, }, wantHasMore: true, wantErr: false, @@ -128,12 +171,74 @@ func TestCatalog_ListRepositories(t *testing.T) { { name: "last2", args: args{ - limit: 10, - after: "repo1", + limit: 10, + after: "repo22", + prefix: "", + searchString: "", + }, + want: []*catalog.Repository{ + {Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, + {Name: "repo3", StorageNamespace: "storage6", DefaultBranch: "main6", CreationDate: now}, + }, + wantHasMore: false, + wantErr: false, + }, + { + name: "common_searchString", + args: args{ + limit: -1, + after: "", + prefix: "", + searchString: "o2", + }, + want: []*catalog.Repository{ + {Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, + {Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, + {Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, + }, + wantHasMore: false, + wantErr: false, + }, + { + name: "common_pagedSearchString1", + args: args{ + limit: 2, + after: "", + prefix: "", + searchString: "o2", + }, + want: []*catalog.Repository{ + {Name: "repo2", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, + {Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, + }, + wantHasMore: true, + wantErr: false, + }, + { + name: "common_pagedSearchString2", + args: args{ + limit: 2, + after: "repo22", + prefix: "", + searchString: "o2", + }, + want: []*catalog.Repository{ + {Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, + }, + wantHasMore: false, + wantErr: false, + }, + { + name: "after_and_searchString", + args: args{ + limit: -1, + after: "repo2", + prefix: "", + searchString: "o2", }, want: []*catalog.Repository{ - {Name: "repo2", StorageNamespace: "storage2", DefaultBranch: "main2", CreationDate: now}, - {Name: "repo3", StorageNamespace: "storage3", DefaultBranch: "main3", CreationDate: now}, + {Name: "repo22", StorageNamespace: "storage4", DefaultBranch: "main4", CreationDate: now}, + {Name: "repo23", StorageNamespace: "storage5", DefaultBranch: "main5", CreationDate: now}, }, wantHasMore: false, wantErr: false, @@ -151,7 +256,7 @@ func TestCatalog_ListRepositories(t *testing.T) { } // test method ctx := context.Background() - got, hasMore, err := c.ListRepositories(ctx, tt.args.limit, "", tt.args.after) + got, hasMore, err := c.ListRepositories(ctx, tt.args.limit, tt.args.prefix, tt.args.searchString, tt.args.after) if tt.wantErr && err == nil { t.Fatal("ListRepositories err nil, expected error") } @@ -162,7 +267,7 @@ func TestCatalog_ListRepositories(t *testing.T) { t.Errorf("ListRepositories hasMore %t, expected %t", hasMore, tt.wantHasMore) } if diff := deep.Equal(got, tt.want); diff != nil { - t.Error("ListRepositories diff found:", diff) + t.Error("ListRepositories diff found:\n", diff) } }) } diff --git a/pkg/gateway/operations/listbuckets.go b/pkg/gateway/operations/listbuckets.go index 8fde6c8151c..8bd9e7a9099 100644 --- a/pkg/gateway/operations/listbuckets.go +++ b/pkg/gateway/operations/listbuckets.go @@ -31,7 +31,7 @@ func (controller *ListBuckets) Handle(w http.ResponseWriter, req *http.Request, var after string for { // list repositories - repos, hasMore, err := o.Catalog.ListRepositories(req.Context(), -1, "", after) + repos, hasMore, err := o.Catalog.ListRepositories(req.Context(), -1, "", "", after) if err != nil { _ = o.EncodeError(w, req, err, errors.Codes.ToAPIErr(errors.ErrInternalError)) return diff --git a/webui/src/lib/api/index.js b/webui/src/lib/api/index.js index cb4f3dccce5..1d9356ac64e 100644 --- a/webui/src/lib/api/index.js +++ b/webui/src/lib/api/index.js @@ -61,8 +61,8 @@ export const parseRawHeaders = (rawHeaders) => { const headerLines = cleanedHeadersString.split('\n'); const parsedHeaders = headerLines.reduce((acc, line) => { let [key, ...value] = line.split(':'); // split into key and the rest of the value - key = key.trim(); - value = value.join(':').trim(); + key = key.trim(); + value = value.join(':').trim(); if (key && value) { acc[key.toLowerCase()] = value; } @@ -458,8 +458,8 @@ class Repositories { return response.json(); } - async list(prefix = "", after = "", amount = DEFAULT_LISTING_AMOUNT) { - const query = qs({prefix, after, amount}); + async list(search = "", after = "", amount = DEFAULT_LISTING_AMOUNT) { + const query = qs({search, after, amount}); const response = await apiRequest(`/repositories?${query}`); if (response.status !== 200) { throw new Error(`could not list repositories: ${await extractError(response)}`); diff --git a/webui/src/pages/repositories/index.jsx b/webui/src/pages/repositories/index.jsx index 3fe13e7599f..d138dcc5b96 100644 --- a/webui/src/pages/repositories/index.jsx +++ b/webui/src/pages/repositories/index.jsx @@ -130,17 +130,17 @@ const GetStarted = ({onCreateSampleRepo, onCreateEmptyRepo, creatingRepo, create ); }; -const RepositoryList = ({ onPaginate, prefix, after, refresh, onCreateSampleRepo, onCreateEmptyRepo, toggleShowActionsBar, creatingRepo, createRepoError }) => { +const RepositoryList = ({ onPaginate, search, after, refresh, onCreateSampleRepo, onCreateEmptyRepo, toggleShowActionsBar, creatingRepo, createRepoError }) => { const {results, loading, error, nextPage} = useAPIWithPagination(() => { - return repositories.list(prefix, after); - }, [refresh, prefix, after]); + return repositories.list(search, after); + }, [refresh, search, after]); useEffect(() => { toggleShowActionsBar(); }, [toggleShowActionsBar]); if (loading) return ; if (error) return ; - if (!after && !prefix && results.length === 0) { + if (!after && !search && results.length === 0) { return ; } @@ -190,10 +190,10 @@ const RepositoriesPage = () => { const [creatingRepo, setCreatingRepo] = useState(false); const [showActionsBar, setShowActionsBar] = useState(false); - const routerPfx = (router.query.prefix) ? router.query.prefix : ""; - const [prefix, setPrefix] = useDebouncedState( + const routerPfx = (router.query.search) ? router.query.search : ""; + const [search, setSearch] = useDebouncedState( routerPfx, - (prefix) => router.push({pathname: `/repositories`, query: {prefix}}) + (search) => router.push({pathname: `/repositories`, query: {search}}) ); const { response, error: err, loading } = useAPI(() => config.getStorageConfig()); @@ -254,8 +254,8 @@ const RepositoriesPage = () => { setPrefix(event.target.value)} + value={search} + onChange={event => setSearch(event.target.value)} /> @@ -267,12 +267,12 @@ const RepositoriesPage = () => { } { const query = {after}; - if (router.query.prefix) query.prefix = router.query.prefix; + if (router.query.search) query.search = router.query.search; router.push({pathname: `/repositories`, query}); }} onCreateSampleRepo={createSampleRepoButtonCallback}