From fea5c102ca0bed98480e6af9ab8dd1d1e7cc00f0 Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Thu, 18 Jul 2024 17:22:41 -0700 Subject: [PATCH 1/5] Add smithy-generated existing spec Signed-off-by: Daniel Widdis --- spec/schemas/search_pipeline._common.yaml | 164 +++++++++++++++++++++- 1 file changed, 163 insertions(+), 1 deletion(-) diff --git a/spec/schemas/search_pipeline._common.yaml b/spec/schemas/search_pipeline._common.yaml index 94fd3747..c0b088ad 100644 --- a/spec/schemas/search_pipeline._common.yaml +++ b/spec/schemas/search_pipeline._common.yaml @@ -23,7 +23,7 @@ components: response_processors: type: array items: - $ref: '#/components/schemas/RequestProcessor' + $ref: '#/components/schemas/ResponseProcessor' phase_results_processors: type: array items: @@ -187,6 +187,168 @@ components: type: string required: - sample_factor + ResponseProcessor: + oneOf: + - type: object + title: personalize_search_ranking + properties: + personalize_search_ranking: + "$ref": "#/components/schemas/PersonalizeSearchRankingResponseProcessor" + required: + - personalize_search_ranking + - type: object + title: retrieval_augmented_generation + properties: + retrieval_augmented_generation: + "$ref": "#/components/schemas/RetrievalAugmentedGenerationResponseProcessor" + required: + - retrieval_augmented_generation + - type: object + title: rename_field + properties: + rename_field: + "$ref": "#/components/schemas/RenameFieldResponseProcessor" + required: + - rename_field + - type: object + title: rerank + properties: + rerank: + "$ref": "#/components/schemas/RerankResponseProcessor" + required: + - rerank + - type: object + title: collapse + properties: + collapse: + "$ref": "#/components/schemas/CollapseResponseProcessor" + required: + - collapse + - type: object + title: truncate_hits + properties: + truncate_hits: + "$ref": "#/components/schemas/TruncateHitsResponseProcessor" + required: + - truncate_hits + PersonalizeSearchRankingResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + campaign_arn: + type: string + recipe: + type: string + weight: + type: number + format: float + item_id_field: + type: string + iam_role_arn: + type: string + required: + - campaign_arn + - recipe + - weight + RetrievalAugmentedGenerationResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + model_id: + type: string + context_field_list: + type: array + items: + type: string + system_prompt: + type: string + user_instructions: + type: string + required: + - context_field_list + - model_id + RenameFieldResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + target_field: + type: string + required: + - field + - target_field + RerankContext: + type: object + properties: + document_fields: + type: array + items: + type: string + required: + - document_fields + MLOpenSearchReranker: + type: object + properties: + model_id: + type: string + required: + - model_id + RerankResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + ml_opensearch: + "$ref": "#/components/schemas/MLOpenSearchReranker" + context: + "$ref": "#/components/schemas/RerankContext" + CollapseResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + context_prefix: + type: string + required: + - field + TruncateHitsResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + target_size: + type: integer + format: int32 + context_prefix: + type: string PhaseResultsProcessor: oneOf: - type: object From a347df36d9124c4bc67b4d94e61eda11c64e8330 Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Thu, 18 Jul 2024 17:31:45 -0700 Subject: [PATCH 2/5] Add specs for sort and split processors Signed-off-by: Daniel Widdis --- spec/schemas/search_pipeline._common.yaml | 51 +++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/spec/schemas/search_pipeline._common.yaml b/spec/schemas/search_pipeline._common.yaml index c0b088ad..fdde558b 100644 --- a/spec/schemas/search_pipeline._common.yaml +++ b/spec/schemas/search_pipeline._common.yaml @@ -231,6 +231,20 @@ components: "$ref": "#/components/schemas/TruncateHitsResponseProcessor" required: - truncate_hits + - type: object + title: sort + properties: + sort: + "$ref": "#/components/schemas/SortResponseProcessor" + required: + - truncate_hits + - type: object + title: split + properties: + split: + "$ref": "#/components/schemas/SplitResponseProcessor" + required: + - truncate_hits PersonalizeSearchRankingResponseProcessor: type: object properties: @@ -349,6 +363,43 @@ components: format: int32 context_prefix: type: string + SortResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + order: + type: string + target_field: + type: string + required: + - field + SplitResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + separator: + type: string + preserve_trailing: + type: boolean + target_field: + type: string + required: + - field + - separator PhaseResultsProcessor: oneOf: - type: object From d034e66c24cf6899d9355f9439a13d9e25b3646a Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Thu, 18 Jul 2024 17:52:25 -0700 Subject: [PATCH 3/5] Add a test Signed-off-by: Daniel Widdis --- tests/_core/search/pipeline.yaml | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/_core/search/pipeline.yaml diff --git a/tests/_core/search/pipeline.yaml b/tests/_core/search/pipeline.yaml new file mode 100644 index 00000000..e8a077b7 --- /dev/null +++ b/tests/_core/search/pipeline.yaml @@ -0,0 +1,37 @@ +$schema: ../../json_schemas/test_story.schema.yaml + +description: | + Test the creation of a search pipeline with sort and split response processors. +epilogues: + - path: /_search/pipeline/sorting_pipeline + method: DELETE + status: [200, 404] +version: '>= 2.16' +chapters: + - synopsis: Create search pipeline for sorting and splitting. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: sorting_pipeline + request_body: + payload: + response_processors: + - sort: + field: "message" + order: "asc" + target_field: "sorted_message" + - split: + field: "message" + separator: ", " + target_field: "split_message" + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: sorting_pipeline + response: + status: 200 From 78507c2faf0e3d5113d6821638d2433a47cb09c3 Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Thu, 18 Jul 2024 18:15:58 -0700 Subject: [PATCH 4/5] Fix lint errors Signed-off-by: Daniel Widdis --- .cspell | 3 + spec/schemas/search_pipeline._common.yaml | 116 +++++++++++----------- tests/_core/search/pipeline.yaml | 14 +-- 3 files changed, 68 insertions(+), 65 deletions(-) diff --git a/.cspell b/.cspell index f292a14d..15eb18da 100644 --- a/.cspell +++ b/.cspell @@ -30,6 +30,8 @@ OSCPU Oversample Rebalance Reindex +Rerank +Reranker Rethrottle Rolespan Rollup @@ -138,6 +140,7 @@ readingform rebalance recoverysource reindex +rerank relo reloadcerts remotestore diff --git a/spec/schemas/search_pipeline._common.yaml b/spec/schemas/search_pipeline._common.yaml index fdde558b..01c96e4e 100644 --- a/spec/schemas/search_pipeline._common.yaml +++ b/spec/schemas/search_pipeline._common.yaml @@ -189,62 +189,62 @@ components: - sample_factor ResponseProcessor: oneOf: - - type: object - title: personalize_search_ranking - properties: - personalize_search_ranking: - "$ref": "#/components/schemas/PersonalizeSearchRankingResponseProcessor" - required: - - personalize_search_ranking - - type: object - title: retrieval_augmented_generation - properties: - retrieval_augmented_generation: - "$ref": "#/components/schemas/RetrievalAugmentedGenerationResponseProcessor" - required: - - retrieval_augmented_generation - - type: object - title: rename_field - properties: - rename_field: - "$ref": "#/components/schemas/RenameFieldResponseProcessor" - required: - - rename_field - - type: object - title: rerank - properties: - rerank: - "$ref": "#/components/schemas/RerankResponseProcessor" - required: - - rerank - - type: object - title: collapse - properties: - collapse: - "$ref": "#/components/schemas/CollapseResponseProcessor" - required: - - collapse - - type: object - title: truncate_hits - properties: - truncate_hits: - "$ref": "#/components/schemas/TruncateHitsResponseProcessor" - required: - - truncate_hits - - type: object - title: sort - properties: - sort: - "$ref": "#/components/schemas/SortResponseProcessor" - required: - - truncate_hits - - type: object - title: split - properties: - split: - "$ref": "#/components/schemas/SplitResponseProcessor" - required: - - truncate_hits + - type: object + title: personalize_search_ranking + properties: + personalize_search_ranking: + $ref: '#/components/schemas/PersonalizeSearchRankingResponseProcessor' + required: + - personalize_search_ranking + - type: object + title: retrieval_augmented_generation + properties: + retrieval_augmented_generation: + $ref: '#/components/schemas/RetrievalAugmentedGenerationResponseProcessor' + required: + - retrieval_augmented_generation + - type: object + title: rename_field + properties: + rename_field: + $ref: '#/components/schemas/RenameFieldResponseProcessor' + required: + - rename_field + - type: object + title: rerank + properties: + rerank: + $ref: '#/components/schemas/RerankResponseProcessor' + required: + - rerank + - type: object + title: collapse + properties: + collapse: + $ref: '#/components/schemas/CollapseResponseProcessor' + required: + - collapse + - type: object + title: truncate_hits + properties: + truncate_hits: + $ref: '#/components/schemas/TruncateHitsResponseProcessor' + required: + - truncate_hits + - type: object + title: sort + properties: + sort: + $ref: '#/components/schemas/SortResponseProcessor' + required: + - truncate_hits + - type: object + title: split + properties: + split: + $ref: '#/components/schemas/SplitResponseProcessor' + required: + - truncate_hits PersonalizeSearchRankingResponseProcessor: type: object properties: @@ -331,9 +331,9 @@ components: ignore_failure: type: boolean ml_opensearch: - "$ref": "#/components/schemas/MLOpenSearchReranker" + $ref: '#/components/schemas/MLOpenSearchReranker' context: - "$ref": "#/components/schemas/RerankContext" + $ref: '#/components/schemas/RerankContext' CollapseResponseProcessor: type: object properties: diff --git a/tests/_core/search/pipeline.yaml b/tests/_core/search/pipeline.yaml index e8a077b7..4172d0e6 100644 --- a/tests/_core/search/pipeline.yaml +++ b/tests/_core/search/pipeline.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: | Test the creation of a search pipeline with sort and split response processors. @@ -17,13 +17,13 @@ chapters: payload: response_processors: - sort: - field: "message" - order: "asc" - target_field: "sorted_message" + field: 'message' + order: 'asc' + target_field: 'sorted_message' - split: - field: "message" - separator: ", " - target_field: "split_message" + field: 'message' + separator: ', ' + target_field: 'split_message' response: status: 200 payload: From 7da742e3197bd5a653f5ac5f58fef2fd76df9de1 Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Thu, 18 Jul 2024 18:35:31 -0700 Subject: [PATCH 5/5] More lint errors, change log Signed-off-by: Daniel Widdis --- CHANGELOG.md | 1 + spec/schemas/search_pipeline._common.yaml | 42 +++++++++++------------ tests/_core/search/pipeline.yaml | 10 +++--- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d520d3f8..b04e63af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `is_hidden` to `/{index}/_alias/{name}` and `/{index}/_aliases/{name}` ([#429](https://github.com/opensearch-project/opensearch-api-specification/pull/429)) - Added `ignore_unmapped` to `GeoDistanceQuery` ([#427](https://github.com/opensearch-project/opensearch-api-specification/pull/427)) - Added missing variants of `indices.put_alias` ([#434](https://github.com/opensearch-project/opensearch-api-specification/pull/434)) +- Added missing search response processors and new `sort` and `split` processors ([#440](https://github.com/opensearch-project/opensearch-api-specification/pull/440)) ### Changed diff --git a/spec/schemas/search_pipeline._common.yaml b/spec/schemas/search_pipeline._common.yaml index 01c96e4e..ebb14ee9 100644 --- a/spec/schemas/search_pipeline._common.yaml +++ b/spec/schemas/search_pipeline._common.yaml @@ -195,56 +195,56 @@ components: personalize_search_ranking: $ref: '#/components/schemas/PersonalizeSearchRankingResponseProcessor' required: - - personalize_search_ranking + - personalize_search_ranking - type: object title: retrieval_augmented_generation properties: retrieval_augmented_generation: $ref: '#/components/schemas/RetrievalAugmentedGenerationResponseProcessor' required: - - retrieval_augmented_generation + - retrieval_augmented_generation - type: object title: rename_field properties: rename_field: $ref: '#/components/schemas/RenameFieldResponseProcessor' required: - - rename_field + - rename_field - type: object title: rerank properties: rerank: $ref: '#/components/schemas/RerankResponseProcessor' required: - - rerank + - rerank - type: object title: collapse properties: collapse: $ref: '#/components/schemas/CollapseResponseProcessor' required: - - collapse + - collapse - type: object title: truncate_hits properties: truncate_hits: $ref: '#/components/schemas/TruncateHitsResponseProcessor' required: - - truncate_hits + - truncate_hits - type: object title: sort properties: sort: $ref: '#/components/schemas/SortResponseProcessor' required: - - truncate_hits + - sort - type: object title: split properties: split: $ref: '#/components/schemas/SplitResponseProcessor' required: - - truncate_hits + - split PersonalizeSearchRankingResponseProcessor: type: object properties: @@ -266,9 +266,9 @@ components: iam_role_arn: type: string required: - - campaign_arn - - recipe - - weight + - campaign_arn + - recipe + - weight RetrievalAugmentedGenerationResponseProcessor: type: object properties: @@ -287,8 +287,8 @@ components: user_instructions: type: string required: - - context_field_list - - model_id + - context_field_list + - model_id RenameFieldResponseProcessor: type: object properties: @@ -303,8 +303,8 @@ components: target_field: type: string required: - - field - - target_field + - field + - target_field RerankContext: type: object properties: @@ -313,14 +313,14 @@ components: items: type: string required: - - document_fields + - document_fields MLOpenSearchReranker: type: object properties: model_id: type: string required: - - model_id + - model_id RerankResponseProcessor: type: object properties: @@ -348,7 +348,7 @@ components: context_prefix: type: string required: - - field + - field TruncateHitsResponseProcessor: type: object properties: @@ -379,7 +379,7 @@ components: target_field: type: string required: - - field + - field SplitResponseProcessor: type: object properties: @@ -398,8 +398,8 @@ components: target_field: type: string required: - - field - - separator + - field + - separator PhaseResultsProcessor: oneOf: - type: object diff --git a/tests/_core/search/pipeline.yaml b/tests/_core/search/pipeline.yaml index 4172d0e6..f394cd88 100644 --- a/tests/_core/search/pipeline.yaml +++ b/tests/_core/search/pipeline.yaml @@ -17,13 +17,13 @@ chapters: payload: response_processors: - sort: - field: 'message' - order: 'asc' - target_field: 'sorted_message' + field: message + order: asc + target_field: sorted_message - split: - field: 'message' + field: message separator: ', ' - target_field: 'split_message' + target_field: split_message response: status: 200 payload: