From 7dee041660986c5eb4bc85c2d19af006730746e9 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 14 Aug 2024 08:03:46 -0400 Subject: [PATCH] Added support for testing multiple distributions. (#483) * Added support for testing multiple distributions. Signed-off-by: dblock * Added x-distributions-included and excluded. Signed-off-by: dblock * Move node fetching to a prologue. Signed-off-by: dblock * Undo semver satisfies changes. Signed-off-by: dblock * Move OPENSEARCH_DISTRIBUTION_OPTION. Signed-off-by: dblock --------- Signed-off-by: dblock --- CHANGELOG.md | 6 ++ DEVELOPER_GUIDE.md | 4 ++ TESTING_GUIDE.md | 55 ++++++++++++++++++- json_schemas/test_story.schema.yaml | 11 ++++ spec/namespaces/_core.yaml | 2 + spec/schemas/nodes._common.yaml | 18 +++++- spec/schemas/nodes.info.yaml | 17 +----- tests/default/_core/info.yaml | 3 + .../_core/search/rest_total_hits_as_int.yaml | 1 - tests/default/cat/health.yaml | 10 ++-- tests/default/cat/indices.yaml | 4 ++ tests/default/cat/nodeattrs.yaml | 2 + tests/default/indices/cache.yaml | 2 + tests/default/indices/dangling.yaml | 2 + tests/default/indices/forcemerge.yaml | 2 + tests/default/indices/segments.yaml | 3 +- tests/default/indices/settings.yaml | 4 ++ tests/default/ml/model_groups.yaml | 2 + tests/default/ml/models.yaml | 2 + tests/default/security/api/certificates.yaml | 22 ++++---- tools/src/OpenSearchHttpClient.ts | 1 + tools/src/_utils/semver.ts | 3 +- tools/src/linter/SchemasValidator.ts | 4 +- tools/src/linter/components/OperationGroup.ts | 2 +- tools/src/merger/OpenApiVersionExtractor.ts | 38 +++++++++++-- tools/src/tester/MergedOpenApiSpec.ts | 8 ++- tools/src/tester/SchemaValidator.ts | 4 +- tools/src/tester/StoryEvaluator.ts | 21 +++++-- tools/src/tester/TestRunner.ts | 16 ++++-- tools/src/tester/test.ts | 7 ++- tools/src/tester/types/story.types.ts | 15 ++++- tools/src/types.ts | 2 + .../merger/OpenApiVersionExtractor.test.ts | 12 ++-- .../fixtures/extractor/expected_1.3.yaml | 6 ++ .../fixtures/extractor/expected_2.0.yaml | 6 ++ tools/tests/tester/MergedOpenApiSpec.test.ts | 53 +++++++++++++++--- .../fixtures/evals/skipped/distributions.yaml | 6 ++ .../specs/complete/namespaces/index.yaml | 18 ++++++ .../stories/skipped/distributions.yaml | 9 +++ tools/tests/tester/helpers.ts | 2 +- tools/tests/tester/integ/TestRunner.test.ts | 7 ++- 41 files changed, 332 insertions(+), 80 deletions(-) create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index f99a07a7..b8fd3cef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) - Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505)) +- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added support for annotating and testing the API spec against multiple OpenSearch distributions ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `read_time`, `write_time`, `queue_size` and `io_time_in_millis` to `IoStatDevice` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `total_rejections_breakup` to `ShardIndexingPressureStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `cancelled_task_percentage` and `current_cancellation_eligible_tasks_count` to `ShardSearchBackpressureTaskCancellationStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Changed @@ -116,6 +121,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) - Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) - Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Fixed required parameters in `NodeInfo` and `NodeOperatingSystemInfo` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Security diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 951f00cd..43c30d7f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -146,6 +146,10 @@ This repository includes several OpenAPI Specification Extensions to fill in any - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them. - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml). - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema. +- `x-distributions-included`: Contains a list of distributions known to include the API. +- `x-distributions-excluded`: Contains a list of distributions known to exclude the API. + +Use `opensearch.org` for the official distribution in `x-distributions-*`, `amazon-managed` for Amazon Managed OpenSearch, and `amazon-serverless` for Amazon OpenSearch Serverless. ## Writing Spec Tests diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 1328cc86..d35c96aa 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -11,6 +11,7 @@ - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) - [Managing Versions](#managing-versions) + - [Managing Distributions](#managing-distributions) - [Waiting for Tasks](#waiting-for-tasks) - [Warnings](#warnings) - [multiple-paths-detected](#multiple-paths-detected) @@ -209,7 +210,59 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne status: 200 ``` -The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. +The test tool will fetch the server version when it starts and use it automatically. The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. + +### Managing Distributions + +OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions-included` or `x-distributions-excluded` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not. + +```yaml +/: + get: + operationId: info.0 + x-distributions-included: + - opensearch.org + - amazon-managed + x-distributions-excluded: + - amazon-serverless + description: Returns basic information about the cluster. +``` + +Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it. + +```yaml +description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org +chapters: + - synopsis: Get server info. + path: / + method: GET + response: + status: 200 +``` + +To test a particular distribution pass `--opensearch-distribution` to the test tool. For example, the following runs tests against an Amazon Managed OpenSearch instance. + +```bash +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +export AWS_SESSION_TOKEN=... +export AWS_REGION=us-west-2 + +export OPENSEARCH_URL=https://....us-west-2.es.amazonaws.com + +npm run test:spec -- --opensearch-distribution=amazon-managed +``` + +The output will visible skip APIs that are not available in the `amazon-managed` distribution. + +``` +PASSED _core/bulk.yaml (.../_core/bulk.yaml) +PASSED _core/info.yaml (.../_core/info.yaml) +SKIPPED indices/forcemerge.yaml (Skipped because distribution amazon-managed is not opensearch.org.) +``` ### Waiting for Tasks diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 6d191650..928ba4ba 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -21,6 +21,8 @@ properties: $ref: '#/definitions/Chapter' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' required: [chapters,description] additionalProperties: false @@ -85,6 +87,8 @@ definitions: $ref: '#/definitions/Output' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' retry: $ref: '#/definitions/Retry' required: [method, path] @@ -106,6 +110,13 @@ definitions: The semver range to execute the story or chapter against. type: string + Distributions: + description: | + The list of distributions that support this API. + type: array + items: + type: string + Retry: description: | Number of times to retry on error. diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 9aa5dbea..bc90411b 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -9,6 +9,8 @@ paths: operationId: info.0 x-operation-group: info x-version-added: '1.0' + x-distributions-excluded: + - amazon-serverless description: Returns basic information about the cluster. externalDocs: url: https://opensearch.org/docs/latest diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index ff0efdc2..cab8696c 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -562,6 +562,14 @@ components: write_operations: description: The total number of write operations for the device completed since starting OpenSearch. type: number + read_time: + type: number + write_time: + type: number + queue_size: + type: number + io_time_in_millis: + $ref: '_common.yaml#/components/schemas/DurationValueUnitMillis' Jvm: type: object properties: @@ -1068,8 +1076,10 @@ components: type: boolean enforced: type: boolean + total_rejections_breakup: + $ref: '#/components/schemas/TotalRejectionsBreakup' total_rejections_breakup_shadow_mode: - $ref: '#/components/schemas/TotalRejectionsBreakupShadowMode' + $ref: '#/components/schemas/TotalRejectionsBreakup' ShardSearchBackpressureStats: type: object properties: @@ -1140,7 +1150,11 @@ components: type: number cancellation_limit_reached_count: type: number - TotalRejectionsBreakupShadowMode: + cancelled_task_percentage: + type: number + current_cancellation_eligible_tasks_count: + type: number + TotalRejectionsBreakup: type: object properties: node_limits: diff --git a/spec/schemas/nodes.info.yaml b/spec/schemas/nodes.info.yaml index 7086c17c..6fbcefdc 100644 --- a/spec/schemas/nodes.info.yaml +++ b/spec/schemas/nodes.info.yaml @@ -102,14 +102,10 @@ components: search_pipelines: $ref: '#/components/schemas/NodeInfoSearchPipelines' required: - - attributes - build_hash - build_type - - host - - ip - name - roles - - transport_address - version NodeInfoHttp: type: object @@ -156,7 +152,7 @@ components: bundled_jdk: type: boolean using_bundled_jdk: - type: boolean + type: [boolean, 'null'] using_compressed_ordinary_object_pointers: oneOf: - type: boolean @@ -167,16 +163,9 @@ components: type: string required: - bundled_jdk - - gc_collectors - - input_arguments - mem - - memory_pools - pid - start_time_in_millis - - version - - vm_name - - vm_vendor - - vm_version NodeInfoJvmMemory: type: object properties: @@ -256,12 +245,8 @@ components: swap: $ref: '#/components/schemas/NodeInfoMemory' required: - - arch - available_processors - - name - - pretty_name - refresh_interval_in_millis - - version NodeInfoOSCPU: type: object properties: diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index 67e24523..9cc644a6 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -2,6 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org chapters: - synopsis: Get server info. path: / diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml index 8e7a0435..32f9b3c2 100644 --- a/tests/default/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -124,4 +124,3 @@ chapters: title: Drive year: 2011 order: 2 - diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index 38865e9e..196cff4b 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -53,7 +53,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat with master response (format=json). method: GET @@ -66,7 +65,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_master: 'true' - synopsis: Cat with cluster_manager response (format=json). @@ -80,7 +78,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_cluster_manager: 'true' - synopsis: Cat in different formats (format=yaml). @@ -93,9 +90,10 @@ chapters: content_type: application/yaml payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -105,9 +103,10 @@ chapters: content_type: application/cbor payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -117,5 +116,4 @@ chapters: content_type: application/smile payload: - node.total: '1' - status: yellow node.data: '1' diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index c20df6fe..6b613082 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -71,6 +71,8 @@ chapters: status: 200 content_type: application/yaml - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: @@ -79,6 +81,8 @@ chapters: status: 200 content_type: application/cbor - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index cb9dc328..01dba1ca 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -3,6 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodeattrs endpoints. chapters: - synopsis: Cat with a json response. + distributions: + - opensearch.org path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml index db4db9dd..2e056a52 100644 --- a/tests/default/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml index 94087d49..0b1b0c78 100644 --- a/tests/default/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. +distributions: + - opensearch.org chapters: - synopsis: Get dangling indexes. path: /_dangling diff --git a/tests/default/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml index d537e3a7..cc1608cb 100644 --- a/tests/default/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -2,6 +2,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml index da87fec7..0445ebd6 100644 --- a/tests/default/indices/segments.yaml +++ b/tests/default/indices/segments.yaml @@ -1,7 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: This story tests the Segments API. - +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 5bfada4b..80906a7e 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -11,6 +11,8 @@ epilogues: status: [200, 404] chapters: - synopsis: Get global settings. + distributions: + - opensearch.org path: /_settings method: GET parameters: @@ -23,6 +25,8 @@ chapters: response: status: 200 - synopsis: Get global settings (cluster_manager_timeout). + distributions: + - opensearch.org path: /_settings method: GET version: '>= 2.0' diff --git a/tests/default/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml index a08efa0c..8c036ae6 100644 --- a/tests/default/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index 4bdac64e..f92ee82d 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/security/api/certificates.yaml b/tests/default/security/api/certificates.yaml index f4f23e81..340a659d 100644 --- a/tests/default/security/api/certificates.yaml +++ b/tests/default/security/api/certificates.yaml @@ -4,20 +4,18 @@ description: Test certificates endpoints. version: '> 2.14' # ADMIN-CERT only. These tests require explicit rest api admin privileges. -chapters: - - synopsis: Get node. - id: get_node - path: /_cat/nodes +prologues: + - path: /_cat/nodes + id: node method: GET parameters: - h: - - id full_id: true - response: - status: 200 - content_type: text/plain + size: 1 + format: json + h: id output: - node_id: payload + id: payload[0].id +chapters: - synopsis: Get all certificates. path: /_plugins/_security/api/certificates method: GET @@ -29,7 +27,7 @@ chapters: path: /_plugins/_security/api/certificates/{node_id} method: GET parameters: - node_id: ${get_node.node_id} + node_id: ${node.id} cert_type: all response: - status: 403 + status: 403 \ No newline at end of file diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index afbb7d5f..314a73d4 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -73,6 +73,7 @@ export interface OpenSearchHttpClientOptions { export type OpenSearchHttpClientCliOptions = { opensearchUrl?: string + opensearchDistribution?: string, opensearchUsername?: string opensearchPassword?: string opensearchInsecure?: boolean diff --git a/tools/src/_utils/semver.ts b/tools/src/_utils/semver.ts index 52de7e89..836c802e 100644 --- a/tools/src/_utils/semver.ts +++ b/tools/src/_utils/semver.ts @@ -9,7 +9,8 @@ import * as semver from 'semver' -export function coerce(version: string) : string { +export function coerce(version?: string): undefined | string { + if (version === undefined) return undefined return semver.coerce(version)?.toString() ?? version } diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts index 06672285..c463e49d 100644 --- a/tools/src/linter/SchemasValidator.ts +++ b/tools/src/linter/SchemasValidator.ts @@ -16,7 +16,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemasValidator { diff --git a/tools/src/linter/components/OperationGroup.ts b/tools/src/linter/components/OperationGroup.ts index dc45765b..f980e56b 100644 --- a/tools/src/linter/components/OperationGroup.ts +++ b/tools/src/linter/components/OperationGroup.ts @@ -13,7 +13,7 @@ import ValidatorBase from './base/ValidatorBase' export default class OperationGroup extends ValidatorBase { static readonly OP_PRIORITY = ['operationId', 'x-operation-group', 'x-ignorable', 'deprecated', - 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', + 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', 'x-distributions-included', 'x-distributions-excluded', 'description', 'externalDocs', 'parameters', 'requestBody', 'responses'] name: string diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 3f2fb396..a2ccbffa 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -17,12 +17,14 @@ import * as semver from '../_utils/semver' export default class OpenApiVersionExtractor { private _spec?: Record private _source_spec: OpenAPIV3.Document - private _target_version: string + private _target_version?: string + private _target_distribution?: string private _logger: Logger - constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { + constructor(source_spec: OpenAPIV3.Document, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this._source_spec = source_spec this._target_version = semver.coerce(target_version) + this._target_distribution = target_distribution this._logger = logger this._spec = undefined } @@ -45,16 +47,37 @@ export default class OpenApiVersionExtractor { #extract() : void { this._logger.info(`Extracting version ${this._target_version} ...`) this.#remove_keys_not_matching_semver() + this.#remove_keys_not_matching_distribution() this.#remove_unused() } #exclude_per_semver(obj: any): boolean { + if (this._target_version == undefined) return false + const x_version_added = semver.coerce(obj['x-version-added'] as string) const x_version_removed = semver.coerce(obj['x-version-removed'] as string) - if (x_version_added && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { + if (x_version_added !== undefined && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { return true - } else if (x_version_removed && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + } else if (x_version_removed !== undefined && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + return true + } + + return false + } + + #exclude_per_distribution(obj: any): boolean { + if (this._target_distribution == undefined) return false + + const x_distributions_included = obj['x-distributions-included'] as string[] + + if (x_distributions_included?.length > 0 && !x_distributions_included.includes(this._target_distribution)) { + return true + } + + const x_distributions_excluded = obj['x-distributions-excluded'] as string[] + + if (x_distributions_excluded?.length > 0 && x_distributions_excluded.includes(this._target_distribution)) { return true } @@ -63,9 +86,16 @@ export default class OpenApiVersionExtractor { // Remove any elements that are x-version-added/removed incompatible with the target server version. #remove_keys_not_matching_semver(): void { + if (this._target_version == undefined) return delete_matching_keys(this._spec, this.#exclude_per_semver.bind(this)) } + // Remove any elements that are x-distributions-included incompatible with the target distribution. + #remove_keys_not_matching_distribution(): void { + if (this._target_distribution === undefined) return + delete_matching_keys(this._spec, this.#exclude_per_distribution.bind(this)) + } + #remove_unused(): void { if (this._spec === undefined) return diff --git a/tools/src/tester/MergedOpenApiSpec.ts b/tools/src/tester/MergedOpenApiSpec.ts index ee21f55a..d4a3e260 100644 --- a/tools/src/tester/MergedOpenApiSpec.ts +++ b/tools/src/tester/MergedOpenApiSpec.ts @@ -20,21 +20,23 @@ export default class MergedOpenApiSpec { logger: Logger file_path: string target_version?: string + target_distribution?: string protected _spec: OpenAPIV3.Document | undefined - constructor (spec_path: string, target_version?: string, logger: Logger = new Logger()) { + constructor (spec_path: string, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this.logger = logger this.file_path = spec_path this.target_version = target_version + this.target_distribution = target_distribution } spec (): OpenAPIV3.Document { if (this._spec) return this._spec const merger = new OpenApiMerger(this.file_path, this.logger) var spec = merger.spec() - if (this.target_version !== undefined) { - const version_extractor = new OpenApiVersionExtractor(spec, this.target_version) + if (this.target_version !== undefined || this.target_distribution !== undefined) { + const version_extractor = new OpenApiVersionExtractor(spec, this.target_version, this.target_distribution) spec = version_extractor.extract() } const ctx = new SpecificationContext(this.file_path) diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts index 7f71e387..01b55b5a 100644 --- a/tools/src/tester/SchemaValidator.ts +++ b/tools/src/tester/SchemaValidator.ts @@ -19,7 +19,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemaValidator { diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 9020968f..3f396a3c 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -26,8 +26,8 @@ export default class StoryEvaluator { this._supplemental_chapter_evaluator = supplemental_chapter_evaluator } - async evaluate({ story, display_path, full_path }: StoryFile, version?: string, dry_run: boolean = false): Promise { - if (version != undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { + async evaluate({ story, display_path, full_path }: StoryFile, version?: string, distribution?: string, dry_run: boolean = false): Promise { + if (version !== undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { return { result: Result.SKIPPED, display_path, @@ -37,13 +37,23 @@ export default class StoryEvaluator { } } + if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.` + } + } + const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path) if (variables_error !== undefined) { return variables_error } const story_outputs = new StoryOutputs() const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) - const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version) + const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs) return { display_path, @@ -76,7 +86,7 @@ export default class StoryEvaluator { } } - async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string): Promise { + async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { if (dry_run) { @@ -85,6 +95,9 @@ export default class StoryEvaluator { } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) + } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) { + const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } }) } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 0b197d64..dc880f94 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -34,19 +34,25 @@ export default class TestRunner { this._result_logger = result_logger } - async run (story_path: string, version?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { + async run (story_path: string, version?: string, distribution?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { let failed = false const story_files = this.story_files(story_path) const results: StoryEvaluations = { evaluations: [] } if (!dry_run) { - const info = await this._http_client.wait_until_available() - console.log(`OpenSearch ${ansi.green(info.version.number)}\n`) - version = info.version.number + if (distribution === 'amazon-serverless') { + // TODO: Fetch OpenSearch version when Amazon Serverless OpenSearch supports multiple. + version = '2.1' + } else { + const info = await this._http_client.wait_until_available() + version = info.version.number + } + + console.log(`OpenSearch ${ansi.green(version)}\n`) } for (const story_file of story_files) { - const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, dry_run) + const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, distribution, dry_run) results.evaluations.push(evaluation) this._result_logger.log(evaluation) if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 134888d7..da025a34 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -47,6 +47,9 @@ const command = new Command() .addOption(new Option('--verbose', 'whether to print the full stack trace of errors').default(false)) .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests').default(false)) .addOption(new Option('--opensearch-version ', 'target OpenSearch schema version').default(undefined)) + .addOption(new Option('--opensearch-distribution ', 'OpenSearch distribution') + .default('opensearch.org') + .env('OPENSEARCH_DISTRIBUTION')) .addOption(OPENSEARCH_URL_OPTION) .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) @@ -63,7 +66,7 @@ const command = new Command() const opts = command.opts() const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) -const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Logger(LogLevel.error)) +const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, opts.opensearchDistribution, new Logger(LogLevel.error)) const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ responseType: 'arraybuffer', logger, ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) @@ -73,7 +76,7 @@ const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapt const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger) -runner.run(opts.testsPath, spec.api_version(), opts.dryRun) +runner.run(opts.testsPath, spec.api_version(), opts.opensearchDistribution, opts.dryRun) .then( ({ results, failed }) => { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index e418ff76..83d307b4 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -50,6 +50,14 @@ export type Payload = {} | any[] | string | number | boolean; * via the `definition` "Version". */ export type Version = string; +/** + * The list of distributions that support this API. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Distributions". + */ +export type Distributions = string[]; /** * Number of times to retry on error. * @@ -94,6 +102,7 @@ export interface Story { epilogues?: SupplementalChapter[]; chapters: Chapter[]; version?: Version; + distributions?: Distributions; } /** * This interface was referenced by `Story`'s JSON-Schema @@ -112,6 +121,7 @@ export interface ChapterRequest { request?: Request; output?: Output; version?: Version; + distributions?: Distributions; retry?: Retry; } /** @@ -131,7 +141,7 @@ export interface Request { * The values are paths to the values in the response. * The values should be in the form: * - `payload.` for the payload - * - `headers.` for the headers + * - `headers.` for the headers. * * * This interface was referenced by `Story`'s JSON-Schema @@ -157,6 +167,9 @@ export interface ExpectedResponse { * via the `definition` "Warnings". */ export interface Warnings { + /** + * Enable/disable warnings about multiple paths being tested in the same story. + */ 'multiple-paths-detected'?: boolean; } /** diff --git a/tools/src/types.ts b/tools/src/types.ts index a9497dbc..b6871d02 100644 --- a/tools/src/types.ts +++ b/tools/src/types.ts @@ -16,6 +16,8 @@ export interface OperationSpec extends OpenAPIV3.OperationObject { 'x-version-deprecated'?: string 'x-deprecation-message'?: string 'x-ignorable'?: boolean + 'x-distributions-included'?: string[] + 'x-distributions-excluded'?: string[] parameters?: OpenAPIV3.ReferenceObject[] requestBody?: OpenAPIV3.ReferenceObject diff --git a/tools/tests/merger/OpenApiVersionExtractor.test.ts b/tools/tests/merger/OpenApiVersionExtractor.test.ts index 61bc4262..ca3e14eb 100644 --- a/tools/tests/merger/OpenApiVersionExtractor.test.ts +++ b/tools/tests/merger/OpenApiVersionExtractor.test.ts @@ -17,7 +17,7 @@ describe('extract() from a merged API spec', () => { const merger = new OpenApiMerger('tools/tests/tester/fixtures/specs/complete') describe('1.3', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3') + const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3', 'ignore') describe('write_to', () => { var temp: tmp.DirResult @@ -43,18 +43,18 @@ describe('extract() from a merged API spec', () => { test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0', 'ignore') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', 'distributed-excluded-amazon-serverless' ]) }) @@ -81,12 +81,12 @@ describe('extract() from a merged API spec', () => { }) describe('2.1', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1', 'oracle-managed') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml index d0fa222f..a4497ccb 100644 --- a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml @@ -35,6 +35,10 @@ paths: x-version-removed: '2.0' added-1.3-removed-2.0: $ref: '#/components/responses/info@added-1.3-removed-2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -106,6 +110,8 @@ components: description: Added in 1.3, removed in 2.0 via attribute in response body. x-version-added: '1.3' x-version-removed: '2.0' + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@removed-2.0: description: Removed in 2.0 via attribute next to ref. nodes.info@200: diff --git a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml index cfb4279d..c684b7af 100644 --- a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml @@ -57,6 +57,10 @@ paths: added-2.0: $ref: '#/components/responses/info@added-2.0' x-version-added: '2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -144,6 +148,8 @@ components: type: object info@added-2.0: description: Added in 2.0 via attribute next to ref. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. nodes.info@200: description: All nodes. content: diff --git a/tools/tests/tester/MergedOpenApiSpec.test.ts b/tools/tests/tester/MergedOpenApiSpec.test.ts index 7e6c6464..db852a03 100644 --- a/tools/tests/tester/MergedOpenApiSpec.test.ts +++ b/tools/tests/tester/MergedOpenApiSpec.test.ts @@ -13,7 +13,7 @@ import MergedOpenApiSpec from "tester/MergedOpenApiSpec" describe('merged API spec', () => { describe('defaults', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, undefined, new Logger()) test('has an api version', () => { expect(spec.api_version()).toEqual('1.2.3') @@ -30,7 +30,8 @@ describe('merged API spec', () => { test('has all responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1' + '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) @@ -65,31 +66,67 @@ describe('merged API spec', () => { }) describe('1.3', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, 'oracle-managed', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', undefined, new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('2.0 amazon-serverless', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'amazon-serverless', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all' + ]) + }) + }) + + describe('2.0 oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'oracle-managed', new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.1', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml new file mode 100644 index 00000000..cc4ef9a1 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions.yaml + +result: SKIPPED +description: This story should be skipped because of distributions. +message: Skipped because distribution opensearch.org is not one of another, some. diff --git a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml index 68337dc6..0c0a0923 100644 --- a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml +++ b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml @@ -28,6 +28,20 @@ paths: $ref: '#/components/responses/info@500' '503': $ref: '#/components/responses/info@503' + distributed-included-all: + $ref: '#/components/responses/info@distributed-all' + x-distributions-included: + - amazon-managed + - amazon-serverless + - opensearch.org + distributed-included-amazon-managed: + $ref: '#/components/responses/info@distributed-amazon-managed' + x-distributions-included: + - amazon-managed + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless components: responses: info@200: @@ -76,6 +90,10 @@ components: info@added-2.1: description: Added in 2.1 via attribute in response body. x-version-added: '2.1' + info@distributed-amazon-managed: + description: Distributed only in AOS. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@500: content: application/json: diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml new file mode 100644 index 00000000..8efebbb2 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml @@ -0,0 +1,9 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of distributions. +distributions: + - another + - some +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index ce11b6ad..d6d7e563 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -141,5 +141,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.16.0')) + }, process.env.OPENSEARCH_VERSION ?? '2.16.0', process.env.OPENSEARCH_DISTRIBUTION ?? 'opensearch.org')) } diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index b0333504..0259c29a 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -16,7 +16,7 @@ test('stories folder', async () => { const info = await opensearch_http_client.wait_until_available() expect(info.version).toBeDefined() - const run = await test_runner.run('tools/tests/tester/fixtures/stories') + const run = await test_runner.run('tools/tests/tester/fixtures/stories', undefined, 'opensearch.org') expect(run.failed).toBeTruthy() @@ -29,14 +29,15 @@ test('stories folder', async () => { } const passed = load_expected_evaluation('passed', true) - const skipped = load_expected_evaluation('skipped/semver', true) + const skipped_semver = load_expected_evaluation('skipped/semver', true) + const skipped_distributions = load_expected_evaluation('skipped/distributions', true) const not_found = load_expected_evaluation('failed/not_found', true) const invalid_data = load_expected_evaluation('failed/invalid_data', true) const chapter_error = load_expected_evaluation('error/chapter_error', true) const output_error = load_expected_evaluation('error/output_error', true) const prologue_error = load_expected_evaluation('error/prologue_error', true) - const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped] + const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped_distributions, skipped_semver] expect(actual_evaluations).toEqual(expected_evaluations) })