Added x-distributions-included and excluded.

Signed-off-by: dblock <[email protected]>

DEVELOPER_GUIDE.md

@@ -146,7 +146,10 @@ This repository includes several OpenAPI Specification Extensions to fill in any
 - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them.
 - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml).
 - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema.
-- `x-distributions`: Contains a list of distributions known to include the API. Use `opensearch.org` for the official distribution, `aos` for Amazon Managed OpenSearch, and `aoss` for Amazon OpenSearch Serverless. 
+- `x-distributions-included`: Contains a list of distributions known to include the API.
+- `x-distributions-excluded`: Contains a list of distributions known to exclude the API.
+
+Use `opensearch.org` for the official distribution in `x-distributions-*`, `amazon-opensearch` for Amazon Managed OpenSearch, and `amazon-serverless` for Amazon OpenSearch Serverless. 
 
 ## Writing Spec Tests
 

TESTING_GUIDE.md

@@ -210,15 +210,17 @@ The test tool will fetch the server version when it starts and use it automatica
 
 ### Managing Distributions
 
-OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not.
+OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions-included` or `x-distributions-excluded` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not.
 
 ```yaml
 /:
   get:
     operationId: info.0
-    x-distributions:
+    x-distributions-included:
       - opensearch.org
-      - aos
+      - amazon-managed
+    x-distributions-excluded:
+      - amazon-serverless
     description: Returns basic information about the cluster.
 ```
 
@@ -227,8 +229,8 @@ Similarly, skip tests that are not applicable to a distribution by listing the d
 ```yaml
 description: Test root endpoint.
 distributions:
+  - amazon-managed
   - opensearch.org
-  - aos
 chapters:
   - synopsis: Get server info.
     path: /
@@ -237,7 +239,26 @@ chapters:
       status: 200
 ```
 
-To test a particular distribution pass `--opensearch-distribution` to the test tool.
+To test a particular distribution pass `--opensearch-distribution` to the test tool. For example, the following runs tests against an Amazon Managed OpenSearch instance.
+
+```bash
+export AWS_ACCESS_KEY_ID=...
+export AWS_SECRET_ACCESS_KEY=...
+export AWS_SESSION_TOKEN=...
+export AWS_REGION=us-west-2
+
+export OPENSEARCH_URL=https://....us-west-2.es.amazonaws.com
+
+npm run test:spec -- --opensearch-distribution=amazon-managed
+```
+
+The output will visible skip APIs that are not available in the `amazon-managed` distribution.
+
+```
+PASSED  _core/bulk.yaml (.../_core/bulk.yaml)
+PASSED  _core/info.yaml (.../_core/info.yaml)
+SKIPPED indices/forcemerge.yaml (Skipped because distribution amazon-managed is not opensearch.org.)
+```
 
 ### Waiting for Tasks
 

spec/namespaces/_core.yaml

@@ -9,9 +9,8 @@ paths:
       operationId: info.0
       x-operation-group: info
       x-version-added: '1.0'
-      x-distributions:
-        - aos
-        - opensearch.org
+      x-distributions-excluded:
+        - amazon-serverless
       description: Returns basic information about the cluster.
       externalDocs:
         url: https://opensearch.org/docs/latest

tests/default/_core/info.yaml

@@ -3,7 +3,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test root endpoint.
 distributions:
-  - aos
+  - amazon-managed
   - opensearch.org
 chapters:
   - synopsis: Get server info.

tools/src/OpenSearchHttpClient.ts

@@ -23,6 +23,7 @@ export const OPENSEARCH_URL_OPTION = new Option('--opensearch-url <url>', 'URL a
   .env('OPENSEARCH_URL')
 
 export const OPENSEARCH_DISTRIBUTION_OPTION = new Option('--opensearch-distribution <key>', 'OpenSearch distribution')
+  .default('opensearch.org')
   .env('OPENSEARCH_DISTRIBUTION')
 
 export const OPENSEARCH_USERNAME_OPTION = new Option('--opensearch-username <username>', 'username to use when authenticating with OpenSearch')

tools/src/linter/SchemasValidator.ts

@@ -16,7 +16,9 @@ const ADDITIONAL_KEYWORDS = [
   'x-version-added',
   'x-version-deprecated',
   'x-version-removed',
-  'x-deprecation-message'
+  'x-deprecation-message',
+  'x-distributions-included',
+  'x-distributions-excluded'
 ]
 
 export default class SchemasValidator {

tools/src/linter/components/OperationGroup.ts

@@ -13,7 +13,7 @@ import ValidatorBase from './base/ValidatorBase'
 
 export default class OperationGroup extends ValidatorBase {
   static readonly OP_PRIORITY = ['operationId', 'x-operation-group', 'x-ignorable', 'deprecated',
-    'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed',
+    'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', 'x-distributions-included', 'x-distributions-excluded',
     'description', 'externalDocs', 'parameters', 'requestBody', 'responses']
 
   name: string

tools/src/merger/OpenApiVersionExtractor.ts

@@ -69,9 +69,15 @@ export default class OpenApiVersionExtractor {
   #exclude_per_distribution(obj: any): boolean {
     if (this._target_distribution == undefined) return false
 
-    const x_distributions = obj['x-distributions'] as string[]
+    const x_distributions_included = obj['x-distributions-included'] as string[]
 
-    if (x_distributions?.length > 0 && !x_distributions.includes(this._target_distribution)) {
+    if (x_distributions_included?.length > 0 && !x_distributions_included.includes(this._target_distribution)) {
+      return true
+    }
+
+    const x_distributions_excluded = obj['x-distributions-excluded'] as string[]
+
+    if (x_distributions_excluded?.length > 0 && x_distributions_excluded.includes(this._target_distribution)) {
       return true
     }
 
@@ -84,7 +90,7 @@ export default class OpenApiVersionExtractor {
     delete_matching_keys(this._spec, this.#exclude_per_semver.bind(this))
   }
 
-  // Remove any elements that are x-distributions incompatible with the target distribution.
+  // Remove any elements that are x-distributions-included incompatible with the target distribution.
   #remove_keys_not_matching_distribution(): void {
     if (this._target_distribution === undefined) return
     delete_matching_keys(this._spec, this.#exclude_per_distribution.bind(this))

tools/src/tester/SchemaValidator.ts

@@ -19,7 +19,9 @@ const ADDITIONAL_KEYWORDS = [
   'x-version-added',
   'x-version-deprecated',
   'x-version-removed',
-  'x-deprecation-message'
+  'x-deprecation-message',
+  'x-distributions-included',
+  'x-distributions-excluded'
 ]
 
 export default class SchemaValidator {

tools/src/tester/StoryEvaluator.ts

@@ -14,8 +14,8 @@ import { overall_result } from './helpers'
 import { StoryOutputs } from './StoryOutputs'
 import SupplementalChapterEvaluator from './SupplementalChapterEvaluator'
 import { ChapterOutput } from './ChapterOutput'
-import * as semver from 'semver'
 import _ from 'lodash'
+import semver from 'semver'
 
 export default class StoryEvaluator {
   private readonly _chapter_evaluator: ChapterEvaluator
@@ -27,7 +27,7 @@ export default class StoryEvaluator {
   }
 
   async evaluate({ story, display_path, full_path }: StoryFile, version?: string, distribution?: string, dry_run: boolean = false): Promise<StoryEvaluation> {
-    if (version != undefined && story.version !== undefined && !semver.satisfies(version, story.version)) {
+    if (version !== undefined && story.version !== undefined && !StoryEvaluator.#semver_satisfies(version, story.version)) {
       return {
         result: Result.SKIPPED,
         display_path,
@@ -43,7 +43,7 @@ export default class StoryEvaluator {
         display_path,
         full_path,
         description: story.description,
-        message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.sort().join(', ')}.`
+        message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.`
       }
     }
 
@@ -92,12 +92,12 @@ export default class StoryEvaluator {
       if (dry_run) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } })
-      } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) {
+      } else if (version != undefined && chapter.version !== undefined && !StoryEvaluator.#semver_satisfies(version, chapter.version)) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } })
       } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
-        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.sort().join(', ')}.`, error: undefined } })
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } })
       } else {
         const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs)
         has_errors = has_errors || evaluation.overall.result === Result.ERROR
@@ -236,4 +236,8 @@ export default class StoryEvaluator {
   static #failed_evaluation(title: string, message: string): ChapterEvaluation {
     return { title, overall: { result: Result.FAILED, message } }
   }
+
+  static #semver_satisfies(version: string, range: string): boolean {
+    return _.every(range.split(','), (portion) => semver.satisfies(version, portion))
+  }
 }

tools/src/types.ts

@@ -16,6 +16,8 @@ export interface OperationSpec extends OpenAPIV3.OperationObject {
   'x-version-deprecated'?: string
   'x-deprecation-message'?: string
   'x-ignorable'?: boolean
+  'x-distributions-included'?: string
+  'x-distributions-excluded'?: string
 
   parameters?: OpenAPIV3.ReferenceObject[]
   requestBody?: OpenAPIV3.ReferenceObject

tools/tests/merger/OpenApiVersionExtractor.test.ts

@@ -43,7 +43,7 @@ describe('extract() from a merged API spec', () => {
     test('has matching responses', () => {
       const spec = extractor.extract()
       expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0'
+        '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-excluded-amazon-serverless'
       ])
     })
   })
@@ -54,7 +54,7 @@ describe('extract() from a merged API spec', () => {
     test('has matching responses', () => {
       const spec = extractor.extract()
       expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'added-2.0'
+        '200', '201', '404', '500', '503', 'added-2.0', 'distributed-excluded-amazon-serverless'
       ])
     })
 
@@ -81,12 +81,12 @@ describe('extract() from a merged API spec', () => {
   })
 
   describe('2.1', () => {
-    const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1', 'ignore')
+    const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1', 'oracle-managed')
 
     test('has matching responses', () => {
       const spec = extractor.extract()
       expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1'
+        '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-excluded-amazon-serverless'
       ])
     })
   })

tools/tests/merger/fixtures/extractor/expected_1.3.yaml

@@ -35,6 +35,10 @@ paths:
           x-version-removed: '2.0'
         added-1.3-removed-2.0:
           $ref: '#/components/responses/[email protected]'
+        distributed-excluded-amazon-serverless:
+          $ref: '#/components/responses/info@distributed-all'
+          x-distributions-excluded:
+            - amazon-serverless
       parameters: []
   /nodes:
     get:
@@ -106,6 +110,8 @@ components:
       description: Added in 1.3, removed in 2.0 via attribute in response body.
       x-version-added: '1.3'
       x-version-removed: '2.0'
+    info@distributed-all:
+      description: Distributed in opensearch.org, AOS and AOSS.
     [email protected]:
       description: Removed in 2.0 via attribute next to ref.
     nodes.info@200:

tools/tests/merger/fixtures/extractor/expected_2.0.yaml

@@ -57,6 +57,10 @@ paths:
         added-2.0:
           $ref: '#/components/responses/[email protected]'
           x-version-added: '2.0'
+        distributed-excluded-amazon-serverless:
+          $ref: '#/components/responses/info@distributed-all'
+          x-distributions-excluded:
+            - amazon-serverless
       parameters: []
   /nodes:
     get:
@@ -144,6 +148,8 @@ components:
             type: object
     [email protected]:
       description: Added in 2.0 via attribute next to ref.
+    info@distributed-all:
+      description: Distributed in opensearch.org, AOS and AOSS.
     nodes.info@200:
       description: All nodes.
       content:

tools/tests/tester/MergedOpenApiSpec.test.ts

@@ -30,7 +30,8 @@ describe('merged API spec', () => {
 
     test('has all responses', () => {
       expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', 'distributed-aos', 'distributed-all'
+        '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1',
+        'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless'
       ])
     })
 
@@ -69,17 +70,19 @@ describe('merged API spec', () => {
 
     test('has matching responses', () => {
       expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-aos', 'distributed-all'
+        '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0',
+        'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless'
       ])
     })
   })
 
-  describe('another', () => {
-    const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, 'another', new Logger())
+  describe('oracle-managed', () => {
+    const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, 'oracle-managed', new Logger())
 
     test('has matching responses', () => {
       expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', 'distributed-all'
+        '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1',
+        'distributed-excluded-amazon-serverless'
       ])
     })
   })
@@ -89,27 +92,30 @@ describe('merged API spec', () => {
 
     test('has matching responses', () => {
       expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'added-2.0', 'distributed-aos', 'distributed-all'
+        '200', '201', '404', '500', '503', 'added-2.0',
+        'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless'
       ])
     })
   })
 
-  describe('2.0 aos', () => {
-    const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'aos', new Logger())
+  describe('2.0 amazon-serverless', () => {
+    const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'amazon-serverless', new Logger())
 
     test('has matching responses', () => {
       expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'added-2.0', 'distributed-aos'
+        '200', '201', '404', '500', '503', 'added-2.0',
+        'distributed-included-all'
       ])
     })
   })
 
-  describe('2.0 another', () => {
-    const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'another', new Logger())
+  describe('2.0 oracle-managed', () => {
+    const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'oracle-managed', new Logger())
 
     test('has matching responses', () => {
       expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'added-2.0', 'distributed-all'
+        '200', '201', '404', '500', '503', 'added-2.0',
+        'distributed-excluded-amazon-serverless'
       ])
     })
   })
@@ -119,7 +125,8 @@ describe('merged API spec', () => {
 
     test('has matching responses', () => {
       expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([
-        '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-aos', 'distributed-all'
+        '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1',
+        'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless'
       ])
     })
   })

tools/tests/tester/fixtures/evals/passed.yaml

@@ -176,6 +176,25 @@ chapters:
     overall:
       result: SKIPPED
       message: Skipped because version 2.16.0 does not satisfy >= 2.999.0.
+  - title: This GET /_cat/health should not be skipped (> 2.0, < 10).
+    overall:
+      result: PASSED
+    path: GET /_cat/health
+    request:
+      parameters:
+        format:
+          result: PASSED
+      request:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload_body:
+        result: PASSED
+      payload_schema:
+        result: PASSED
+      output_values:
+        result: SKIPPED
 epilogues:
   - title: DELETE /books
     overall: