Skip to content

Commit

Permalink
Added CBOR support. (#371)
Browse files Browse the repository at this point in the history
Signed-off-by: dblock <[email protected]>
  • Loading branch information
dblock authored Jul 2, 2024
1 parent d25459f commit 176310c
Show file tree
Hide file tree
Showing 16 changed files with 549 additions and 174 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added support for `application/yaml` responses ([#363](https://github.com/opensearch-project/opensearch-api-specification/pull/363))
- Added test for search with seq_no_primary_term ([#367](https://github.com/opensearch-project/opensearch-api-specification/pull/367))
- Added a linter for parameter sorting ([#369](https://github.com/opensearch-project/opensearch-api-specification/pull/369))

- Added support for `application/cbor` responses ([#371](https://github.com/opensearch-project/opensearch-api-specification/pull/371))

### Changed

- Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189))
Expand Down
253 changes: 233 additions & 20 deletions package-lock.json

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,12 @@
"@eslint/eslintrc": "^3.0.2",
"@eslint/js": "^9.1.1",
"@stylistic/eslint-plugin": "^2.3.0",
"@types/cbor-js": "^0.1.1",
"@types/jest": "^29.5.12",
"@types/qs": "^6.9.15",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"ajv-errors": "^3.0.0",
"cbor": "^9.0.2",
"eslint": "^8.57.0",
"eslint-config-standard-with-typescript": "^43.0.1",
"eslint-plugin-eslint-comments": "^3.2.0",
Expand All @@ -50,7 +52,6 @@
"eslint-plugin-promise": "^6.1.1",
"eslint-plugin-yml": "^1.14.0",
"globals": "^15.0.0",
"jest": "^29.7.0",
"json-diff-ts": "^4.0.1",
"json-schema-to-typescript": "^14.0.4",
"qs": "^6.12.1",
Expand Down
10 changes: 10 additions & 0 deletions spec/namespaces/cat.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -853,6 +853,11 @@ components:
type: array
items:
$ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord'
application/cbor:
schema:
type: array
items:
$ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord'
cat.help@200:
description: ''
content:
Expand All @@ -874,6 +879,11 @@ components:
type: array
items:
$ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord'
application/cbor:
schema:
type: array
items:
$ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord'
cat.master@200:
description: ''
content:
Expand Down
13 changes: 13 additions & 0 deletions tests/cat/health.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,3 +69,16 @@ chapters:
status: yellow
node.data: '1'
discovered_cluster_manager: 'true'
- synopsis: Cat in different formats (format=cbor).
method: GET
path: /_cat/health
parameters:
format: cbor
response:
status: 200
content_type: application/yaml
payload:
- node.total: '1'
status: yellow
node.data: '1'
discovered_cluster_manager: 'true'
8 changes: 8 additions & 0 deletions tests/cat/indices.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,11 @@ chapters:
response:
status: 200
content_type: application/yaml
- synopsis: Cat in different formats (format=cbor).
method: GET
path: /_cat/indices
parameters:
format: cbor
response:
status: 200
content_type: application/cbor
11 changes: 7 additions & 4 deletions tools/src/OpenSearchHttpClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
*/

import { Option } from '@commander-js/extra-typings'
import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from 'axios'
import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse, type ResponseType } from 'axios'
import * as https from 'node:https'
import { sleep } from './helpers'

Expand All @@ -34,7 +34,8 @@ export interface OpenSearchHttpClientOptions {
url?: string
username?: string
password?: string
insecure?: boolean
insecure?: boolean,
responseType: ResponseType | undefined
}

export type OpenSearchHttpClientCliOptions = { [K in keyof OpenSearchHttpClientOptions as `opensearch${Capitalize<K>}`]: OpenSearchHttpClientOptions[K] }
Expand All @@ -44,7 +45,8 @@ export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptio
url: opts.opensearchUrl,
username: opts.opensearchUsername,
password: opts.opensearchPassword,
insecure: opts.opensearchInsecure
insecure: opts.opensearchInsecure,
responseType: opts.opensearchResponseType
}
}

Expand Down Expand Up @@ -79,7 +81,8 @@ export class OpenSearchHttpClient {
password: opts.password
}
: undefined,
httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) })
httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }),
responseType: opts?.responseType,
})
}

Expand Down
2 changes: 1 addition & 1 deletion tools/src/dump-cluster-spec/dump-cluster-spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ const command = new Command()

const opts = command.opts()

main({ output: opts.output, opensearch: get_opensearch_opts_from_cli(opts) })
main({ output: opts.output, opensearch: get_opensearch_opts_from_cli({ opensearchResponseType: undefined, ...opts }) })
.catch(e => {
if (e instanceof Error) {
console.error(`ERROR: ${e.stack}`)
Expand Down
27 changes: 12 additions & 15 deletions tools/src/tester/ChapterEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,21 @@ import type SchemaValidator from './SchemaValidator'
import { type StoryOutputs } from './StoryOutputs'
import { ChapterOutput } from './ChapterOutput'
import { Operation, atomizeChangeset, diff } from 'json-diff-ts'
import YAML from 'yaml'
import _ from 'lodash'
import { Logger } from 'Logger'
import { to_json } from '../helpers'

export default class ChapterEvaluator {
private readonly logger: Logger
private readonly _operation_locator: OperationLocator
private readonly _chapter_reader: ChapterReader
private readonly _schema_validator: SchemaValidator

constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator) {
constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator, logger: Logger) {
this._operation_locator = spec_parser
this._chapter_reader = chapter_reader
this._schema_validator = schema_validator
this.logger = logger
}

async evaluate(chapter: Chapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
Expand Down Expand Up @@ -86,8 +89,8 @@ export default class ChapterEvaluator {

#evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation {
if (expected_payload == null) return { result: Result.PASSED }
const content_type = response.content_type ?? 'application/json'
const payload = this.#deserialize_payload(response.payload, content_type)
const payload = response.payload
this.logger.info(`${to_json(payload)}`)
const delta = atomizeChangeset(diff(expected_payload, payload))
const messages: string[] = _.compact(delta.map((value, _index, _array) => {
switch (value.type) {
Expand All @@ -102,18 +105,12 @@ export default class ChapterEvaluator {

#evaluate_payload_schema(response: ActualResponse, operation: ParsedOperation): Evaluation {
const content_type = response.content_type ?? 'application/json'
const content = operation.responses[response.status]?.content[content_type]
const schema = content?.schema
const content = operation.responses[response.status]?.content
// eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
const content_type_content = content ? content[content_type] : undefined
const schema = content_type_content?.schema
if (schema == null && content != null) return { result: Result.PASSED }
if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` }
return this._schema_validator.validate(schema, this.#deserialize_payload(response.payload, content_type))
}

#deserialize_payload(payload: any, content_type: string): any {
if (payload === undefined) return undefined
switch (content_type) {
case 'application/yaml': return YAML.parse(payload as string)
default: return payload
}
return this._schema_validator.validate(schema, response.payload)
}
}
33 changes: 26 additions & 7 deletions tools/src/tester/ChapterReader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import { type StoryOutputs } from './StoryOutputs'
import { Logger } from 'Logger'
import { to_json, to_ndjson } from '../helpers'
import qs from 'qs'
import YAML from 'yaml'
import CBOR from 'cbor'

export default class ChapterReader {
private readonly _client: OpenSearchHttpClient
Expand Down Expand Up @@ -43,22 +45,23 @@ export default class ChapterReader {
return qs.stringify(params, { arrayFormat: 'comma' })
}
}).then(r => {
this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(r.data)}`)
response.status = r.status
response.content_type = r.headers['content-type'].split(';')[0]
response.payload = r.data
response.content_type = r.headers['content-type']?.split(';')[0]
response.payload = this.#deserialize_payload(r.data, response.content_type)
this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(response.payload)}`)
}).catch(e => {
if (e.response == null) {
this.logger.info(`<= ERROR: ${e}`)
throw e
}
response.status = e.response.status
response.content_type = e.response.headers['content-type'].split(';')[0]
response.payload = e.response.data?.error
response.message = e.response.data?.error?.reason ?? e.response.statusText
response.content_type = e.response.headers['content-type']?.split(';')[0]
const payload = this.#deserialize_payload(e.response.data, response.content_type)
response.payload = payload?.error
response.message = payload.error?.reason ?? e.response.statusText
response.error = e

this.logger.info(`<= ${response.status} (${response.content_type}) | ${to_json(response.payload ?? response.message)}`)
this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload ?? response.message}`)
})
return response as ActualResponse
}
Expand All @@ -80,4 +83,20 @@ export default class ChapterReader {
const query_params = Object.fromEntries(Object.entries(parameters).filter(([key]) => !path_params.has(key)))
return [parsed_path, query_params]
}

#deserialize_payload(payload: any, content_type: any): any {
if (payload === undefined) return undefined
if (content_type === undefined) return payload
switch (content_type) {
case 'text/plain': return this.#deserialize_payload_data(payload as string)
case 'application/json': return payload.length == 0 ? {} : JSON.parse(this.#deserialize_payload_data(payload as string))
case 'application/yaml': return payload.length == 0 ? {} : YAML.parse(this.#deserialize_payload_data(payload as string))
case 'application/cbor': return payload.length == 0 ? {} : CBOR.decode(payload as string)
default: return this.#deserialize_payload_data(payload as string)
}
}

#deserialize_payload_data(payload: string): string {
return Buffer.from(payload, 'binary').toString()
}
}
4 changes: 2 additions & 2 deletions tools/src/tester/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ const opts = command.opts()
const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn)

const spec = (new MergedOpenApiSpec(opts.specPath, new Logger(LogLevel.error))).spec()
const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli(opts))
const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts }))
const chapter_reader = new ChapterReader(http_client, logger)
const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger))
const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger), logger)
const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
const story_validator = new StoryValidator()
const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)
Expand Down
Loading

0 comments on commit 176310c

Please sign in to comment.