From a5459bf02eaccf1d1789b522c3fa1deac3457248 Mon Sep 17 00:00:00 2001 From: Brett Uglow Date: Tue, 6 Oct 2020 13:16:49 +1100 Subject: [PATCH] feat(security): add support for OAS securitySchemes (http & apiKey) --- README.md | 87 +++++-- fixtures/invalidSpecV2.json | 3 + fixtures/output/lint2.json | 3 - fixtures/specWithValidExamples.json | 183 ++++++++++++++ jest.config.js | 8 +- package-lock.json | 8 +- package.json | 2 +- src/cli.js | 53 +++- src/e2e.spec.js | 28 ++- src/lint.js | 40 +++- src/lint.spec.js | 71 +++++- src/utils.js | 171 +++++++++++-- src/utils.spec.js | 358 ++++++++++++++++++++++++++++ src/validate.js | 16 ++ 14 files changed, 954 insertions(+), 77 deletions(-) create mode 100644 fixtures/specWithValidExamples.json create mode 100644 src/validate.js diff --git a/README.md b/README.md index 444f7e3..17d2ff2 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - [Linting](#linting) - [Building](#building) - [Comparing](#comparing) +- [Validating](#validating) - Deploying (Coming soon) - [Config file](#config-file) @@ -32,11 +33,12 @@ oaat --help ## Usage -This tool does 4 things: +This tool does 5 things: - `oaat record` records API responses to requests specified in `x-examples` fields in an OpenAPI 3.x spec file. - `oaat lint` lints an OpenAPI 3.x spec file (basic formatting; tools like [speccy](https://www.npmjs.com/package/speccy) provide more capability, but don't do formatting). -- `oaat compare` compares the earlier-recorded responses to the last responses for endpoints in an OpenAPI 3.x spec file. - `oaat build` creates an OpenAPI 3.x spec file with [API Gateway][api-gateway-url] headers, optionally with mock responses for the APIs. +- `oaat compare` compares the earlier-recorded responses to the last responses for endpoints in an OpenAPI 3.x spec file. +- `oaat validate` validates that a spec-file is compliant with the OpenAPI 3.x specification. - (Coming soon) `oaat deploy` deploys an OpenAPI 3.x spec file that has the API Gateway headers to API Gateway. ## Recording @@ -54,12 +56,13 @@ Usage: oaat record [options] [serverUrl] Record the responses of API spec file endpoint requests (optionally use a different server to make requests) Options: - -o, --output Output file (if different to jsonFile) - -c, --config Config file to override default config - -q, --quiet No logging - -v, --verbose Verbose logging - -d, --dry-run Dry run (no changes made) - -h, --help display help for command + -o, --output Output file (if different to jsonFile) + -c, --config Config file to override default config + -s, --sec-tokens Pass security token(s) matching the "key" in spec.securitySchemes, with a "value" + -q, --quiet No logging + -v, --verbose Verbose logging + -d, --dry-run Dry run (no changes made) + -h, --help display help for command ``` To get valid example responses - to use for mocking & as documentation - we need to add some custom @@ -190,6 +193,15 @@ The above JavaScript module exports an async function which returns an object as The `queueWrapper()` function is there to combine multiple requests into a single request, in scenarios where multiple endpoint-examples require the same async-value. +### Security tokens and headers + +APIs are often protected with security tokens and headers. To call these APIs, the security token can +be passed to the command line via `-s securityHeaderType=123someValue,nextToken=nextValue,...`. Alternatively, +the security token values can be specified in the `securitySchemes` section of [configuration file](config-file). +By default, `securitySchemes` contains no keys. + +`oaat` will **always** pick the first security scheme when there are multiple security schemes available for an endpoint. + ### Disabling endpoint recording Sometimes it may be necessary to disable the recording of certain endpoints, while keeping the endpoint in the spec. @@ -249,18 +261,21 @@ There are lots of good tools that can check the syntax and style of Open API Sep ```shell script $ oaat lint --help -Usage: oaat lint [options] +Usage: oaat lint [options] [serverUrl] Tidy the API Spec up a bit Options: - -o, --output Output file (if different to jsonFile) - -c, --config Config file to override default config - -q, --quiet no logging - -v, --verbose verbose logging - -h, --help display help for command + -o, --output Output file (if different to jsonFile) + -c, --config Config file to override default config + -s, --sec-tokens Pass security token(s) matching the "key" in spec.securitySchemes, with a "value" + -q, --quiet no logging + -v, --verbose verbose logging + -h, --help display help for command ``` +Note: `serverUrl` and `sec-tokens` are only required when the `config.lint.syncExamples` is `true`. + See the [configuration file](config-file) for further options. ## Building @@ -270,7 +285,7 @@ Creates an OpenAPI 3.x spec file with [API Gateway][api-gateway-url] headers, op ### Command ```shell script -$ oaat lint --help +$ oaat build --help Usage: oaat build [options] [serverUrl] Adds custom headers & Swagger UI endpoint to allow deployment of spec file to AWS API Gateway with documentation @@ -339,7 +354,6 @@ by specifying multiple API paths as property keys (see example below). } ``` - ## Comparing Compares the earlier-recorded responses to the last responses for endpoints in an OpenAPI 3.x spec file. @@ -349,17 +363,35 @@ snapshots, and comparing those to the latest responses. ### Command ```shell script -$ oaat lint --help -Usage: oaat compare [options] [serverUrl] +Usage: oaat compare [options] [serverUrl] -Adds custom headers & Swagger UI endpoint to allow deployment of spec file to AWS API Gateway with documentation +Compares recorded responses (referenced by the spec file) to the latest responses + +Options: + -c, --config Config file to override default config + -s, --sec-tokens Pass security token(s) matching the "key" in spec.securitySchemes, with a "value" + -m --compare-mode Comparison mode: "value" (default), "type", "schema" + -q, --quiet no logging + -v, --verbose verbose logging + -h, --help display help for command +``` + +## Validating + +This command validates the JSON spec file against the Open API 3.x schema. +Any errors are listed in the output. + +### Command + +```shell script +$ oaat validate --help +Usage: oaat validate [options] + +Validate the API spec file against the OAS 3.x schema Options: - -c, --config Config file to override default config - -m, --compare-mode Compares by "value" (default), "type", "schema" - -q, --quiet No logging - -v, --verbose Verbose logging - -h, --help display help for command + -h, --help display help for command + ``` ## Config file @@ -426,6 +458,8 @@ module.exports = { sortComponentsAlphabetically: true, // Updates the parameter and requestBody examples using the x-examples from the 200 response + // Requires a API server to be in the spec or specified in the command line if any paramaters come from scripts + // Requires config.securitySchemes (or `sec-tokens` on the command line) to be specified if any APIs specify the "security" property. syncExamples: true }, @@ -447,6 +481,11 @@ module.exports = { // A URI (URL or data:image/x-icon;base64 encode image) for the favicon webFaviconHref: 'url or data:image/x-icon;base64', }, + // If you wish to record your security tokens here, you may. + securitySchemes: { + schemeName1: { value: 'static value' }, + schemeName2: { script: 'path/to/script.js' } + } }; ``` diff --git a/fixtures/invalidSpecV2.json b/fixtures/invalidSpecV2.json index 8c7f637..143219c 100644 --- a/fixtures/invalidSpecV2.json +++ b/fixtures/invalidSpecV2.json @@ -1,5 +1,8 @@ { "swagger": "2.0", + "servers": [{ + "url": "foo" + }], "info": { "description": "VFE API", "version": "1.0.0", diff --git a/fixtures/output/lint2.json b/fixtures/output/lint2.json index 987e499..1709a90 100644 --- a/fixtures/output/lint2.json +++ b/fixtures/output/lint2.json @@ -118,9 +118,6 @@ }, "id_2": { "value": "2" - }, - "badParam": { - "value": "wrong-param" } } } diff --git a/fixtures/specWithValidExamples.json b/fixtures/specWithValidExamples.json new file mode 100644 index 0000000..fa3b735 --- /dev/null +++ b/fixtures/specWithValidExamples.json @@ -0,0 +1,183 @@ +{ + "openapi": "3.0.0", + "info": { + "description": "Fake Online REST API for Testing and Prototyping", + "version": "1.0.0", + "title": "JSON Placeholder" + }, + "tags": [ + { + "name": "posts" + } + ], + "servers": [ + { + "url": "https://jsonplaceholder.typicode.com" + } + ], + "paths": { + "/posts": { + "get": { + "tags": [ + "posts" + ], + "summary": "Get all available posts", + "parameters": [ + { + "name": "id", + "in": "query", + "description": "Filter by post ID", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "userId", + "in": "query", + "description": "Filter by user ID", + "required": false, + "schema": { + "type": "string" + }, + "examples": { + "products": { + "value": "products" + }, + "service": { + "value": "services" + }, + "eligibility-enterprise": { + "value": "eligibility" + }, + "eligibility-inactive": { + "value": "eligibility" + }, + "invalidToken": { + "value": "eligibility" + } + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Post" + } + } + } + }, + "x-mock-file": "", + "x-examples": { + "default": { + "parameters": [ + { + "value": null + }, + { + "value": null + } + ] + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Post": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "userId": { + "type": "integer", + "format": "int64" + }, + "title": { + "type": "string" + }, + "body": { + "type": "string" + } + } + }, + "User": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "name": { + "type": "string" + }, + "username": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, + "phone": { + "type": "string" + }, + "website": { + "type": "string" + }, + "company": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "catchPhrase": { + "type": "string" + }, + "bs": { + "type": "string" + } + } + }, + "address": { + "type": "object", + "properties": { + "street": { + "type": "string" + }, + "suite": { + "type": "string" + }, + "city": { + "type": "string" + }, + "zipcode": { + "type": "string" + }, + "geo": { + "type": "object", + "properties": { + "lat": { + "type": "string" + }, + "lng": { + "type": "string" + } + } + } + } + } + } + } + } + } +} diff --git a/jest.config.js b/jest.config.js index 1fba8de..c8f538a 100644 --- a/jest.config.js +++ b/jest.config.js @@ -12,10 +12,10 @@ module.exports = { coverageDirectory: '/test-reports/coverage', coverageThreshold: { global: { - statements: 50, - branches: 50, - functions: 50, - lines: 50, + statements: 55, + branches: 55, + functions: 55, + lines: 55, }, }, testTimeout: 20000, diff --git a/package-lock.json b/package-lock.json index 8d3914b..ee9ad72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "oaat", - "version": "1.0.2", + "version": "1.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -9424,9 +9424,9 @@ } }, "openapi-enforcer": { - "version": "1.10.7", - "resolved": "https://registry.npmjs.org/openapi-enforcer/-/openapi-enforcer-1.10.7.tgz", - "integrity": "sha512-MPeYrXmwp+/RcFBeVGN+2RctEWMosd4RaVvJYxdC1tXQtPvfrcbOwAqlqBJEC8HXo99Pot/QVa2Tm2lqmcc1Eg==", + "version": "1.10.8", + "resolved": "https://registry.npmjs.org/openapi-enforcer/-/openapi-enforcer-1.10.8.tgz", + "integrity": "sha512-j/TWSija3MXl+pFGye+Iwbka5r32FqBCHeSzOxwX+driTj4Z2nWmrn0n9WRg+2cEX1v/6tKq10jl3X/Pkl+lEA==", "requires": { "axios": "^0.19.2", "json-schema-ref-parser": "^6.0.1" diff --git a/package.json b/package.json index a2cf62b..8d3603d 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "lodash": "4.17.20", "mkdirp": "1.0.4", "node-fetch": "2.6.1", - "openapi-enforcer": "1.10.7", + "openapi-enforcer": "1.10.8", "p-pipe": "3.1.0", "upath": "1.2.0", "winston": "3.2.1" diff --git a/src/cli.js b/src/cli.js index 2e17793..d80f4d8 100755 --- a/src/cli.js +++ b/src/cli.js @@ -6,22 +6,30 @@ const { getLogAndConfig } = require('./utils'); program.name('oaat').version(version).usage(''); +// NOTE: Any commands which require resolved-parameter values WILL need +// an optional serverUrl and optional sec-tokens, if scripts or security is +// used by the API. + program - .command('lint ') + .command('lint [serverUrl]') .description('Tidy the API Spec up a bit') .option('-o, --output ', 'Output file (if different to jsonFile)') .option('-c, --config ', 'Config file to override default config') + .option( + '-s, --sec-tokens ', + 'Pass security token(s) matching the "key" in spec.securitySchemes, with a "value"', + ) .option('-q, --quiet', 'no logging') .option('-v, --verbose', 'verbose logging') // eslint-disable-next-line @getify/proper-arrows/where - .action(async (jsonFile, cmd) => { + .action(async (jsonFile, serverUrl, cmd) => { const { log, config } = getLogAndConfig('lint', cmd); const { lintCommand } = require('./lint'); try { - await lintCommand(jsonFile, config); + await lintCommand(jsonFile, serverUrl, config); } catch (err) { - log.error(err); + handleErrors(log, config, err); } }); @@ -32,6 +40,10 @@ program ) .option('-o, --output ', 'Output file (if different to jsonFile)') .option('-c, --config ', 'Config file to override default config') + .option( + '-s, --sec-tokens ', + 'Pass security token(s) matching the "key" in spec.securitySchemes, with a "value"', + ) .option('-q, --quiet', 'No logging') .option('-v, --verbose', 'Verbose logging') .option('-d, --dry-run', 'Dry run (no changes made)') @@ -44,8 +56,7 @@ program try { await recordCommand(jsonFile, serverUrl, config); } catch (err) { - // console.error(err); - log.error(err); + handleErrors(log, config, err); } }); @@ -75,7 +86,7 @@ program try { await addGatewayInfo(jsonFile, serverUrl, config); } catch (err) { - log.error(err); + handleErrors(log, config, err); } }); @@ -83,6 +94,10 @@ program .command('compare [serverUrl]') .description('Compares recorded responses (referenced by the spec file) to the latest responses') .option('-c, --config ', 'Config file to override default config') + .option( + '-s, --sec-tokens ', + 'Pass security token(s) matching the "key" in spec.securitySchemes, with a "value"', + ) .option('-m --compare-mode ', 'Comparison mode: "value" (default), "type", "schema"') .option('-q, --quiet', 'no logging') .option('-v, --verbose', 'verbose logging') @@ -100,7 +115,22 @@ program // eslint-disable-next-line no-process-exit return process.exit(errorCode); } catch (err) { - log.error(err); + handleErrors(log, config, err); + } + }); + +program + .command('validate ') + .description('Validate the API spec file against the OAS 3.x schema') + // eslint-disable-next-line @getify/proper-arrows/where + .action(async (jsonFile, cmd) => { + const { log, config } = getLogAndConfig('validate', cmd); + + const { validateCommand } = require('./validate'); + try { + await validateCommand(jsonFile, config); + } catch (err) { + handleErrors(log, config, err); } }); @@ -109,3 +139,10 @@ program.parse(process.argv); if (!process.argv.slice(2).length) { program.help(); } + +function handleErrors(log, config, err) { + log.error(err); + if (config.verbose) { + console.error(err); + } +} diff --git a/src/e2e.spec.js b/src/e2e.spec.js index 4a24c4e..c659837 100644 --- a/src/e2e.spec.js +++ b/src/e2e.spec.js @@ -14,10 +14,11 @@ describe('CLI', () => { -h, --help display help for command Commands: - lint [options] Tidy the API Spec up a bit + lint [options] [serverUrl] Tidy the API Spec up a bit record [options] [serverUrl] Record the responses of API spec file endpoint requests (optionally use a different server to make requests) build [options] [serverUrl] Adds custom headers & Swagger UI endpoint to allow deployment of spec file to AWS API Gateway with documentation compare [options] [serverUrl] Compares recorded responses (referenced by the spec file) to the latest responses + validate Validate the API spec file against the OAS 3.x schema help [command] display help for command " `); @@ -33,10 +34,11 @@ describe('CLI', () => { -h, --help display help for command Commands: - lint [options] Tidy the API Spec up a bit + lint [options] [serverUrl] Tidy the API Spec up a bit record [options] [serverUrl] Record the responses of API spec file endpoint requests (optionally use a different server to make requests) build [options] [serverUrl] Adds custom headers & Swagger UI endpoint to allow deployment of spec file to AWS API Gateway with documentation compare [options] [serverUrl] Compares recorded responses (referenced by the spec file) to the latest responses + validate Validate the API spec file against the OAS 3.x schema help [command] display help for command " `); @@ -366,6 +368,28 @@ describe('CLI', () => { }); }); }); + + describe('validate', () => { + it('should not display an error when the spec contains valid examples', async () => { + const result = await runCommand(`validate ./fixtures/specWithValidExamples.json`); + + expect(result).toMatchInlineSnapshot(` + "info: Validation complete. + " + `); + }); + + it('should display an error when the spec is not valid', async () => { + const result = await runCommand(`validate ./fixtures/invalidSpecV2.json`); + + expect(result).toMatchInlineSnapshot(` + "error: One or more errors exist in the OpenApi definition + Property not allowed: swagger + Missing required property: openapi + " + `); + }); + }); }); function runCommand(args = '', cwd = process.cwd()) { diff --git a/src/lint.js b/src/lint.js index 75dcb3b..081dd2a 100644 --- a/src/lint.js +++ b/src/lint.js @@ -1,25 +1,27 @@ const { pipe, + addParamsToFetchConfig, readJsonFile, writeOutputFile, getAbsSpecFilePath, getFetchConfigForAPIEndpoints, getExampleObject, - addParamsToFetchConfig, + isSuccessStatusCode, validateSpecObj, } = require('./utils'); const logger = require('winston'); -async function lintCommand(specFile, config) { +async function lintCommand(specFile, server, config) { const specObj = readJsonFile(specFile); const absSpecFilePath = getAbsSpecFilePath(specFile); + const serverUrl = server || specObj.servers[0].url; // Read the file, lint it, write it const pipeline = pipe(validateSpecObj, lintSpec, writeOutputFile, () => { logger.info('Linting complete.'); }); - return pipeline({ specObj, specFile, absSpecFilePath, config }); + return pipeline({ specObj, serverUrl, specFile, absSpecFilePath, config }); } function lintSpec(params) { @@ -51,6 +53,11 @@ function sortChildKeys(obj, childProp) { .reduce((newDefs, key) => ({ ...newDefs, [key]: obj[childProp][key] }), {}); } +/** + * Uses the x-examples object to generate an Open API Spec `examples` object + * @param params + * @return {Promise<*>} + */ async function syncExamples(params) { const { specObj, config, fetchConfigs: existingFetchConfig } = params; let fetchConfigs = existingFetchConfig; @@ -71,7 +78,8 @@ async function syncExamples(params) { fetchConfigs.forEach((fc) => { const example = getExampleObject(fc.apiEndpoint.responses[fc.expectedStatusCode], fc.exampleName); - if (!example) { + // If there is no example, or the example is for a non-2xx response, ignore it + if (!example || !isSuccessStatusCode(Number(fc.expectedStatusCode))) { return; } @@ -82,21 +90,39 @@ async function syncExamples(params) { specRef.parameters.forEach((param, i) => { param.examples = param.examples || {}; param.examples[fc.exampleName] = param.examples[fc.exampleName] || {}; - param.examples[fc.exampleName].value = example.parameters[i].value; + param.examples[fc.exampleName].value = fc.resolvedParams[i]; }); } if (specRef.requestBody && example.requestBody) { // Note: Pointing directly to the application/json content! - const specExamples = specRef.requestBody.content['application/json']; + const isRequestRef = specRef.requestBody.$ref; + // // It is risky to update a shared $ref, as each $ref-user could have examples with names that over-write the other! + const specExamples = (isRequestRef ? getRequestBodyRefObject(specObj, isRequestRef) : specRef.requestBody) + .content['application/json']; specExamples.examples = specExamples.examples || {}; specExamples.examples[fc.exampleName] = specExamples.examples[fc.exampleName] || {}; - specExamples.examples[fc.exampleName].value = JSON.parse(fc.config.body); + specExamples.examples[fc.exampleName].value = fc.resolvedRequestBody; } }); return params; } +// Resolve "#/components/requestBodies/BodyRequest" into the object +function getRequestBodyRefObject(specObj, strRef) { + if (!strRef.startsWith('#')) { + throw new Error('$ref does not begin with "#"'); + } + const parts = strRef.split('/').slice(1); // ignore the first "#/" + let ref = specObj; + + while (parts.length) { + const path = parts.shift(); + ref = ref[path]; + } + return ref; +} + module.exports = { lintCommand, lintSpec, sortPaths, sortComponents, syncExamples }; diff --git a/src/lint.spec.js b/src/lint.spec.js index 9cd0b76..a68a3f0 100644 --- a/src/lint.spec.js +++ b/src/lint.spec.js @@ -205,7 +205,7 @@ describe('Lint', () => { }, 'x-examples': { default: { - parameters: [{ value: 1234 }], + parameters: [{ script: 'scriptValue1.js' }], requestBody: { script: 'scriptValue2.js' }, }, }, @@ -226,7 +226,7 @@ describe('Lint', () => { expect.objectContaining({ examples: { default: { - value: 1234, + value: 'async value 1', }, }, }), @@ -243,6 +243,73 @@ describe('Lint', () => { ); }); + it('should resolve the requestBody $ref and add the examples to it', async () => { + const specObj = { + paths: { + '/posts': { + get: { + tags: ['posts'], + operationId: 'getPosts', + summary: 'Get all available posts', + requestBody: { + $ref: '#/components/requestBodies/BodyInput', + }, + responses: { + 200: { + description: 'successful operation', + content: { + 'application/json': { + schema: { + type: 'array', + items: { + $ref: '#/components/schemas/Post', + }, + }, + }, + }, + 'x-examples': { + default: { + requestBody: { value: { clientId: 123 } }, + }, + }, + }, + }, + }, + }, + }, + components: { + requestBodies: { + BodyInput: { + description: 'Example request body', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/Info', + }, + }, + }, + }, + }, + }, + }; + + const config = { + syncExamples: true, + }; + const absSpecFilePath = '../fixtures'; + + const { specObj: output } = await syncExamples({ specObj, absSpecFilePath, config }); + expect(output.components.requestBodies.BodyInput.content['application/json']).toEqual( + expect.objectContaining({ + examples: { + default: { + value: { clientId: 123 }, + }, + }, + }), + ); + }); + it('should resolve script-parameters relative to the absSpecFilePath', async () => { const specObj = { paths: { diff --git a/src/utils.js b/src/utils.js index 70c9f6c..3f4d8da 100644 --- a/src/utils.js +++ b/src/utils.js @@ -29,7 +29,7 @@ function getFetchConfigForAPIEndpoints(params) { function concurrentFunctionProcessor(fnArray, maxConcurrent = 15) { return new Promise((resolve) => { // https://nodesource.com/blog/understanding-streams-in-nodejs/ - // Create a stream from an array of functions, that execute those functions + // Create a stream from an array of functions, then execute those functions // until we reach the maxConcurrent limit. When a function has resolved, // we can resume the stream and process the next function. const { Readable } = require('stream'); @@ -94,6 +94,7 @@ function transformEndpointMethod(path, method, data) { // Iterate over the EXAMPLE_PROP_NAME, to allow multiple examples for each path-method-response function transformEndpointMethodResponse(path, method, statusCode, data) { + // To cater for the "200" response endpoints that don't NEED an explicit x-examples object, we use { default: {} }; const newEndPoints = Object.keys(data.responses[statusCode][EXAMPLE_PROP_NAME] || { default: {} }) .map((exampleName, exampleIndex) => buildFetchConfigWithoutParameters(path, method, Number(statusCode), exampleName, exampleIndex, data), @@ -109,6 +110,7 @@ function transformEndpointMethodResponse(path, method, statusCode, data) { // If there parameters but no EXAMPLE_PROP_NAME in the response[statusCode], then we don't build it. function buildFetchConfigWithoutParameters(path, method, statusCode, exampleName, exampleIndex, data) { updateExampleObject(data.responses[statusCode], exampleName); // Make sure we have an examples object + const examples = data.responses[statusCode][EXAMPLE_PROP_NAME]; // Define config base-object, which we will change as needed @@ -117,7 +119,7 @@ function buildFetchConfigWithoutParameters(path, method, statusCode, exampleName query: {}, // Needed so that we can build query parameters url: path, config: { method }, - apiEndpoint: data, + apiEndpoint: data, // everything under paths[url][method] expectedStatusCode: Number(statusCode), existingResponseFile: examples[exampleName].responseFile, // This is useful for the integration tests ignorePathsList: data.responses[statusCode][IGNORE_PROPERTY_PROP_NAME] || [], // Used by integration tests @@ -158,10 +160,10 @@ const paramTypeMapping = { fc.config.body = JSON.stringify(paramValue); return fc; }, - query: (fc, paramValue, queryParam) => { + query: (fc, paramValue, paramKey) => { // If we get a null value, it means there is no value for the query string param if (paramValue !== null) { - fc.query[queryParam] = paramValue; + fc.query[paramKey] = paramValue; } return fc; }, @@ -177,6 +179,9 @@ const paramTypeMapping = { Object.assign(fc.config.headers, { [paramKey]: paramValue }); return fc; }, + cookie: () => { + throw new Error('Cookies are not implement. Pull request welcome.'); + }, }; /* eslint-enable @getify/proper-arrows/where */ @@ -190,16 +195,32 @@ const paramTypeMapping = { */ async function addParamsToFetchConfig(params) { // console.log('add', fetchConfigs ); - const { fetchConfigs, config, serverUrl, absSpecFilePath } = params; + const { fetchConfigs, serverUrl, absSpecFilePath } = params; + const { simultaneousRequests } = params.config; const newConfigs = await concurrentFunctionProcessor( fetchConfigs.map((fconfig) => () => resolveFetchConfigParams({ fconfig, serverUrl, absSpecFilePath })), - config.simultaneousRequests, + simultaneousRequests, ); - logger.debug('addParamsToFetchConfig Params', newConfigs); + // Add the security info, if required + const configsWithSecurity = await addSecurityConfig({ ...params, fetchConfigs: newConfigs }); + + // Last step, convert fconfig.query into a URL + + // In the case where we have query parameters, append them to the url + // We can't do this directly in the paramTypeMapping.query() function because + // we don't know when to use '?' or '&' until we know all of the query params. + const finalConfigs = configsWithSecurity.map((fconfig) => { + if (Object.keys(fconfig.query).length) { + fconfig.url += `?${qs.stringify(fconfig.query)}`; + } + return fconfig; + }); - return { ...params, fetchConfigs: newConfigs }; + logger.debug('addParamsToFetchConfig Params', finalConfigs); + + return { ...params, fetchConfigs: finalConfigs }; } async function resolveFetchConfigParams(params) { @@ -219,6 +240,10 @@ async function resolveFetchConfigParams(params) { absSpecFilePath, ); fconfig = paramTypeMapping[paramType](fconfig, paramValue, pathParam); + + // Add the parameter value to the fconfig.resolvedParams array + fconfig.resolvedParams = fconfig.resolvedParams || []; + fconfig.resolvedParams[index] = paramValue; }), ); @@ -230,11 +255,7 @@ async function resolveFetchConfigParams(params) { absSpecFilePath, ); fconfig = paramTypeMapping.body(fconfig, paramValue); - } - - // In the case where we have query parameters, append them to the url - if (Object.keys(fconfig.query).length) { - fconfig.url += `?${qs.stringify(fconfig.query)}`; + fconfig.resolvedRequestBody = paramValue; } return fconfig; @@ -249,16 +270,100 @@ function getParamValue(serverUrl, inputObj, absSpecFilePath) { if (inputObj.value !== undefined) { return inputObj.value; } - throw new Error( - `${EXAMPLE_PROP_NAME}.example.parameters object must contain a \`script\` OR \`value\` property. Received: ${JSON.stringify( - inputObj, - )}`, + throw new Error(`Object must contain a \`script\` OR \`value\` property. Received: ${JSON.stringify(inputObj)}`); +} + +/** + * Security can be specified at the global level, or the endpoint level. + * Once we work out whether an endpoint requires security, we need to + * add it to the fetch config. + * @param params + * @returns fetchConfigs + */ +async function addSecurityConfig(params) { + const { fetchConfigs, specObj, config, serverUrl, absSpecFilePath } = params; + const globalSecurity = specObj.security; + const securityKeyValues = config.securitySchemes; + + // If there are no securitySchemes, don't do anything + if (!(specObj.components && specObj.components.securitySchemes)) { + return fetchConfigs; + } + + const securitySchemes = specObj.components.securitySchemes; + + // Loop over the endpoints, looking for security; + const newConfigs = await Promise.all( + fetchConfigs.map((fconfig) => + updateConfigWithSecurityScheme({ + fetchConfig: fconfig, + securitySchemes, + apiSecurity: fconfig.apiEndpoint.security || globalSecurity, + securityKeyValues, + serverUrl, + absSpecFilePath, + }), + ), ); + + return newConfigs; } +/** + * Resolves the value/script + * @param secParams.fetchConfig + * @param secParams.securitySchemes + * @param secParams.secSchema + * @param secParams.securityKeyValues + * @param secParams.serverUrl + * @param secParams.absSpecFilePath + * @return {*} + */ +async function updateConfigWithSecurityScheme(secParams) { + let { fetchConfig, securitySchemes, apiSecurity, securityKeyValues, serverUrl, absSpecFilePath } = secParams; + if (!apiSecurity || !Array.isArray(apiSecurity) || apiSecurity.length === 0) { + return fetchConfig; + } + + const selectedScheme = apiSecurity[0]; + + // The selectedScheme is a map of schemes. Loop over each scheme-key + await Promise.all( + Object.keys(selectedScheme).map(async (schemeName) => { + const schemaConfig = securitySchemes[schemeName]; + logger.debug(`Resolving ${schemeName} in securitySchemes, ${JSON.stringify(securityKeyValues)}`); + + // TODO: try..catch to return a more helpful error + const securityValue = await getParamValue(serverUrl, securityKeyValues[schemeName], absSpecFilePath); + + fetchConfig = securityTypeMapping[schemaConfig.type](schemaConfig, securityValue, fetchConfig); + }), + ); + + return fetchConfig; +} + +/* eslint-disable @getify/proper-arrows/where */ +const securityTypeMapping = { + http: (secConfig, securityValue, fetchConfig) => { + const scheme = (secConfig.scheme || '').toLowerCase(); + if (scheme === 'bearer') { + return paramTypeMapping.header(fetchConfig, `Bearer ${securityValue}`, 'Authorization'); + } + if (scheme === 'basic') { + return paramTypeMapping.header(fetchConfig, `Basic ${securityValue}`, 'Authorization'); + } + return fetchConfig; // default + }, + apiKey: (secConfig, securityValue, fetchConfig) => { + const paramType = secConfig.in; // This can be "header", "query" or "cookie". Only header and query are implemented at the moment (cookie should be avoided) + return paramTypeMapping[paramType](fetchConfig, securityValue, secConfig.name); + }, +}; + /** * Gets the example from the object, or returns undefined. - * @param obj The api-spec response-status object (path.method.responses.status) + * @param responseStatusObj The api-spec response-status object (path.method.responses.status) * @param exampleName The name of the example * @return {any} The example object, or undefined */ @@ -278,8 +383,8 @@ function getExampleObject(responseStatusObj, exampleName) { * creating the example object if necessary, along with a key and value if provided * @param {object} responseStatusObj path.method.responses.status object * @param {string} exampleName Name of the example - * @param {string} key Optional key - * @param {object} value Optional value + * @param {string} [key] Optional key + * @param {object} [value] Optional value */ function updateExampleObject(responseStatusObj, exampleName, key, value) { if (!responseStatusObj[EXAMPLE_PROP_NAME]) { @@ -319,26 +424,43 @@ function writeJsonFile(fileName, data, config) { function getLogAndConfig(commandName, cmd) { const { configureLogger } = require('./logger'); const log = configureLogger({ logLevel: cmd.verbose ? 'verbose' : 'info', silent: Boolean(cmd.quiet) }); + log.debug('command args', cmd); // Read the default config file const defaultConfig = require('./defaultConfig'); - const config = { ...defaultConfig[commandName], ...defaultConfig.global }; + const config = { ...defaultConfig[commandName], ...defaultConfig.global, securitySchemes: {} }; // If a config file has been specified, try to merge it into the defaultConfig if (cmd.config) { try { const merge = require('lodash/merge'); const externalConfig = require(join(process.cwd(), cmd.config)); - merge(config, { ...externalConfig[commandName], ...externalConfig.global }); + merge(config, { + ...externalConfig[commandName], + ...externalConfig.global, + securitySchemes: { ...config.securitySchemes, ...externalConfig.securitySchemes }, + }); } catch (err) { log.error('Could not load specified config file', err); } } + config.verbose = cmd.verbose; config.dryRun = Boolean(cmd.dryRun); config.outputFile = cmd.output; + // Add the security tokens from the command line to the config, if they exist + if (cmd.secTokens) { + config.securitySchemes = config.securitySchemes || {}; + // Split the tokens into key-value strings, then split again and add to securitySchemes + const tokens = cmd.secTokens.split(','); + tokens.forEach((keyValue) => { + const [key, value] = keyValue.split('='); + config.securitySchemes[key] = { value }; + }); + } + log.debug('config', config); return { @@ -418,6 +540,10 @@ async function validateSpecObj(params) { return { ...params, openapi }; } +function isSuccessStatusCode(statusCodeNumber) { + return statusCodeNumber >= 200 && statusCodeNumber < 300; +} + module.exports = { addParamsToFetchConfig, concurrentFunctionProcessor, @@ -428,6 +554,7 @@ module.exports = { getFetchConfigForAPIEndpoints, getLogAndConfig, getPathsToIgnore: getResponsePropertyPathsToIgnore, + isSuccessStatusCode, pipe, readJsonFile, returnExitCode, diff --git a/src/utils.spec.js b/src/utils.spec.js index 5e55d34..d9b8d5d 100644 --- a/src/utils.spec.js +++ b/src/utils.spec.js @@ -573,6 +573,7 @@ describe('utils', () => { fetchConfigs: input, serverUrl: 'https://example.com/', config, + specObj: {}, }); expect(fetchConfigs).toEqual([ { @@ -632,6 +633,7 @@ describe('utils', () => { fetchConfigs: input, serverUrl: 'https://example.com/', config, + specObj: {}, }); expect(fetchConfigs).toEqual([ expect.objectContaining({ @@ -685,6 +687,7 @@ describe('utils', () => { fetchConfigs: input, serverUrl: 'https://example.com/', config, + specObj: {}, }); expect(fetchConfigs).toEqual([ expect.objectContaining({ @@ -756,6 +759,7 @@ describe('utils', () => { fetchConfigs: input, serverUrl: 'https://example.com/', config, + specObj: {}, }); expect(fetchConfigs).toEqual([ expect.objectContaining({ @@ -821,6 +825,7 @@ describe('utils', () => { fetchConfigs: input, serverUrl: 'https://example.com/', config, + specObj: {}, }); expect(fetchConfigs).toEqual([ expect.objectContaining({ @@ -894,6 +899,7 @@ describe('utils', () => { serverUrl: 'https://example.com/', absSpecFilePath, config, + specObj: {}, }); expect(fetchConfigs).toEqual([ expect.objectContaining({ @@ -906,6 +912,333 @@ describe('utils', () => { }), ]); }); + + describe('with security', () => { + it('should add an Authorization header when the API has security with type http and scheme bearer', async () => { + const input = [ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get' }, + apiEndpoint: { + security: [{ scheme1: [] }], + responses: { + 200: { + description: 'Successful Operation', + }, + }, + }, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]; + // The securitySchemes key contains the user-specified security values + const config = { simultaneousRequests: 15, securitySchemes: { scheme1: { value: 'abc123' } } }; + const specObj = { + components: { + securitySchemes: { + scheme1: { + type: 'http', + scheme: 'bearer', + }, + }, + }, + }; + + const { fetchConfigs } = await addParamsToFetchConfig({ + fetchConfigs: input, + serverUrl: 'https://example.com/', + config, + specObj, + }); + expect(fetchConfigs).toEqual([ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get', headers: { Authorization: 'Bearer abc123' } }, // <--- + apiEndpoint: input[0].apiEndpoint, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]); + }); + + it('should add an Authorization header when the API has security with type http and scheme basic', async () => { + const input = [ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get' }, + apiEndpoint: { + security: [{ scheme1: [] }], + responses: { + 200: { + description: 'Successful Operation', + }, + }, + }, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]; + // The securitySchemes key contains the user-specified security values + const config = { simultaneousRequests: 15, securitySchemes: { scheme1: { value: 'abc123' } } }; + const specObj = { + components: { + securitySchemes: { + scheme1: { + type: 'http', + scheme: 'basic', + }, + }, + }, + }; + + const { fetchConfigs } = await addParamsToFetchConfig({ + fetchConfigs: input, + serverUrl: 'https://example.com/', + config, + specObj, + }); + expect(fetchConfigs).toEqual([ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get', headers: { Authorization: 'Basic abc123' } }, // <--- + apiEndpoint: input[0].apiEndpoint, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]); + }); + + it('should add an API key header when the API has global security with type apiKey and format "header", from a script', async () => { + const input = [ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get' }, + apiEndpoint: { + // Using global security + responses: { + 200: { + description: 'Successful Operation', + }, + }, + }, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]; + // The securitySchemes key contains the user-specified security values + const config = { + simultaneousRequests: 15, + securitySchemes: { scheme1: { script: '../fixtures/scriptValue1.js' } }, + }; + const specObj = { + security: [{ scheme1: [] }], + components: { + securitySchemes: { + scheme1: { + type: 'apiKey', + in: 'header', + name: 'x-api-key', + }, + }, + }, + }; + + const { fetchConfigs } = await addParamsToFetchConfig({ + fetchConfigs: input, + serverUrl: 'https://example.com/', + config, + specObj, + absSpecFilePath: '../fixtures', // <--- + }); + expect(fetchConfigs).toEqual([ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get', headers: { 'x-api-key': 'async value 1' } }, // <--- + apiEndpoint: input[0].apiEndpoint, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]); + }); + + it('should add an API key query string when the API has global security with type apiKey and format "header", from a script', async () => { + const input = [ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get' }, + apiEndpoint: { + // Using global security + responses: { + 200: { + description: 'Successful Operation', + 'x-examples': { + default: { + parameters: [{ value: 'queryParamValue1' }], + }, + }, + }, + }, + parameters: [ + { + name: 'q1', + in: 'query', + description: 'query param', + required: true, + schema: { + type: 'string', + }, + }, + ], + }, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]; + // The securitySchemes key contains the user-specified security values + const config = { + simultaneousRequests: 15, + securitySchemes: { scheme1: { script: '../fixtures/scriptValue1.js' } }, + }; + const specObj = { + security: [{ scheme1: [] }], + components: { + securitySchemes: { + scheme1: { + type: 'apiKey', + in: 'query', + name: 'secret', + }, + }, + }, + }; + + const { fetchConfigs } = await addParamsToFetchConfig({ + fetchConfigs: input, + serverUrl: 'https://example.com/', + config, + specObj, + absSpecFilePath: '../fixtures', + }); + expect(fetchConfigs).toEqual([ + { + path: '/api', + query: { + q1: 'queryParamValue1', + secret: 'async value 1', + }, + resolvedParams: ['queryParamValue1'], + url: '/api?q1=queryParamValue1&secret=async%20value%201', // <--- (not very secret ¯\_(ツ)_/¯ ) + config: { method: 'get' }, + apiEndpoint: input[0].apiEndpoint, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]); + }); + + it('should add multiple security schemes when they are specified', async () => { + const input = [ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get' }, + apiEndpoint: { + security: [{ scheme1: [], scheme2: [] }], + responses: { + 200: { + description: 'Successful Operation', + }, + }, + }, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]; + // The securitySchemes key contains the user-specified security values + const config = { + simultaneousRequests: 15, + securitySchemes: { scheme1: { value: 'abc123' }, scheme2: { value: 'xyz789' } }, + }; + const specObj = { + components: { + securitySchemes: { + scheme1: { + type: 'http', + scheme: 'bearer', + }, + scheme2: { + type: 'apiKey', + in: 'header', + name: 'x-api-key', + }, + }, + }, + }; + + const { fetchConfigs } = await addParamsToFetchConfig({ + fetchConfigs: input, + serverUrl: 'https://example.com/', + config, + specObj, + }); + expect(fetchConfigs).toEqual([ + { + path: '/api', + query: {}, + url: '/api', + config: { method: 'get', headers: { Authorization: 'Bearer abc123', 'x-api-key': 'xyz789' } }, // <--- + apiEndpoint: input[0].apiEndpoint, + expectedStatusCode: 200, + existingResponseFile: undefined, + ignorePathsList: [], + exampleName: 'default', + exampleIndex: 0, + }, + ]); + }); + }); }); describe('getLogAndConfig()', () => { @@ -917,6 +1250,7 @@ describe('utils', () => { expect(config).toEqual({ dryRun: false, outputFile: undefined, + securitySchemes: {}, simultaneousRequests: 15, sortComponentsAlphabetically: true, sortPathsAlphabetically: true, @@ -936,6 +1270,29 @@ describe('utils', () => { responseFilenameFn: expect.any(Function), outputFile: undefined, removeUnusedResponses: true, + securitySchemes: {}, + simultaneousRequests: 15, + updateResponseWhenInexactMatch: true, + updateResponseWhenTypesMatch: true, + }); + }); + + it('should include the security tokens from the command line, if present', () => { + const commandData = { secTokens: 'foo=bar,car=tar' }; + const { log, config } = getLogAndConfig('record', commandData); + + expect(typeof log.info).toEqual('function'); + expect(config).toEqual({ + andLint: true, + dryRun: false, + responseBasePath: 'responses/', + responseFilenameFn: expect.any(Function), + outputFile: undefined, + removeUnusedResponses: true, + securitySchemes: { + foo: { value: 'bar' }, + car: { value: 'tar' }, + }, simultaneousRequests: 15, updateResponseWhenInexactMatch: true, updateResponseWhenTypesMatch: true, @@ -956,6 +1313,7 @@ describe('utils', () => { responseFilenameFn: expect.any(Function), outputFile: undefined, removeUnusedResponses: false, // <-- from altConfig.js + securitySchemes: {}, simultaneousRequests: 15, updateResponseWhenInexactMatch: true, updateResponseWhenTypesMatch: true, diff --git a/src/validate.js b/src/validate.js new file mode 100644 index 0000000..e03e913 --- /dev/null +++ b/src/validate.js @@ -0,0 +1,16 @@ +const { pipe, readJsonFile, getAbsSpecFilePath, validateSpecObj } = require('./utils'); +const logger = require('winston'); + +async function validateCommand(specFile, config) { + const specObj = readJsonFile(specFile); + const absSpecFilePath = getAbsSpecFilePath(specFile); + + // Read the file, lint it, write it + const pipeline = pipe(validateSpecObj, () => { + logger.info('Validation complete.'); + }); + + return pipeline({ specObj, specFile, absSpecFilePath, config }); +} + +module.exports = { validateCommand };