From 7bd8f8ec1aad147a70635ec3c6c97f297a0e281d Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 28 Feb 2024 16:59:43 -0800 Subject: [PATCH 1/2] feat: implement selectors --- package.json | 3 +- packages/core/package.json | 4 +- packages/core/src/lib.js | 1 + packages/core/src/policy.js | 1 + packages/core/src/policy/api.js | 1 + packages/core/src/policy/api.ts | 88 ++++++ packages/core/src/policy/selector.js | 260 ++++++++++++++++ packages/core/test/policy/selector.spec.js | 19 ++ packages/core/test/policy/selector.vector.js | 303 +++++++++++++++++++ pnpm-lock.yaml | 44 +++ 10 files changed, 722 insertions(+), 2 deletions(-) create mode 100644 packages/core/src/policy.js create mode 100644 packages/core/src/policy/api.js create mode 100644 packages/core/src/policy/api.ts create mode 100644 packages/core/src/policy/selector.js create mode 100644 packages/core/test/policy/selector.spec.js create mode 100644 packages/core/test/policy/selector.vector.js diff --git a/package.json b/package.json index d77f6d9c..2d40ecd4 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "devDependencies": { "mocha": "^10.1.0", "prettier": "2.8.8", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "entail": "^2.1.2" }, "prettier": { "trailingComma": "es5", diff --git a/packages/core/package.json b/packages/core/package.json index 2e18d482..7aa3be83 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -24,6 +24,7 @@ "test:web": "playwright-test test/*.spec.js --cov && nyc report", "test:node": "c8 --check-coverage --branches 100 --functions 100 --lines 100 mocha test/*.spec.js", "test": "c8 --check-coverage --branches 100 --functions 100 --lines 100 mocha --bail test/*.spec.js", + "test:policy": "c8 --check-coverage --branches 100 --functions 100 --lines 100 entail test/policy/*.spec.js", "coverage": "c8 --reporter=html mocha test/*.spec.js && npm_config_yes=true npx st -d coverage -p 8080", "check": "tsc --build", "build": "tsc --build" @@ -44,7 +45,8 @@ "mocha": "^10.1.0", "nyc": "^15.1.0", "playwright-test": "^8.2.0", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "entail": "^2.1.2" }, "type": "module", "main": "src/lib.js", diff --git a/packages/core/src/lib.js b/packages/core/src/lib.js index 87ff9413..c7630018 100644 --- a/packages/core/src/lib.js +++ b/packages/core/src/lib.js @@ -22,3 +22,4 @@ export * as DID from '@ipld/dag-ucan/did' export * as Signature from '@ipld/dag-ucan/signature' export * from './result.js' export * as Schema from './schema.js' +export * as Selector from './policy/selector.js' diff --git a/packages/core/src/policy.js b/packages/core/src/policy.js new file mode 100644 index 00000000..04e6bf92 --- /dev/null +++ b/packages/core/src/policy.js @@ -0,0 +1 @@ +export * as Selector from './policy/selector.js' diff --git a/packages/core/src/policy/api.js b/packages/core/src/policy/api.js new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/packages/core/src/policy/api.js @@ -0,0 +1 @@ +export {} diff --git a/packages/core/src/policy/api.ts b/packages/core/src/policy/api.ts new file mode 100644 index 00000000..30c8832b --- /dev/null +++ b/packages/core/src/policy/api.ts @@ -0,0 +1,88 @@ +import type { Variant, Result, Phantom } from '@ucanto/interface' + +export type { Variant, Result } + +export type SelectorParseResult = Result + +export interface ParseError extends Error { + readonly name: 'ParseError' +} + +/** + * Selector use [jq](https://devdocs.io/jq/) notation. + */ +export type Selector = `.${string}` & Phantom<{ Selector: SelectorPath }> +export type SelectorPath = [IdentitySegment, ...SelectorSegment[]] + +export type SelectionResult = Variant<{ + one: Data + many: Data[] + error: ParseError | TypeError +}> + +export type SelectorSegment = + | IdentitySegment + | IteratorSegment + | IndexSegment + | KeySegment + | SliceSegment + +export type Data = + | number + | string + | boolean + | null + | Uint8Array + | ListData + | Dictionary + +export interface ListData extends Array {} +export interface Dictionary { + [key: string]: Data +} + +/** + * Represents an identity selector per jq. + * @see https://devdocs.io/jq/index#identity + * + * @example + * ```js + * assert.deepEqual( + * select('.', {x: 1}), + * { one: { x: 1 } } + * ) + * ``` + */ +export type IdentitySegment = '.' & Phantom<{ Identity: '.' }> + +/** + * Selector that returns all elements of an array. Selecting `.[]` from + * `[1,2,3]` will produce selection containing those numbers. The form `.foo.[]` + * is identical to `.foo[]`. + * + * You can also use the `[]` on the object, and it will return all the values of + * the object. + * + * @example + * ```js + * assert.deepEqual( + * select('.[]', [ + * {"name":"JSON", "good":true}, + * {"name":"XML", "good":false} + * ]), + * { + * many: [ + * {"name":"JSON", "good":true}, + * {"name":"XML", "good":false} + * ] + * } + * ) + * ``` + */ +export type IteratorSegment = '[]' & Phantom<{ Iterator: '[]' }> + +export type IndexSegment = number & Phantom<{ Index: number }> + +export type KeySegment = string & Phantom<{ Key: string }> + +export type SliceSegment = [start?: number, end?: number] diff --git a/packages/core/src/policy/selector.js b/packages/core/src/policy/selector.js new file mode 100644 index 00000000..d5beb881 --- /dev/null +++ b/packages/core/src/policy/selector.js @@ -0,0 +1,260 @@ +import * as API from './api.js' + +/** + * + * @param {API.Selector} selector + * @param {API.Data} subject + */ +export const select = (selector, subject) => { + const { error, ok: path } = parse(selector) + if (error) { + return { error } + } else { + return resolve(path, subject) + } +} + +/** + * @param {API.SelectorSegment[]} path + * @param {API.Data} subject + * @returns {API.SelectionResult} + */ +export const resolve = (path, subject) => { + let current = subject + for (const [offset, segment] of path.entries()) { + if (segment === Identity) { + continue + } + // If the segment is iterator, we are going to descend into the members of + // the current object or array. + else if (segment === Iterator) { + const many = [] + // However if we can only descend if the current subject is an object or + // an array. + if (current && typeof current === 'object') { + const subpath = path.slice(offset + 1) + const keys = Array.isArray(current) + ? current.keys() + : Object.keys(current).sort() + for (const key of keys) { + const member = current[/** @type {keyof current} */ (key)] + const result = resolve(subpath, member) + if (result.error) { + return result + } else if (result.many) { + many.push(...result.many) + } else { + many.push(result.one) + } + } + return { many } + } else { + return { + error: new TypeError(`Can not iterate over ${typeof current}`), + } + } + } else if (typeof segment === 'number') { + if (Array.isArray(current) || ArrayBuffer.isView(current)) { + current = segment in current ? current[segment] : null + } else { + return { + error: new TypeError( + `Can not index ${ + current === null ? null : typeof current + } with number ${segment}` + ), + } + } + } else if (typeof segment === 'string') { + if (current == null) { + return { one: null } + } else if (ArrayBuffer.isView(current)) { + return { + error: new TypeError( + `Can not access field ${JSON.stringify(segment)} on Uint8Array` + ), + } + } else if (Array.isArray(current)) { + return { + error: new TypeError( + `Can not access field ${JSON.stringify(segment)} on array` + ), + } + } else if (typeof current === 'object') { + const key = segment.replace(/\\\"/g, '"') + current = key in current ? current[key] : null + } else { + return { + error: new TypeError( + `Can not access field ${JSON.stringify( + segment + )} on ${typeof current}` + ), + } + } + } else { + const [start = 0, end] = segment + if ( + Array.isArray(current) || + ArrayBuffer.isView(current) || + typeof current === 'string' + ) { + current = current.slice(start, end) + } else { + return { + error: new TypeError( + `Can not slice ${start}:${end} from ${typeof current}` + ), + } + } + } + } + + return { one: current } +} + +export const Identity = '.' +export const Iterator = '[]' + +/** + * @param {API.Selector} selector + * @returns {Iterable} + */ +export function* tokenize(selector) { + const { length } = selector + let offset = 0 + let column = 0 + let context = '' + + while (column < length) { + const char = selector[column] + if (char === '"' && selector[column - 1] !== '\\') { + column++ + context = context === '"' ? '' : '"' + continue + } + + if (context === '"') { + column++ + continue + } + + switch (char) { + case '.': { + if (offset < column) { + yield selector.slice(offset, column) + } + offset = column + column++ + break + } + case '[': { + if (offset < column) { + yield selector.slice(offset, column) + } + offset = column + column++ + break + } + case ']': { + column++ + yield selector.slice(offset, column) + offset = column + break + } + default: { + column++ + } + } + } + + if (offset < column && context != '"') { + yield selector.slice(offset, column) + } +} + +/** + * + * @param {API.Selector} selector + * @returns {API.SelectorParseResult} + */ +export const parse = selector => { + if (selector[0] !== Identity) { + return { + error: new ParseError(`Selector must start with identity segment "."`), + } + } + + const segments = [] + let offset = 0 + for (const token of tokenize(selector)) { + const segment = token.replace(/\s+/g, '') + switch (segment) { + case Identity: { + if (segments[segments.length - 1] === Identity) { + return { + error: new ParseError( + `Selector contains unsupported recursive descent segment ".."` + ), + } + } + segments.push(segment) + break + } + case Iterator: { + segments.push(segment) + break + } + default: { + if (segment[0] === '[' && segment[segment.length - 1] === ']') { + const lookup = segment.slice(1, -1) + if (/^\d+$/.test(lookup)) { + segments.push(parseInt(lookup, 10)) + } else if (lookup[0] === '"' && lookup[lookup.length - 1] === '"') { + segments.push(lookup.slice(1, -1)) + } else if (/^((\-?\d+:\-?\d*)|(\-?\d*:\-?\d+))$/.test(lookup)) { + const [left, right] = lookup.split(':') + const start = left !== '' ? parseInt(left, 10) : undefined + const end = right !== '' ? parseInt(right, 10) : undefined + segments.push([start, end]) + } else { + return { + error: new ParseError( + `Selector contains invalid segment ${JSON.stringify( + segment + )} at ${offset}:${offset + token.length}` + ), + } + } + } else if (/^\.[a-zA-Z_]*$/.test(segment)) { + segments.push(segment.slice(1)) + } else { + return { + error: new ParseError( + `Selector contains invalid segment ${JSON.stringify( + segment + )} at ${offset}:${offset + token.length}` + ), + } + } + } + } + offset += token.length + } + + if (offset < selector.length) { + return { + error: new ParseError( + `Selector contains invalid segment ${JSON.stringify( + selector.slice(offset) + )} with unterminated string literal` + ), + } + } + + return { ok: /** @type {API.SelectorPath} */ (segments) } +} + +class ParseError extends Error { + name = /** @type {const} */ ('ParseError') +} diff --git a/packages/core/test/policy/selector.spec.js b/packages/core/test/policy/selector.spec.js new file mode 100644 index 00000000..454d0cf5 --- /dev/null +++ b/packages/core/test/policy/selector.spec.js @@ -0,0 +1,19 @@ +import * as Selector from '../../src/policy/selector.js' +import Vector from './selector.vector.js' + +/** + * @type {import('entail').Suite} + */ +export const testSelector = Object.fromEntries( + Vector.map(({ data, at, out }) => [ + `echo '${JSON.stringify(data)}' | jq '${at}'`, + assert => { + const result = Selector.select(at, data) + if (out.error) { + assert.match(result.error, out.error) + } else { + assert.deepEqual(result, out) + } + }, + ]) +) diff --git a/packages/core/test/policy/selector.vector.js b/packages/core/test/policy/selector.vector.js new file mode 100644 index 00000000..b13f90ed --- /dev/null +++ b/packages/core/test/policy/selector.vector.js @@ -0,0 +1,303 @@ +import * as API from '../../src/policy/api.js' + +export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionResult & {error?: RegExp}}[]} */ ([ + { + at: /** @type {any} */ ('x'), + data: { x: 1 }, + out: { error: /ParseError.*must start with.*\./ }, + }, + { + at: '..', + data: {}, + out: { error: /ParseError.*recursive descent.*\.\./ }, + }, + { + at: '.', + data: { x: 1 }, + out: { one: { x: 1 } }, + }, + // iterator + { + at: '.[]', + data: { x: 1, y: 2, z: 3 }, + out: { many: [1, 2, 3] }, + }, + { + at: '.[]', + data: { b: 2, a: 1 }, + out: { many: [1, 2] }, + message: 'keys are sorted', + }, + { + at: '.[]', + data: [1, 2, 3], + out: { many: [1, 2, 3] }, + }, + // nested iterator + { + at: '.[].x', + data: [{ x: 1 }, { x: 2 }, { y: 3 }, { x: 4 }], + out: { many: [1, 2, null, 4] }, + }, + { + at: '.[].x[]', + data: [{ x: 1 }, { x: 2 }, { y: 3 }, { x: 4 }], + out: { error: /TypeError: Can not iterate over number/ }, + }, + { + at: '.[].xs[]', + data: [{ xs: [1, 2] }, { xs: [3, 4] }], + out: { many: [1, 2, 3, 4] }, + }, + { + at: '.xs.length', + data: { xs: [1, 2, 3] }, + out: { error: /Can not access field "length" on array/ }, + }, + // slices + { + at: '.xs[0:]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: [1, 2, 3, 4, 5] }, + }, + { + at: '.xs[-2:]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: [4, 5] }, + }, + { + at: '.xs[:-2]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: [1, 2, 3] }, + }, + { + at: '.xs[-1:-2]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: [] }, + }, + { + at: '.xs[0:--2]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { error: /ParseError.*invalid segment.*--2/ }, + }, + { + at: '.xs[0:+2]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { error: /ParseError.*invalid segment.*\+2/ }, + }, + { + at: '.xs[:]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { error: /ParseError.*invalid segment/ }, + }, + { + at: '.xs[5:1]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: [] }, + }, + { + at: '.xs[0:0]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: [] }, + }, + { + at: '.xs[0:1]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: [1] }, + }, + { + at: '.xs[0:1][]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { many: [1] }, + }, + { + at: '.xs[0:1]', + data: { xs: { [0]: 1, [1]: 2, [2]: 3 } }, + out: { error: /Can not slice 0:1 from object/ }, + }, + // index access + { + at: '.xs[0]', + data: { xs: [1, 2, 3, 4, 5] }, + out: { one: 1 }, + }, + { + at: '.xs[0]', + data: { xs: [] }, + out: { one: null }, + }, + { + at: '.b[0:1]', + data: { b: new Uint8Array([1, 2, 3, 4, 5]) }, + out: { one: new Uint8Array([1]) }, + }, + { + at: '.b[2]', + data: { b: new Uint8Array([1, 2, 3, 4, 5]) }, + out: { one: 3 }, + }, + + { + at: '.b[0:-1][]', + data: { b: new Uint8Array([1, 2, 3, 4, 5]) }, + out: { many: [1, 2, 3, 4] }, + }, + + { + at: '.b[0]', + data: { b: new Uint8Array([]) }, + out: { one: null }, + }, + { + at: '.b.length', + data: { b: new Uint8Array([1, 2, 3]) }, + out: { error: /Can not access field "length" on Uint8Array/ }, + }, + { + at: '.t.length', + data: { t: 'hello' }, + out: { error: /Can not access field "length" on string/ }, + }, + { + at: '.t[0:2]', + data: { t: 'hello' }, + out: { one: 'he' }, + }, + { + at: '.t[-2:]', + data: { t: 'hello' }, + out: { one: 'lo' }, + }, + { + at: '.t[]', + data: { t: 'hello' }, + out: { error: /Can not iterate over string/ }, + }, + { + at: '.t[0]', + data: { t: 'hello' }, + // This is how jq does it, but perhaps it is better to return 'h' instead ? + out: { error: /Can not index string with number 0/ }, + }, + + { + at: '.o[0]', + data: { o: { [0]: true } }, + out: { error: /Can not index object with number 0/ }, + }, + + { + at: '.q[0]', + data: { q: 5 }, + out: { error: /Can not index number with number 0/ }, + }, + { + at: '.q[0]', + data: { q: null }, + // jq return null instead 🤷‍♂️ + out: { error: /Can not index null with number 0/ }, + }, + { + at: '.q[0]', + data: { q: true }, + out: { error: /Can not index boolean with number 0/ }, + }, + { + at: '.q[0]', + data: { q: false }, + out: { error: /Can not index boolean with number 0/ }, + }, + + // property access + { + at: '.q.0', + data: { q: { 0: true } }, + out: { error: /ParseError/ }, + }, + { + at: '.q.foo', + data: { q: null }, + out: { one: null }, + }, + { + at: '.q.toString', + data: { q: true }, + out: { error: /Can not access field "toString" on boolean/ }, + }, + { + at: '.q.toString', + data: { q: 0 }, + out: { error: /Can not access field "toString" on number/ }, + }, + { + at: '.q.slice', + data: { q: 'hello' }, + out: { error: /Can not access field "slice" on string/ }, + }, + + { + at: '.q.slice', + data: { q: [] }, + out: { error: /Can not access field "slice" on array/ }, + }, + { + at: '.q.slice', + data: { q: new Uint8Array([]) }, + out: { error: /Can not access field "slice" on Uint8Array/ }, + }, + { + at: '.q.bar', + data: { q: { bar: 3 } }, + out: { one: 3 }, + }, + { + at: '.q.baz', + data: { q: { bar: 3 } }, + out: { one: null }, + }, + { + at: '.q.baz.bar', + data: { q: { bar: 3 } }, + out: { one: null }, + }, + { + at: '.q["bar"]', + data: { q: { bar: 3 } }, + out: { one: 3 }, + }, + { + at: '.q["bar.baz"]', + data: { q: { 'bar.baz': true } }, + out: { one: true }, + }, + { + at: '.q["bar\\"baz"]', + data: { q: { 'bar"baz': true } }, + out: { one: true }, + }, + { + at: '.q["bar" ]', + data: { q: { bar: true } }, + out: { one: true }, + }, + { + at: '.q[ "bar" ]', + data: { q: { bar: true } }, + out: { one: true }, + }, + { + at: '.q[ \t "bar" \n ]', + data: { q: { bar: true } }, + out: { one: true }, + }, + { + at: '.q["bar\\"]', + data: { q: { bar: true } }, + out: { error: /ParseError.*unterminated string literal/ }, + }, + { + at: '.q[bar]', + data: { q: { bar: true } }, + out: { error: /ParseError/ }, + }, +]) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 70dc25fa..3c48324c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: devDependencies: + entail: + specifier: ^2.1.2 + version: 2.1.2 mocha: specifier: ^10.1.0 version: 10.2.0 @@ -97,6 +100,9 @@ importers: chai: specifier: ^4.3.6 version: 4.3.7 + entail: + specifier: ^2.1.2 + version: 2.1.2 mocha: specifier: ^10.1.0 version: 10.2.0 @@ -1419,6 +1425,11 @@ packages: resolution: {integrity: sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==} dev: true + /dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + dev: true + /diff@5.0.0: resolution: {integrity: sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==} engines: {node: '>=0.3.1'} @@ -1446,6 +1457,17 @@ packages: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} dev: true + /entail@2.1.2: + resolution: {integrity: sha512-/icW51VHeJo5j6z6/80vO6R7zAdvHDODHYcc2jItrhRvP/zfTlm2b+xcEkp/Vt3UI6R5651Stw0AGpE1Gzkm6Q==} + hasBin: true + dependencies: + dequal: 2.0.3 + globby: 13.1.4 + kleur: 4.1.5 + sade: 1.8.1 + uvu: 0.5.6 + dev: true + /es-abstract@1.21.1: resolution: {integrity: sha512-QudMsPOz86xYz/1dG1OuGBKOELjCh99IIWHLzy5znUB6j8xG2yMA7bfTV86VSqKF+Y/H08vQPR+9jyXpuC6hfg==} engines: {node: '>= 0.4'} @@ -1771,6 +1793,17 @@ packages: slash: 4.0.0 dev: true + /globby@13.1.4: + resolution: {integrity: sha512-iui/IiiW+QrJ1X1hKH5qwlMQyv34wJAYwH1vrf8b9kBA4sNiif3gKsMHa+BrdnOpEudWjpotfa7LrTzB1ERS/g==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + dir-glob: 3.0.1 + fast-glob: 3.2.12 + ignore: 5.2.4 + merge2: 1.4.1 + slash: 4.0.0 + dev: true + /gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} dependencies: @@ -3168,6 +3201,17 @@ packages: hasBin: true dev: true + /uvu@0.5.6: + resolution: {integrity: sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==} + engines: {node: '>=8'} + hasBin: true + dependencies: + dequal: 2.0.3 + diff: 5.0.0 + kleur: 4.1.5 + sade: 1.8.1 + dev: true + /v8-to-istanbul@9.1.0: resolution: {integrity: sha512-6z3GW9x8G1gd+JIIgQQQxXuiJtCXeAjp6RaPEPLv62mH3iPHPxV6W3robxtCzNErRo6ZwTmzWhsbNvjyEBKzKA==} engines: {node: '>=10.12.0'} From 5a82ba82c2e8bdc438a70b9e434c17dc9411959d Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Sat, 2 Mar 2024 00:29:31 -0800 Subject: [PATCH 2/2] chore: update selector implementation --- packages/core/src/policy/api.ts | 30 +- packages/core/src/policy/selector.js | 300 +++++++++++------ packages/core/test/policy/selector.spec.js | 4 +- packages/core/test/policy/selector.vector.js | 323 +++++++++++++------ 4 files changed, 459 insertions(+), 198 deletions(-) diff --git a/packages/core/src/policy/api.ts b/packages/core/src/policy/api.ts index 30c8832b..edc42e9b 100644 --- a/packages/core/src/policy/api.ts +++ b/packages/core/src/policy/api.ts @@ -8,24 +8,38 @@ export interface ParseError extends Error { readonly name: 'ParseError' } +export interface ResolutionError extends Error { + readonly name: 'ResolutionError' +} + /** * Selector use [jq](https://devdocs.io/jq/) notation. */ export type Selector = `.${string}` & Phantom<{ Selector: SelectorPath }> -export type SelectorPath = [IdentitySegment, ...SelectorSegment[]] +export type SelectorPath = [{ Identity: {} }, ...SelectorSegment[]] export type SelectionResult = Variant<{ one: Data many: Data[] - error: ParseError | TypeError + error: ParseError | ResolutionError }> -export type SelectorSegment = - | IdentitySegment - | IteratorSegment - | IndexSegment - | KeySegment - | SliceSegment +export type ResolutionResult = Variant<{ + one: Data + many: Data[] + error: ResolutionError +}> + +export type SelectorSegment = Variant<{ + Identity: {} + Iterator: { optional: boolean } + Index: { optional: boolean; index: number } + Key: { optional: boolean; key: string } + Slice: { + optional?: boolean + range: [undefined, number] | [number, undefined] | [number, number] + } +}> export type Data = | number diff --git a/packages/core/src/policy/selector.js b/packages/core/src/policy/selector.js index d5beb881..7b365ab6 100644 --- a/packages/core/src/policy/selector.js +++ b/packages/core/src/policy/selector.js @@ -4,6 +4,7 @@ import * as API from './api.js' * * @param {API.Selector} selector * @param {API.Data} subject + * @returns {API.SelectionResult} */ export const select = (selector, subject) => { const { error, ok: path } = parse(selector) @@ -17,17 +18,18 @@ export const select = (selector, subject) => { /** * @param {API.SelectorSegment[]} path * @param {API.Data} subject - * @returns {API.SelectionResult} + * @param {(string|number)[]} [at] + * @returns {API.ResolutionResult} */ -export const resolve = (path, subject) => { +export const resolve = (path, subject, at = []) => { let current = subject for (const [offset, segment] of path.entries()) { - if (segment === Identity) { + if (segment.Identity) { continue } // If the segment is iterator, we are going to descend into the members of // the current object or array. - else if (segment === Iterator) { + else if (segment.Iterator) { const many = [] // However if we can only descend if the current subject is an object or // an array. @@ -38,7 +40,7 @@ export const resolve = (path, subject) => { : Object.keys(current).sort() for (const key of keys) { const member = current[/** @type {keyof current} */ (key)] - const result = resolve(subpath, member) + const result = resolve(subpath, member, [...at, key]) if (result.error) { return result } else if (result.many) { @@ -50,61 +52,83 @@ export const resolve = (path, subject) => { return { many } } else { return { - error: new TypeError(`Can not iterate over ${typeof current}`), + error: new ResolutionError({ + reason: `Can not iterate over ${typeof current}`, + at, + }), } } - } else if (typeof segment === 'number') { - if (Array.isArray(current) || ArrayBuffer.isView(current)) { - current = segment in current ? current[segment] : null + } else if (segment.Index) { + const { index, optional } = segment.Index + at.push(index) + if (isIndexed(current)) { + current = index < 0 ? current[current.length + index] : current[index] + if (current === undefined) { + if (optional) { + current = null + } else { + return { + error: new ResolutionError({ + at, + reason: `Index ${index} is out of bounds`, + }), + } + } + } + } else if (optional) { + current = null } else { return { - error: new TypeError( - `Can not index ${ + error: new ResolutionError({ + reason: `Can not index ${ current === null ? null : typeof current - } with number ${segment}` - ), + } with number ${index}`, + at, + }), } } - } else if (typeof segment === 'string') { - if (current == null) { - return { one: null } - } else if (ArrayBuffer.isView(current)) { - return { - error: new TypeError( - `Can not access field ${JSON.stringify(segment)} on Uint8Array` - ), - } - } else if (Array.isArray(current)) { - return { - error: new TypeError( - `Can not access field ${JSON.stringify(segment)} on array` - ), + } else if (segment.Key) { + const { key, optional } = segment.Key + at.push(key) + if (isDictionary(current)) { + current = current[key] + if (current === undefined) { + if (optional) { + current = null + } else { + return { + error: new ResolutionError({ + at, + reason: `Object has no property named ${JSON.stringify(key)}`, + }), + } + } } - } else if (typeof current === 'object') { - const key = segment.replace(/\\\"/g, '"') - current = key in current ? current[key] : null + } else if (optional) { + current = null } else { return { - error: new TypeError( - `Can not access field ${JSON.stringify( - segment - )} on ${typeof current}` - ), + error: new ResolutionError({ + reason: `Can not access field ${JSON.stringify(key)} on ${typeOf( + current + )}`, + at, + }), } } - } else { - const [start = 0, end] = segment - if ( - Array.isArray(current) || - ArrayBuffer.isView(current) || - typeof current === 'string' - ) { + } else if (segment.Slice) { + const { range, optional } = segment.Slice + const [start = 0, end] = range + if (isIndexed(current)) { current = current.slice(start, end) + } else if (optional) { + current = null } else { return { - error: new TypeError( - `Can not slice ${start}:${end} from ${typeof current}` - ), + error: new ResolutionError({ + reason: `Can not slice from ${typeof current}`, + at, + }), } } } @@ -113,8 +137,37 @@ export const resolve = (path, subject) => { return { one: current } } +/** + * @param {API.Data} value + */ +const typeOf = value => { + if (value === null) { + return 'null' + } else if (ArrayBuffer.isView(value)) { + return 'bytes' + } else if (Array.isArray(value)) { + return 'array' + } else { + return typeof value + } +} + +/** + * @param {unknown} value + * @returns {value is API.ListData|Uint8Array|string} + */ +const isIndexed = value => + ArrayBuffer.isView(value) || Array.isArray(value) || typeof value === 'string' + +/** + * @param {API.Data} value + * @returns {value is API.Dictionary} + */ +const isDictionary = value => typeOf(value) === 'object' + export const Identity = '.' export const Iterator = '[]' +export const OptionalIterator = '[]?' /** * @param {API.Selector} selector @@ -156,12 +209,6 @@ export function* tokenize(selector) { column++ break } - case ']': { - column++ - yield selector.slice(offset, column) - offset = column - break - } default: { column++ } @@ -173,88 +220,159 @@ export function* tokenize(selector) { } } +/** @type {API.SelectorSegment} */ +const IDENTITY = { Identity: {} } + /** * - * @param {API.Selector} selector + * @param {API.Selector} source * @returns {API.SelectorParseResult} */ -export const parse = selector => { - if (selector[0] !== Identity) { +export const parse = source => { + if (source[0] !== Identity) { return { - error: new ParseError(`Selector must start with identity segment "."`), + error: new ParseError({ + reason: `Selector must start with identity segment "."`, + source: source, + column: 0, + token: source[0], + }), } } + /** @type {API.SelectorSegment[]} */ const segments = [] - let offset = 0 - for (const token of tokenize(selector)) { - const segment = token.replace(/\s+/g, '') + let token = '' + let column = 0 + for (token of tokenize(source)) { + const trimmed = token.replace(/\s+/g, '') + const optional = trimmed[trimmed.length - 1] === '?' + const segment = optional ? trimmed.slice(0, -1) : trimmed switch (segment) { case Identity: { - if (segments[segments.length - 1] === Identity) { + if (segments[segments.length - 1] === IDENTITY) { return { - error: new ParseError( - `Selector contains unsupported recursive descent segment ".."` - ), + error: new ParseError({ + source, + reason: `Selector contains unsupported recursive descent segment ".."`, + column, + token, + }), } } - segments.push(segment) + segments.push(IDENTITY) break } case Iterator: { - segments.push(segment) + segments.push({ Iterator: { optional } }) break } default: { if (segment[0] === '[' && segment[segment.length - 1] === ']') { const lookup = segment.slice(1, -1) - if (/^\d+$/.test(lookup)) { - segments.push(parseInt(lookup, 10)) - } else if (lookup[0] === '"' && lookup[lookup.length - 1] === '"') { - segments.push(lookup.slice(1, -1)) - } else if (/^((\-?\d+:\-?\d*)|(\-?\d*:\-?\d+))$/.test(lookup)) { + // Is it an indexed access e.g. [3] + if (/^-?\d+$/.test(lookup)) { + segments.push({ + Index: { optional, index: parseInt(lookup, 10) }, + }) + } + // Is it a quoted key access e.g. ["key"] + else if (lookup[0] === '"' && lookup[lookup.length - 1] === '"') { + segments.push({ + Key: { optional, key: lookup.slice(1, -1).replace(/\\\"/g, '"') }, + }) + } + // Is this a slice access e.g. [3:5] or [:5] or [3:] + else if (/^((\-?\d+:\-?\d*)|(\-?\d*:\-?\d+))$/.test(lookup)) { const [left, right] = lookup.split(':') const start = left !== '' ? parseInt(left, 10) : undefined const end = right !== '' ? parseInt(right, 10) : undefined - segments.push([start, end]) - } else { + segments.push({ + Slice: { + optional, + range: /** @type {[number, number]} */ ([start, end]), + }, + }) + } + // Otherwise this is an error + else { return { - error: new ParseError( - `Selector contains invalid segment ${JSON.stringify( - segment - )} at ${offset}:${offset + token.length}` - ), + error: new ParseError({ + source, + column, + token, + }), } } - } else if (/^\.[a-zA-Z_]*$/.test(segment)) { - segments.push(segment.slice(1)) + } else if (/^\.[a-zA-Z_]*?$/.test(segment)) { + segments.push({ + Key: { optional, key: segment.slice(1) }, + }) } else { return { - error: new ParseError( - `Selector contains invalid segment ${JSON.stringify( - segment - )} at ${offset}:${offset + token.length}` - ), + error: new ParseError({ + source, + column, + token, + }), } } } } - offset += token.length + column += token.length } - if (offset < selector.length) { + if (column < source.length) { return { - error: new ParseError( - `Selector contains invalid segment ${JSON.stringify( - selector.slice(offset) - )} with unterminated string literal` - ), + error: new ParseError({ + source, + reason: `Unterminated string literal`, + column: column - token.length, + token: token, + }), } } return { ok: /** @type {API.SelectorPath} */ (segments) } } -class ParseError extends Error { +class ParseError extends SyntaxError { + /** + * @param {object} input + * @param {string} [input.reason] + * @param {string} input.source + * @param {string} input.token + * @param {number} input.column + * + */ + constructor({ + source, + column, + token, + reason = `Selector contains invalid segment:\n "${source}"\n ${' '.repeat( + column + )} ${'~'.repeat(token.length)}`, + }) { + super(reason) + this.reason = reason + this.source = source + this.column = column + this.token = token + } name = /** @type {const} */ ('ParseError') } + +class ResolutionError extends ReferenceError { + /** + * @param {object} input + * @param {string} [input.reason] + * @param {(string|number)[]} input.at + */ + constructor({ at, reason = `Can not resolve path ${at.join('')}` }) { + super(reason) + this.reason = reason + this.at = at + } + + name = /** @type {const} */ ('ResolutionError') +} diff --git a/packages/core/test/policy/selector.spec.js b/packages/core/test/policy/selector.spec.js index 454d0cf5..06399e52 100644 --- a/packages/core/test/policy/selector.spec.js +++ b/packages/core/test/policy/selector.spec.js @@ -5,8 +5,8 @@ import Vector from './selector.vector.js' * @type {import('entail').Suite} */ export const testSelector = Object.fromEntries( - Vector.map(({ data, at, out }) => [ - `echo '${JSON.stringify(data)}' | jq '${at}'`, + Vector.map(({ data, at, out, tag = '' }) => [ + `${tag}echo '${JSON.stringify(data)}' | jq '${at}'`, assert => { const result = Selector.select(at, data) if (out.error) { diff --git a/packages/core/test/policy/selector.vector.js b/packages/core/test/policy/selector.vector.js index b13f90ed..f357ff00 100644 --- a/packages/core/test/policy/selector.vector.js +++ b/packages/core/test/policy/selector.vector.js @@ -1,6 +1,6 @@ import * as API from '../../src/policy/api.js' -export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionResult & {error?: RegExp}}[]} */ ([ +export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionResult & {error?: RegExp}, tag?:string}[]} */ ([ { at: /** @type {any} */ ('x'), data: { x: 1 }, @@ -16,6 +16,158 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe data: { x: 1 }, out: { one: { x: 1 } }, }, + // property access + { + at: '.x', + data: { x: 1 }, + out: { one: 1 }, + }, + { + at: '.x', + data: { y: 1 }, + out: { error: /ResolutionError/ }, + }, + { + at: '.q.0', + data: { q: { 0: true } }, + out: { error: /ParseError/ }, + }, + { + at: '.q.foo', + data: { q: null }, + out: { one: null }, + }, + { + at: '.q.foo', + data: { q: null }, + out: { error: /ResolutionError/ }, + }, + { + at: '.q.foo?', + data: { q: null }, + out: { one: null }, + }, + { + at: '.q.toString', + data: { q: true }, + out: { error: /Can not access field "toString" on boolean/ }, + }, + { + at: '.q.toString?', + data: { q: true }, + out: { one: null }, + }, + { + at: '.q.toString', + data: { q: 0 }, + out: { error: /Can not access field "toString" on number/ }, + }, + { + at: '.q.toString?', + data: { q: 0 }, + out: { one: null }, + }, + { + at: '.q.slice', + data: { q: 'hello' }, + out: { error: /Can not access field "slice" on string/ }, + }, + { + at: '.q.slice?', + data: { q: 'hello' }, + out: { one: null }, + }, + { + at: '.q.slice', + data: { q: [] }, + out: { error: /Can not access field "slice" on array/ }, + }, + { + at: '.q.slice?', + data: { q: [] }, + out: { one: null }, + }, + { + at: '.q.slice', + data: { q: new Uint8Array([]) }, + out: { error: /Can not access field "slice" on bytes/ }, + }, + { + at: '.q.slice?', + data: { q: new Uint8Array([]) }, + out: { one: null }, + }, + { + at: '.q.bar', + data: { q: { bar: 3 } }, + out: { one: 3 }, + }, + { + at: '.q.baz', + data: { q: { bar: 3 } }, + out: { error: /ResolutionError/ }, + }, + { + at: '.q.baz?', + data: { q: { bar: 3 } }, + out: { one: null }, + }, + { + at: '.q.baz.bar', + data: { q: { bar: 3 } }, + out: { error: /ResolutionError/ }, + }, + { + at: '.q.baz?.bar', + data: { q: { bar: 3 } }, + out: { error: /ResolutionError/ }, + }, + { + at: '.q.baz?.bar?', + data: { q: { bar: 3 } }, + out: { one: null }, + }, + { + at: '.q["bar"]', + data: { q: { bar: 3 } }, + out: { one: 3 }, + }, + { + at: '.q["bar.baz"]', + data: { q: { 'bar.baz': true } }, + out: { one: true }, + }, + { + at: '.q["bar\\"baz"]', + data: { q: { 'bar"baz': true } }, + out: { one: true }, + }, + { + at: '.q["bar" ]', + data: { q: { bar: true } }, + out: { one: true }, + }, + { + at: '.q[ "bar" ]', + data: { q: { bar: true } }, + out: { one: true }, + }, + { + at: '.q[ \t "bar" \n ]', + data: { q: { bar: true } }, + out: { one: true }, + }, + { + at: '.q["bar\\"]', + data: { q: { bar: true } }, + out: { error: /ParseError.*unterminated string literal/i }, + }, + { + at: '.q[bar]', + data: { q: { bar: true } }, + out: { error: /ParseError/ }, + }, + // iterator { at: '.[]', @@ -37,12 +189,17 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe { at: '.[].x', data: [{ x: 1 }, { x: 2 }, { y: 3 }, { x: 4 }], + out: { error: /ResolutionError/ }, + }, + { + at: '.[].x?', + data: [{ x: 1 }, { x: 2 }, { y: 3 }, { x: 4 }], out: { many: [1, 2, null, 4] }, }, { at: '.[].x[]', data: [{ x: 1 }, { x: 2 }, { y: 3 }, { x: 4 }], - out: { error: /TypeError: Can not iterate over number/ }, + out: { error: /ResolutionError: Can not iterate over number/ }, }, { at: '.[].xs[]', @@ -54,6 +211,16 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe data: { xs: [1, 2, 3] }, out: { error: /Can not access field "length" on array/ }, }, + { + at: '.[].[]', + data: [{ x: 1 }, { x: 2 }, { x: 3 }, 0], + out: { error: /Can not iterate over number/ }, + }, + { + at: '.[].[]?', + data: [{ x: 1 }, { x: 2 }, { x: 3 }], + out: { many: [1, 2, 3] }, + }, // slices { at: '.xs[0:]', @@ -78,12 +245,12 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe { at: '.xs[0:--2]', data: { xs: [1, 2, 3, 4, 5] }, - out: { error: /ParseError.*invalid segment.*--2/ }, + out: { error: /ParseError.*invalid segment.*\n.*--2/g }, }, { at: '.xs[0:+2]', data: { xs: [1, 2, 3, 4, 5] }, - out: { error: /ParseError.*invalid segment.*\+2/ }, + out: { error: /ParseError.*invalid segment.*\n.*\+2/g }, }, { at: '.xs[:]', @@ -113,7 +280,13 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe { at: '.xs[0:1]', data: { xs: { [0]: 1, [1]: 2, [2]: 3 } }, - out: { error: /Can not slice 0:1 from object/ }, + out: { error: /Can not slice from object/ }, + }, + { + at: '.xs[0:1]?', + data: { xs: { [0]: 1, [1]: 2, [2]: 3 } }, + // TODO: Should this be [] instead ? + out: { one: null }, }, // index access { @@ -124,6 +297,11 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe { at: '.xs[0]', data: { xs: [] }, + out: { error: /out of bounds/ }, + }, + { + at: '.xs[0]?', + data: { xs: [] }, out: { one: null }, }, { @@ -146,12 +324,17 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe { at: '.b[0]', data: { b: new Uint8Array([]) }, + out: { error: /out of bounds/ }, + }, + { + at: '.b[0]?', + data: { b: new Uint8Array([]) }, out: { one: null }, }, { at: '.b.length', data: { b: new Uint8Array([1, 2, 3]) }, - out: { error: /Can not access field "length" on Uint8Array/ }, + out: { error: /Can not access field "length" on bytes/ }, }, { at: '.t.length', @@ -176,8 +359,20 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe { at: '.t[0]', data: { t: 'hello' }, - // This is how jq does it, but perhaps it is better to return 'h' instead ? - out: { error: /Can not index string with number 0/ }, + // This is different from jq, which errors instead. + out: { one: 'h' }, + }, + { + at: '.t[-2]', + data: { t: 'hello' }, + // This is different from jq, which errors instead. + out: { one: 'l' }, + }, + { + at: '.t[10]', + data: { t: 'hello' }, + // This is different from jq, which errors instead. + out: { error: /out of bounds/ }, }, { @@ -185,12 +380,23 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe data: { o: { [0]: true } }, out: { error: /Can not index object with number 0/ }, }, + { + at: '.o[0]?', + data: { o: { [0]: true } }, + out: { one: null }, + }, { at: '.q[0]', data: { q: 5 }, out: { error: /Can not index number with number 0/ }, }, + + { + at: '.q[0]?', + data: { q: 5 }, + out: { one: null }, + }, { at: '.q[0]', data: { q: null }, @@ -198,106 +404,29 @@ export default /** @type {{at: API.Selector, data:API.Data, out: API.SelectionRe out: { error: /Can not index null with number 0/ }, }, { - at: '.q[0]', - data: { q: true }, - out: { error: /Can not index boolean with number 0/ }, - }, - { - at: '.q[0]', - data: { q: false }, - out: { error: /Can not index boolean with number 0/ }, - }, - - // property access - { - at: '.q.0', - data: { q: { 0: true } }, - out: { error: /ParseError/ }, - }, - { - at: '.q.foo', + at: '.q[0]?', data: { q: null }, + // jq return null instead 🤷‍♂️ out: { one: null }, }, { - at: '.q.toString', + at: '.q[0]', data: { q: true }, - out: { error: /Can not access field "toString" on boolean/ }, - }, - { - at: '.q.toString', - data: { q: 0 }, - out: { error: /Can not access field "toString" on number/ }, - }, - { - at: '.q.slice', - data: { q: 'hello' }, - out: { error: /Can not access field "slice" on string/ }, - }, - - { - at: '.q.slice', - data: { q: [] }, - out: { error: /Can not access field "slice" on array/ }, - }, - { - at: '.q.slice', - data: { q: new Uint8Array([]) }, - out: { error: /Can not access field "slice" on Uint8Array/ }, - }, - { - at: '.q.bar', - data: { q: { bar: 3 } }, - out: { one: 3 }, - }, - { - at: '.q.baz', - data: { q: { bar: 3 } }, - out: { one: null }, + out: { error: /Can not index boolean with number 0/ }, }, { - at: '.q.baz.bar', - data: { q: { bar: 3 } }, + at: '.q[0]?', + data: { q: true }, out: { one: null }, }, { - at: '.q["bar"]', - data: { q: { bar: 3 } }, - out: { one: 3 }, - }, - { - at: '.q["bar.baz"]', - data: { q: { 'bar.baz': true } }, - out: { one: true }, - }, - { - at: '.q["bar\\"baz"]', - data: { q: { 'bar"baz': true } }, - out: { one: true }, - }, - { - at: '.q["bar" ]', - data: { q: { bar: true } }, - out: { one: true }, - }, - { - at: '.q[ "bar" ]', - data: { q: { bar: true } }, - out: { one: true }, - }, - { - at: '.q[ \t "bar" \n ]', - data: { q: { bar: true } }, - out: { one: true }, - }, - { - at: '.q["bar\\"]', - data: { q: { bar: true } }, - out: { error: /ParseError.*unterminated string literal/ }, + at: '.q[0]', + data: { q: false }, + out: { error: /Can not index boolean with number 0/ }, }, { - at: '.q[bar]', - data: { q: { bar: true } }, - out: { error: /ParseError/ }, + at: '.q[0]?', + data: { q: false }, + out: { one: null }, }, ])