CLDR-16836 kbd: ebnf: move tests to js out of shell, check XML files

.github/workflows/keyboard.yml

@@ -38,6 +38,6 @@ jobs:
       - name: Compile Keyboards
         run: kmc --error-reporting build keyboards/3.0/*.xml
       - name: Check ABNF
-        run: bash tools/scripts/keyboard-abnf-tests/check-keyboard-abnf.sh
+        run: 'cd tools/scripts/keyboard-abnf-tests && npm ci && npm t'
       - name: Run Kbd Charts
         run: 'cd docs/charts/keyboards && npm ci && npm run build'

tools/scripts/keyboard-abnf-tests/.gitignore

@@ -0,0 +1 @@
+/node_modules

tools/scripts/keyboard-abnf-tests/README.md

@@ -0,0 +1,23 @@
+# Keyboard-abnf-tests
+
+Tests for and against the ABNF files, written in Node.js
+
+## To use
+
+- `npm ci`
+- `npm t`
+
+## To update
+
+Note there are four files. There's a `.d` directory for each ABNF file in keyboards/abnf/.  The "pass" files are expected to pass the ABNF and the "fail" to fail it.  Lines beginning with # are comments and skipped.
+
+- transform-from-required.d/from-match.pass.txt
+- transform-from-required.d/from-match.fail.txt
+- transform-to-required.d/to-replacement.pass.txt
+- transform-to-required.d/to-replacement.fail.txt
+
+## Copyright
+
+Copyright © 1991-2025 Unicode, Inc.
+All rights reserved.
+[Terms of use](https://www.unicode.org/copyright.html)

tools/scripts/keyboard-abnf-tests/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@unicode-org/keyboard-abnf-tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "node --test"
+  },
+  "keywords": [],
+  "author": "Steven R. Loomis <[email protected]>",
+  "license": "Unicode-3.0",
+  "description": "Tests for the keyboard ABNF",
+  "private": true,
+  "dependencies": {
+    "abnf": "^4.3.1",
+    "peggy": "^4.2.0"
+  }
+}

tools/scripts/keyboard-abnf-tests/test/abnf-valid.test.mjs

@@ -0,0 +1,24 @@
+// Copyright (c) 2025 Unicode, Inc.
+// For terms of use, see http://www.unicode.org/copyright.html
+// SPDX-License-Identifier: Unicode-3.0
+
+import * as abnf from "abnf";
+import { test } from "node:test";
+import * as assert from "node:assert";
+import { forEachAbnf } from "./util.mjs";
+
+function check_refs(parsed) {
+  const errs = abnf.checkRefs(parsed);
+  if (!errs) return 0;
+  for (const err of errs) {
+    console.error(err);
+  }
+  return 3;
+}
+
+await forEachAbnf(async ({ abnfFile, abnfText, abnfPath }) => {
+  await test(`Test validity: ${abnfFile}`, async (t) => {
+    const parsed = await abnf.parseFile(abnfPath);
+    assert.equal(check_refs(parsed), 0);
+  });
+});

tools/scripts/keyboard-abnf-tests/test/datadriven.test.mjs

@@ -0,0 +1,70 @@
+// Copyright (c) 2025 Unicode, Inc.
+// For terms of use, see http://www.unicode.org/copyright.html
+// SPDX-License-Identifier: Unicode-3.0
+
+import * as abnf from "abnf";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { test } from "node:test";
+import { basename, join } from "node:path";
+import * as assert from "node:assert";
+import { forEachAbnf } from "./util.mjs";
+import peggy from "peggy";
+
+async function assertTest({ t, abnfPath, testText, expect }) {
+  const parsed = await abnf.parseFile(abnfPath);
+  const opts = {
+    grammarSource: abnfPath,
+    trace: false,
+  };
+  const text = parsed.toFormat({ format: "peggy" });
+  const parser = peggy.generate(text, opts);
+  for (const str of testText
+    .trim()
+    .split("\n")
+    .filter((l) => !/^#/.test(l))) {
+    await t.test(`"${str}"`, async (t) => {
+      const fn = () => parser.parse(str, opts);
+      if (!expect) {
+        assert.throws(fn, `Expected this expression to fail parsing`);
+      } else {
+        const results = fn();
+        assert.ok(results);
+      }
+    });
+  }
+}
+
+await forEachAbnf(async ({ abnfFile, abnfText, abnfPath }) => {
+  await test(`Test Data: ${abnfFile}`, async (t) => {
+    const stub = basename(abnfFile, ".abnf");
+    const testDir = `./${stub}.d`;
+    assert.ok(existsSync(testDir), `No test dir: ${testDir}`);
+    const tests = readdirSync(testDir)?.filter((f) =>
+      /^.*\.(pass|fail)\.txt/.test(f)
+    );
+    assert.ok(tests && tests.length, `No tests in ${testDir}`);
+    for (const testFile of tests) {
+      if (testFile.endsWith(".pass.txt")) {
+        await t.test(`${stub}/${testFile}`, async (t) => {
+          await assertTest({
+            t,
+            abnfPath,
+            testText: readFileSync(join(testDir, testFile), "utf-8"),
+            expect: true,
+          });
+        });
+      } else if (testFile.endsWith(".fail.txt")) {
+        await t.test(`${stub}/${testFile}`, async (t) => {
+          await assertTest({
+            t,
+            abnfPath,
+            testText: readFileSync(join(testDir, testFile), "utf-8"),
+            expect: false,
+          });
+        });
+      } else throw Error(`Unknown testFile ${testFile}`);
+    }
+    // const parsed = await abnf.parseFile(abnfPath);
+    // assert.equal(check_refs(parsed), 0);
+  });
+});

tools/scripts/keyboard-abnf-tests/test/util.mjs

@@ -0,0 +1,23 @@
+// Copyright (c) 2025 Unicode, Inc.
+// For terms of use, see http://www.unicode.org/copyright.html
+// SPDX-License-Identifier: Unicode-3.0
+
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+
+export const ABNF_DIR = "../../../keyboards/abnf";
+
+/**
+ *
+ * @param {function} callback  given abnfFile, abnfPath, abnfText
+ */
+export async function forEachAbnf(callback) {
+  return await Promise.all(
+    readdirSync(ABNF_DIR).map((abnfFile) => {
+      if (!/^.*\.abnf$/.test(abnfFile)) return;
+      const abnfPath = join(ABNF_DIR, abnfFile);
+      const abnfText = readFileSync(abnfPath);
+      return callback({ abnfFile, abnfPath, abnfText });
+    })
+  );
+}