Update test runner.

.eslintrc.js

@@ -10,7 +10,9 @@ module.exports = {
     'digitalbazaar'
   ],
   ignorePatterns: [
+    'coverage/',
     'dist/',
+    'test-suites',
     'tests/webidl/WebIDLParser.js',
     'tests/webidl/idlharness.js',
     'tests/webidl/testharness.js'

CHANGELOG.md

@@ -1,13 +1,22 @@
 # jsonld ChangeLog
 
-## 8.1.2 - 2023-03-xx
+## 8.2.0 - 2023-03-xx
 
 ### Changed
 - Update for latest [rdf-canon][] changes: test suite location, README, links,
   and identifiers.
   - Skip test with 'U' escapes. Will enable when [rdf-canonize][] dependency is
     updated.
 - Test on Node.js 20.x.
+- Align test and benchmark code with [rdf-canonize][].
+  - **NOTE**: This changes various testing and benchmark runner features and
+    options.
+  - Update env var usage.
+  - Use more common code between Node.js and karma tests.
+  - Conditionally load test suites.
+  - Fix various minor bugs.
+  - Add multiple jobs benchmarking support.
+- Update benchmark compare script.
 
 ### Fixed
 - Improve safe mode for `@graph` use cases.

README.md

@@ -389,13 +389,13 @@ Node.js tests can be run with a simple command:
     npm test
 
 If you installed the test suites elsewhere, or wish to run other tests, use
-the `JSONLD_TESTS` environment var:
+the `TESTS` environment var:
 
-    JSONLD_TESTS="/tmp/org/test-suites /tmp/norm/tests" npm test
+    TESTS="/tmp/org/test-suites /tmp/norm/tests" npm test
 
 This feature can be used to run the older json-ld.org test suite:
 
-    JSONLD_TESTS=/tmp/json-ld.org/test-suite npm test
+    TESTS=/tmp/json-ld.org/test-suite npm test
 
 Browser testing can be done with Karma:
 
@@ -419,7 +419,7 @@ Remote context tests are also available:
     # run the context server in the background or another terminal
     node tests/remote-context-server.js
 
-    JSONLD_TESTS=`pwd`/tests npm test
+    TESTS=`pwd`/tests npm test
 
 To generate EARL reports:
 
@@ -432,7 +432,7 @@ To generate EARL reports:
 To generate an EARL report with the `json-ld-api` and `json-ld-framing` tests
 as used on the official [JSON-LD Processor Conformance][] page
 
-    JSONLD_TESTS="`pwd`/../json-ld-api/tests `pwd`/../json-ld-framing/tests" EARL="jsonld-js-earl.jsonld" npm test
+    TESTS="`pwd`/../json-ld-api/tests `pwd`/../json-ld-framing/tests" EARL="jsonld-js-earl.jsonld" npm test
 
 The EARL `.jsonld` output can be converted to `.ttl` using the [rdf][] tool:
 
@@ -449,14 +449,14 @@ Benchmarks
 Benchmarks can be created from any manifest that the test system supports.
 Use a command line with a test suite and a benchmark flag:
 
-    JSONLD_TESTS=/tmp/benchmark-manifest.jsonld JSONLD_BENCHMARK=1 npm test
+    TESTS=/tmp/benchmark-manifest.jsonld BENCHMARK=1 npm test
 
 EARL reports with benchmark data can be generated with an optional environment
 details:
 
-    JSONLD_TESTS=`pwd`/../json-ld.org/benchmarks/b001-manifiest.jsonld JSONLD_BENCHMARK=1 EARL=earl-test.jsonld TEST_ENV=1 npm test
+    TESTS=`pwd`/../json-ld.org/benchmarks/b001-manifiest.jsonld BENCHMARK=1 EARL=earl-test.jsonld TEST_ENV=1 npm test
 
-See `tests/test.js` for more `TEST_ENV` control and options.
+See `tests/test.js` for more `TEST_ENV` and `BENCHMARK` control and options.
 
 These reports can be compared with the `benchmarks/compare/` tool and at the
 [JSON-LD Benchmarks][] site.

benchmarks/compare/compare.js

@@ -27,7 +27,7 @@ yargs(hideBin(process.argv))
   })
   .option('env', {
     alias: 'e',
-    choices: ['none', 'all', 'combined'],
+    choices: ['none', 'all', 'present', 'combined'],
     default: 'none',
     description: 'Output environment format'
   })
@@ -50,18 +50,22 @@ async function compare({
     fn: f,
     content: await fs.readFile(f, 'utf8')
   })));
+  //console.log(contents);
   const results = contents
     .map(c => ({
       fn: c.fn,
       content: JSON.parse(c.content),
       // map of test id => assertion
       testMap: new Map()
     }))
+    .map(c => {
+      //console.log('C', c);
+      return c;
+    })
     .map(c => ({
       ...c,
-      // FIXME process properly
-      env: c.content['@included'][0],
-      label: c.content['@included'][0]['jldb:label']
+      env: c.content['@included']?.[0] || {},
+      label: c.content['@included']?.[0]?.['jldb:label']
     }));
   //console.log(JSON.stringify(results, null, 2));
   // order of tests found in each result set
@@ -96,14 +100,17 @@ async function compare({
         hz(r.testMap.get(t))))
       .map(d => relative ? d.toFixed(2) + '%' : d.toFixed(2))
   ]);
-  //console.log(compared);
-  //console.log(results);
+  //console.log('COMPARED', compared);
+  //console.log('RESULTS', results);
   const fnprefixlen = commonPathPrefix(file).length;
+  function label(res) {
+    return res.label || res.fn.slice(fnprefixlen);
+  }
   console.log('## Comparison');
   console.log(markdownTable([
     [
       'Test',
-      ...results.map(r => r.label || r.fn.slice(fnprefixlen))
+      ...results.map(label)
     ],
     ...compared
   ], {
@@ -130,15 +137,58 @@ async function compare({
     ['Comment', 'jldb:comment']
   ];
 
+  // show all properites
   if(env === 'all') {
     console.log();
     console.log('## Environment');
-    console.log(markdownTable([
-      envProps.map(p => p[0]),
-      ...results.map(r => envProps.map(p => r.env[p[1]] || ''))
+    //const data = results.map(r => envProps.map(p => {
+    //  return (p[1] === 'jldb:label') ? label(r) : r.env[p[1]] || '';
+    //}));
+    const data = results.map(r => [
+      label(r),
+      ...envProps.slice(1).map(p => r.env[p[1]] || '')
+    ]);
+    if(data.length > 0) {
+      console.log(markdownTable([
+        envProps.map(p => p[0]),
+        ...data
+      ]));
+    } else {
+      console.log('*not specified*');
+    }
+  }
+
+  // show present properites
+  if(env === 'present') {
+    console.log();
+    console.log('## Environment');
+    // get all data
+    const data = results.map(r => [
+      label(r),
+      ...envProps.slice(1).map(p => r.env[p[1]] || '')
+    ]);
+    // count present truthy fields per col
+    const propCounts = envProps.slice(1)
+      .map(p => results.reduce((c, r) => r.env[p[1]] ? ++c : c, 0));
+    const presentProps = [
+      envProps[0],
+      ...envProps.slice(1).filter((v, i) => propCounts[i] > 0)
+    ];
+    const presentData = data.map(d => ([
+      d[0],
+      ...d.slice(1).filter((v, i) => propCounts[i] > 0)
     ]));
+    if(data.length > 0) {
+      console.log(markdownTable([
+        presentProps.map(p => p[0]),
+        ...presentData
+      ]));
+    } else {
+      console.log('*not specified*');
+    }
   }
 
+  // show combined grouping of properties
   if(env === 'combined') {
     console.log();
     console.log('## Environment');
@@ -149,11 +199,16 @@ async function compare({
       );
       return [key, values.size ? [...values].join(', ') : []];
     }
-    console.log(markdownTable([
-      ['Key', 'Values'],
-      ...envProps
-        .map(p => envline(p[0], p[1]))
-        .filter(p => p[1].length)
-    ]));
+    const data = envProps
+      .map(p => envline(p[0], p[1]))
+      .filter(p => p[1].length);
+    if(data.length > 0) {
+      console.log(markdownTable([
+        ['Key', 'Values'],
+        ...data
+      ]));
+    } else {
+      console.log('*not specified*');
+    }
   }
 }

karma.conf.js

@@ -1,17 +1,12 @@
 /**
- * Karam configuration for jsonld.js.
+ * Karma configuration for jsonld.js.
  *
- * Set dirs, manifests, or js to run:
- *   JSONLD_TESTS="f1 f2 ..."
- * Output an EARL report:
- *   EARL=filename
- * Bail with tests fail:
- *   BAIL=true
+ * See ./test/test.js for env options.
  *
  * @author Dave Longley
  * @author David I. Lehn
  *
- * Copyright (c) 2011-2017 Digital Bazaar, Inc. All rights reserved.
+ * Copyright (c) 2011-2023 Digital Bazaar, Inc. All rights reserved.
  */
 const os = require('os');
 const webpack = require('webpack');
@@ -67,11 +62,10 @@ module.exports = function(config) {
       plugins: [
         new webpack.DefinePlugin({
           'process.env.BAIL': JSON.stringify(process.env.BAIL),
+          'process.env.BENCHMARK': JSON.stringify(process.env.BENCHMARK),
           'process.env.EARL': JSON.stringify(process.env.EARL),
+          'process.env.TESTS': JSON.stringify(process.env.TESTS),
           'process.env.TEST_ENV': JSON.stringify(process.env.TEST_ENV),
-          'process.env.JSONLD_BENCHMARK':
-            JSON.stringify(process.env.JSONLD_BENCHMARK),
-          'process.env.JSONLD_TESTS': JSON.stringify(process.env.JSONLD_TESTS),
           'process.env.TEST_ROOT_DIR': JSON.stringify(__dirname),
           'process.env.VERBOSE_SKIP': JSON.stringify(process.env.VERBOSE_SKIP),
           // for 'auto' test env
@@ -149,10 +143,10 @@ module.exports = function(config) {
         [
           'envify', {
             BAIL: process.env.BAIL,
+            BENCHMARK: process.env.BENCHMARK,
             EARL: process.env.EARL,
+            TESTS: process.env.TESTS,
             TEST_ENV: process.env.TEST_ENV,
-            JSONLD_BENCHMARK: process.env.JSONLD_BENCHMARK,
-            JSONLD_TESTS: process.env.JSONLD_TESTS,
             TEST_ROOT_DIR: __dirname,
             VERBOSE_SKIP: process.env.VERBOSE_SKIP,
             // for 'auto' test env

package.json

@@ -69,6 +69,7 @@
     "karma-sourcemap-loader": "^0.3.7",
     "karma-tap-reporter": "0.0.6",
     "karma-webpack": "^4.0.2",
+    "klona": "^2.0.5",
     "mocha": "^8.3.2",
     "mocha-lcov-reporter": "^1.3.0",
     "nyc": "^15.1.0",
@@ -99,7 +100,7 @@
     "fetch-json-ld-org-test-suite": "if [ ! -e test-suites/json-ld.org ]; then git clone --depth 1 https://github.com/json-ld/json-ld.org.git test-suites/json-ld.org; fi",
     "fetch-rdf-canon-test-suite": "if [ ! -e test-suites/rdf-canon ]; then git clone --depth 1 https://github.com/w3c/rdf-canon.git test-suites/rdf-canon; fi",
     "test": "npm run test-node",
-    "test-node": "cross-env NODE_ENV=test mocha --delay -t 30000 -A -R ${REPORTER:-spec} tests/test.js",
+    "test-node": "cross-env NODE_ENV=test mocha --delay -t 30000 -A -R ${REPORTER:-spec} tests/test-node.js",
     "test-karma": "cross-env NODE_ENV=test karma start",
     "coverage": "cross-env NODE_ENV=test nyc --reporter=lcov --reporter=text-summary npm test",
     "coverage-ci": "cross-env NODE_ENV=test nyc --reporter=lcovonly npm run test",