Add review, query commands and pre-recorded answers support

.changeset/strict-pets-kneel.md

@@ -0,0 +1,5 @@
+---
+"@siteimprove/alfa-formatter-json": minor
+---
+
+**Changed:** The JSON-formatter now uses high verbosity when serializing the page and low when serializing the outcomes.

.changeset/wise-rocks-start.md

@@ -0,0 +1,5 @@
+---
+"@siteimprove/alfa-command": minor
+---
+
+**Added:** A new factory method `withSubcommandsAndArguments` is now available.

packages/alfa-cli/src/alfa.ts

@@ -12,6 +12,8 @@ import { Err } from "@siteimprove/alfa-result";
 import * as pkg from "./pkg.js";
 
 import audit from "./alfa/command/audit.js";
+import interview from "./alfa/command/interview.js";
+import query from "./alfa/command/query.js";
 import scrape from "./alfa/command/scrape.js";
 
 const {
@@ -31,9 +33,11 @@ const application = Command.withSubcommands(
   },
   (self) => ({
     audit: audit(self),
+    interview: interview(self),
+    query: query(self),
     scrape: scrape(self),
   }),
-  None
+  None,
 );
 
 application
@@ -54,7 +58,7 @@ application
       output = result.getErrUnsafe();
     }
 
-    output = output.trimRight();
+    output = output.trimEnd();
 
     if (output.length > 0) {
       stream.write(output + "\n");

packages/alfa-cli/src/alfa/command/audit/flags.ts

@@ -1,16 +1,26 @@
 import { Flag } from "@siteimprove/alfa-command";
 
+import { DEFAULT_ALFA_DIR } from "../common/alfa-dir.js";
 import * as scrape from "../scrape/flags.js";
 
 export const Flags = {
   ...scrape.Flags,
 
   help: Flag.help("Display the help information."),
 
+  alfaDir: Flag.string(
+    "alfa-dir",
+    `The directory to look for a pre-recorded answers file. If an answers file
+    is found it will be used to answer questions during the audit. Defaults to
+    "${DEFAULT_ALFA_DIR}".`,
+  )
+    .type("path")
+    .default(DEFAULT_ALFA_DIR),
+
   output: Flag.string(
     "output",
     `The path to write results to. If no path is provided, results are written
-    to stdout.`
+    to stdout.`,
   )
     .type("path")
     .alias("o")
@@ -19,7 +29,7 @@ export const Flags = {
   interviewer: Flag.string(
     "interviewer",
     `The interviewer to use for answering questions during the audit. If not
-    provided, questions will be left unanswered.`
+    provided, questions will be left unanswered.`,
   )
     .type("name or package")
     .alias("i")
@@ -28,13 +38,13 @@ export const Flags = {
   format: Flag.string("format", "The reporting format to use.")
     .type("name or package")
     .alias("f")
-    .default("earl"),
+    .default("json"),
 
   outcomes: Flag.string(
     "outcome",
     `The type of outcome to include in the results. If not provided, all types
     of outcomes are included. This flag can be repeated to include multiple
-    types of outcomes.`
+    types of outcomes.`,
   )
     .choices("passed", "failed", "inapplicable", "cantTell")
     .repeatable()
@@ -43,15 +53,15 @@ export const Flags = {
   cpuProfile: Flag.string(
     "cpu-profile",
     `The path to write a CPU profile of the audit to. If no path is provided,
-    no CPU profile is made.`
+    no CPU profile is made.`,
   )
     .type("path")
     .optional(),
 
   heapProfile: Flag.string(
     "heap-profile",
     `The path to write a heap profile of the audit to. If no path is provided,
-    no heap profile is made.`
+    no heap profile is made.`,
   )
     .type("path")
     .optional(),

packages/alfa-cli/src/alfa/command/audit/run.ts

@@ -7,11 +7,15 @@ import type { Command } from "@siteimprove/alfa-command";
 import { Formatter } from "@siteimprove/alfa-formatter";
 import { Interviewer } from "@siteimprove/alfa-interviewer";
 import { Iterable } from "@siteimprove/alfa-iterable";
-import { Option, None } from "@siteimprove/alfa-option";
+import { None, Option } from "@siteimprove/alfa-option";
 import type { Err } from "@siteimprove/alfa-result";
 import { Result } from "@siteimprove/alfa-result";
 import { Page } from "@siteimprove/alfa-web";
 
+// TODO: replace with experimental rules once published
+// import { experimentalRules, type Flattened } from "@siteimprove/alfa-rules";
+// const { R98, R101 } = experimentalRules;
+// const rules = [R98, R101] as Array<Flattened.Rule>;
 import rules from "@siteimprove/alfa-rules";
 
 import { Profiler } from "../../profiler.js";
@@ -21,6 +25,10 @@ import type { Flags } from "./flags.js";
 
 import * as scrape from "../scrape/run.js";
 
+import { answersPath } from "../common/alfa-dir.js";
+import { createAnsweringOracle } from "../common/answering-oracle.js";
+import { readAnswers } from "../common/question-store.js";
+
 export const run: Command.Runner<typeof Flags, typeof Arguments> = async ({
   flags,
   args: { url: target },
@@ -34,7 +42,7 @@ export const run: Command.Runner<typeof Flags, typeof Arguments> = async ({
   const interviewer = Option.from(
     await flags.interviewer
       .map((interviewer) => Interviewer.load<any, any, any, any>(interviewer))
-      .getOr(undefined)
+      .getOr(undefined),
   );
 
   if (interviewer.some((interviewer) => interviewer.isErr())) {
@@ -65,14 +73,28 @@ export const run: Command.Runner<typeof Flags, typeof Arguments> = async ({
     json = result.get();
   }
 
-  const page = Page.from(JSON.parse(json));
+  const pageResult = Page.from(JSON.parse(json));
+
+  if (pageResult.isErr()) {
+    return pageResult;
+  }
+  const page = pageResult.getUnsafe();
 
-  const oracle = interviewer
-    // The early .some ensured it is an Ok.
+  // If oracle is provided by --interviewer, use that, otherwise check if there are any pre-recorded answers.
+  let oracle = interviewer
     .map((interviewer) => interviewer.getUnsafe()(page, rules))
     .getOr(undefined);
 
-  const audit = Audit.of(page.getUnsafe(), rules, oracle);
+  if (oracle === undefined) {
+    const answersFilePath = answersPath(flags.alfaDir);
+    if (fs.existsSync(answersFilePath)) {
+      process.stderr.write(`Using answers from ${answersFilePath}\n`);
+      const answers = readAnswers(answersFilePath);
+      oracle = createAnsweringOracle(answers, page.document);
+    }
+  }
+
+  const audit = Audit.of(page, rules, oracle);
 
   for (const _ of flags.cpuProfile) {
     await Profiler.CPU.start();

packages/alfa-cli/src/alfa/command/common/alfa-dir.ts

@@ -0,0 +1,83 @@
+/// <reference types="node" />
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+
+/**
+ * @internal
+ */
+export const DEFAULT_ALFA_DIR = ".alfa";
+
+/**
+ * @internal
+ */
+export const QUESTIONS_FILE = "questions.json";
+
+/**
+ * @internal
+ */
+export const ANSWERS_FILE = "answers.json";
+
+/**
+ * @internal
+ */
+export const SESSION_FILE = "session.json";
+
+/**
+ * @internal
+ */
+export const SCRAPE_FILE = "scrape.json";
+
+/**
+ * Resolve the alfa directory to an absolute path.
+ *
+ * @internal
+ */
+export function resolveAlfaDir(alfaDir: string): string {
+  return path.resolve(alfaDir);
+}
+
+/**
+ * Get the path to the questions file within an alfa directory.
+ *
+ * @internal
+ */
+export function questionsPath(alfaDir: string): string {
+  return path.join(resolveAlfaDir(alfaDir), QUESTIONS_FILE);
+}
+
+/**
+ * Get the path to the answers file within an alfa directory.
+ *
+ * @internal
+ */
+export function answersPath(alfaDir: string): string {
+  return path.join(resolveAlfaDir(alfaDir), ANSWERS_FILE);
+}
+
+/**
+ * Get the path to the session file within an alfa directory.
+ *
+ * @internal
+ */
+export function sessionPath(alfaDir: string): string {
+  return path.join(resolveAlfaDir(alfaDir), SESSION_FILE);
+}
+
+/**
+ * Get the path to the cached scrape file within an alfa directory.
+ *
+ * @internal
+ */
+export function scrapePath(alfaDir: string): string {
+  return path.join(resolveAlfaDir(alfaDir), SCRAPE_FILE);
+}
+
+/**
+ * Create the alfa directory if it does not already exist.
+ *
+ * @internal
+ */
+export function ensureAlfaDir(alfaDir: string) {
+  return fs.mkdir(resolveAlfaDir(alfaDir), { recursive: true });
+}