diff --git a/.github/workflows/validation-suite.yml b/.github/workflows/validation-suite.yml new file mode 100644 index 0000000..4787336 --- /dev/null +++ b/.github/workflows/validation-suite.yml @@ -0,0 +1,62 @@ +name: Validation Suite + +on: + schedule: + # Run quarterly: Jan, Apr, Jul, Oct on the 1st at 00:00 UTC + - cron: '0 0 1 1,4,7,10 *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + name: Run validation on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build glassbox binary + run: | + go build -o bin/glassbox ./cmd/glassbox + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install npm dependencies + run: npm ci --prefer-offline + + - name: Build TypeScript artifacts (if present) + run: | + npm run build --if-present || true + + - name: Make validation runner executable + run: | + chmod +x scripts/run-validation-suite.sh || true + + - name: Run validation suite + id: run_validation + env: + GLASSBOX_BINARY: ${{ github.workspace }}/bin/glassbox + ARTIFACT_MAX_MB: '50' + run: | + scripts/run-validation-suite.sh || true + + - name: Upload validation artifacts + uses: actions/upload-artifact@v4 + with: + name: validation-${{ matrix.os }}-${{ github.run_id }} + path: ci-artifacts/validation/ + retention-days: 30 diff --git a/VALIDATION_RUN.md b/VALIDATION_RUN.md new file mode 100644 index 0000000..d492612 --- /dev/null +++ b/VALIDATION_RUN.md @@ -0,0 +1,37 @@ +# Running the Validation Suite + +This file documents how to run the scheduled validation suite locally and in CI. + +Prerequisites (local) +- Go toolchain installed and on PATH (matched to `go.mod`). +- Node.js (v20 recommended) and `npm` available for TypeScript artifacts. +- Bash and `python3` available for the runner scripts. + +Local run + +```bash +# Build the Go binary +go build -o bin/glassbox ./cmd/glassbox + +# Install Node deps (if needed) and build TS artifacts +npm ci --prefer-offline +npm run build --if-present + +# Make scripts executable then run the validation runner +chmod +x scripts/run-validation-suite.sh scripts/generate-compat-report.sh +GLASSBOX_BINARY=./bin/glassbox scripts/run-validation-suite.sh + +# After a run, artifacts are written to ci-artifacts/validation/ +# You can aggregate compatibility reports across runs: +scripts/generate-compat-report.sh ci-artifacts/validation ci-artifacts/validation/compatibility_report.md +``` + +CI notes +- The GitHub workflow `.github/workflows/validation-suite.yml` runs the suite + quarterly on `ubuntu-latest` and `windows-latest`. Artifacts are uploaded + with a 30-day retention. The workflow will run in hosted runners that include + the required toolchains. + +Troubleshooting +- If `go` or `bash` are not available locally, install them or run the suite + inside a CI environment or container that matches the workflow (Ubuntu). diff --git a/docs/validation-suite.md b/docs/validation-suite.md new file mode 100644 index 0000000..f9b2c62 --- /dev/null +++ b/docs/validation-suite.md @@ -0,0 +1,62 @@ +# Validation Suite + +Purpose +- A scheduled, versioned validation suite exercising primary user journeys + using deterministic, sanitized fixtures. + +Goals +- Exercise transaction debug, replay/trace, profile export, session save, audit + signing surface, protocol registration, and release metadata verification. +- Run against at least Linux and one additional platform (Windows). +- Produce sanitized artifacts and a machine-readable summary so failures map to + the broken stage and include fingerprints for regression detection. + +Running locally + +Build the binary and run the suite locally: + +```bash +go build -o bin/glassbox ./cmd/glassbox +npm ci --prefer-offline +npm run build --if-present +chmod +x scripts/run-validation-suite.sh +GLASSBOX_BINARY=./bin/glassbox scripts/run-validation-suite.sh +``` + +CI Integration + +- A GitHub Actions workflow `.github/workflows/validation-suite.yml` runs the + suite quarterly on `ubuntu-latest` and `windows-latest` and uploads artifacts + under `validation--` with a 30-day retention. Artifacts are + redacted via `scripts/redact-logs.sh` before upload to remove test-only + secrets and PEM blocks. + +Reporting & Retention + +- Per-run artifacts are retained by GitHub Actions for 30 days (configured in + the workflow). The validation runner writes `ci-artifacts/validation/summary.json` + and `report.md` which are uploaded as part of the artifact bundle. + +- A helper script `scripts/generate-compat-report.sh` aggregates multiple + `summary.json` files into `compatibility_report.md` to support quarterly + compatibility reviews and follow-up issue tracking. + +- Long-term retention of specific failing-run artifacts should be performed by + maintainers downloading the artifact and storing it in an internal archive + if required for extended investigations. By default, the CI retention policy + is intentionally short to limit exposure of debugging data. + +Fixtures + +- Deterministic fixtures live under `test/validation/fixtures/` and must avoid + any real keys, hashes, or network credentials. Audit fixtures must use the + `testonly_` filename prefix. + +Acceptance checklist + +- Every journey has a deterministic fixture and documented prerequisites. +- Failures produce per-journey logs and a consolidated `report.md` describing + exit codes and fingerprints. +- The workflow runs quarterly and covers Linux + Windows. +- Artifacts are redacted via `scripts/redact-logs.sh` before upload and + retained per CI policy. diff --git a/scripts/generate-compat-report.sh b/scripts/generate-compat-report.sh new file mode 100644 index 0000000..6376c05 --- /dev/null +++ b/scripts/generate-compat-report.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Aggregate one or more validation summary.json files into a quarterly compatibility report. +# Usage: scripts/generate-compat-report.sh + +INPUT_DIR=${1:-ci-artifacts/validation} +OUTFILE=${2:-ci-artifacts/validation/compatibility_report.md} + +echo "Generating compatibility report from: $INPUT_DIR -> $OUTFILE" + +echo "# Quarterly Compatibility Report" > "$OUTFILE" +echo "Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$OUTFILE" +echo "" >> "$OUTFILE" + +for f in $(find "$INPUT_DIR" -name summary.json -print 2>/dev/null); do + echo "Processing $f" + echo "## Run: $f" >> "$OUTFILE" + python3 - <> "$OUTFILE" +import json,sys +f=sys.argv[1] +data=json.load(open(f)) +for e in data: + print('- %s: exit=%s fingerprint=%s' % (e.get('name'), e.get('exitcode'), e.get('fingerprint'))) +PY +done + +echo "Compatibility report written to $OUTFILE" diff --git a/scripts/run-validation-suite.sh b/scripts/run-validation-suite.sh new file mode 100644 index 0000000..92ac66f --- /dev/null +++ b/scripts/run-validation-suite.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Validation suite runner +# - Executes a small set of deterministic journeys against the built +# `glassbox` binary using fixtures in `test/validation/fixtures/`. +# - Produces `ci-artifacts/validation/summary.json`, per-journey logs, and +# `report.md` suitable for CI artifact upload. Runs redaction if available. + +GLASSBOX_BINARY=${GLASSBOX_BINARY:-bin/glassbox} +OUTDIR=${OUTDIR:-ci-artifacts/validation} +ARTIFACT_MAX_MB=${ARTIFACT_MAX_MB:-50} +mkdir -p "$OUTDIR" + +SUMMARY_JSON="$OUTDIR/summary.json" +echo '[]' > "$SUMMARY_JSON" + +run_cmd() { + name="$1"; shift + outfile="$OUTDIR/${name}.log" + echo "=== RUN: $name ($(date -u +%Y-%m-%dT%H:%M:%SZ)) ===" > "$outfile" + if "$@" >> "$outfile" 2>&1; then + code=0 + else + code=$? + fi + if command -v sha256sum >/dev/null 2>&1; then + fp=$(sha256sum "$outfile" | awk '{print $1}') + else + fp=$(shasum -a 256 "$outfile" | awk '{print $1}') + fi + + SUMMARY_JSON="$SUMMARY_JSON" \ + RUN_NAME="$name" RUN_CODE="$code" RUN_FP="$fp" RUN_OUT="$outfile" \ + python3 - <<'PY' +import os,json +f=os.environ['SUMMARY_JSON'] +try: + a=json.load(open(f)) +except Exception: + a=[] +a.append({ + 'name': os.environ['RUN_NAME'], + 'exitcode': int(os.environ['RUN_CODE']), + 'fingerprint': os.environ['RUN_FP'], + 'output': os.environ['RUN_OUT'], +}) +json.dump(a, open(f,'w'), indent=2) +PY +} + +echo "Validation runner: using binary: $GLASSBOX_BINARY" + +# Canonical tx hash from regression guide +CANONICAL_TX=5c0a1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab + +# 1) Transaction debug (dry-run path to validate inputs deterministically) +run_cmd debug "$GLASSBOX_BINARY" debug --dry-run --network testnet "$CANONICAL_TX" || true + +# 2) Trace command (load a deterministic trace fixture) +run_cmd trace "$GLASSBOX_BINARY" trace test/validation/fixtures/trace/sample.trace.json || true + +# 3) Profile: analyze a trace fixture and export JSON +run_cmd profile "$GLASSBOX_BINARY" profile test/validation/fixtures/trace/sample.trace.json --out-json "$OUTDIR/profile.json" || true + +# 4) Session save plan (dry run of session save to exercise save validations) +run_cmd session_plan "$GLASSBOX_BINARY" session save --plan || true + +# 5) Audit verify directory using test-only keys +run_cmd audit_verify_dir "$GLASSBOX_BINARY" audit:verify-dir --dir test/validation/fixtures/audit || true + +# 6) WASM local replay dry-run: validates local WASM replay path +run_cmd wasm_replay "$GLASSBOX_BINARY" debug --wasm test/validation/fixtures/sourcemap/placeholder.wasm --demo || true + +# 7) Protocol register dry-run (OS writes are not performed in dry-run) +run_cmd protocol_register "$GLASSBOX_BINARY" protocol:register --dry-run || true + +# 8) Release verification: get version metadata +run_cmd version "$GLASSBOX_BINARY" version --json || true + +# Run redaction if available before uploading +if [ -f scripts/redact-logs.sh ]; then + echo "Running redact-logs.sh on $OUTDIR" + bash scripts/redact-logs.sh "$OUTDIR" "$ARTIFACT_MAX_MB" || true +fi + +# Produce a human-readable markdown report from the JSON summary +REPORT_MD="$OUTDIR/report.md" +python3 - <<'PY' +import json,os,time +f=os.environ.get('SUMMARY_JSON','ci-artifacts/validation/summary.json') +try: + data=json.load(open(f)) +except Exception: + data=[] +out=[] +out.append('# Validation Suite Report') +out.append('Generated: %s' % time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())) +out.append('') +out.append('| Journey | Exit code | Fingerprint | Log |') +out.append('|---|---:|---|---|') +for e in data: + name=e.get('name') + code=e.get('exitcode') + fp=e.get('fingerprint') + logpath=e.get('output') + out.append('| %s | %s | %s | %s |' % (name, code, fp, logpath)) + +open('%s' % ('%s' % (os.path.join(os.path.dirname(f),'report.md'))),'w').write('\n'.join(out)) +print('Wrote report.md') +PY + +echo "Validation run complete. Artifacts in: $OUTDIR" +echo "Summary: $SUMMARY_JSON" + +exit 0 diff --git a/test/validation/README.md b/test/validation/README.md new file mode 100644 index 0000000..a26e1e4 --- /dev/null +++ b/test/validation/README.md @@ -0,0 +1,9 @@ +# Validation Fixtures + +Fixtures used by the scheduled validation suite live under `test/validation/fixtures/`. + +Rules +- Use deterministic, minimal data. +- Do not include real keys, hashes, or credentials. +- Audit fixtures must use `testonly_` semantics or filename prefixes and be + marked clearly at the top of the file. diff --git a/test/validation/fixtures/audit/testonly_key.pem b/test/validation/fixtures/audit/testonly_key.pem new file mode 100644 index 0000000..f9b6516 --- /dev/null +++ b/test/validation/fixtures/audit/testonly_key.pem @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +TESTONLY-PRIVATE-KEY-FOR-VALIDATION-SUITE +-----END PRIVATE KEY----- diff --git a/test/validation/fixtures/sourcemap/placeholder.wasm b/test/validation/fixtures/sourcemap/placeholder.wasm new file mode 100644 index 0000000..a9cf37b --- /dev/null +++ b/test/validation/fixtures/sourcemap/placeholder.wasm @@ -0,0 +1 @@ +;; placeholder binary for source-mapping validation