sdk%ci: use comment syntax for doc splicing, add uv lockfile, use for dependency tracking, make CodeQL runner multi-lingual, add symlink linter (#33)

Author: kwvg

.github/scripts/depgraph.js

@@ -0,0 +1,295 @@
+/*!
+ * Copyright (c) 2026-present, The Dash Core developers
+ * SPDX-License-Identifier: MIT
+ * See the accompanying file LICENSE or https://opensource.org/license/MIT
+ */
+
+// @ts-check
+
+// Submits `uv.lock` to the dependency graph. GitHub currently natively parses
+// `Cargo.lock` but cannot parse `uv.lock`, this script parses it for submission
+// to the dependency graph.
+
+const fs = require("node:fs");
+
+// Submission tag, keyed to overwrite autogenerated results from `pyproject.toml`.
+const PY_MANIFEST_KEY = "pyproject.toml";
+
+// Identification of this script.
+const DETECTOR_PROFILE = {
+  name: "depgraph.js",
+  version: "1.0.0",
+  url: "https://github.com/dashpay/base-sdk",
+};
+
+// Matches `name[extras]==version`, capturing name in 1 and version in 2, ends at whitespace, marker or backslash.
+const RE_PIN = /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?==([^\s;\\]+)/;
+
+// Matches a distribution name, an extras suffix allowed, and nothing else.
+const RE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]*\])?$/;
+
+// Matches an unindented comment.
+const RE_HEADER = /^#/;
+
+// Matches an indented comment.
+const RE_OWNED = /^\s+#/;
+
+// Matches an indented `# via`, capturing what trails it, which may be empty.
+const RE_VIA = /^\s+#\s+via\b(.*)$/;
+
+// Matches an indented comment holding one token, captured.
+const RE_VIA_ITEM = /^\s+#\s+(\S.*)$/;
+
+/**
+ * PEP 503-style text normalisation.
+ *
+ * @param {string} name
+ * @returns {string}
+ */
+function normalise(name) {
+  return name.toLowerCase().replace(/[-_.]+/g, "-");
+}
+
+/**
+ * The package URL for a pinned distribution, local version encoded.
+ *
+ * @param {string} name normalised name
+ * @param {string} version
+ * @returns {string}
+ */
+function purlFor(name, version) {
+  return `pkg:pypi/${name}@${version.replace(/\+/g, "%2B")}`;
+}
+
+/**
+ * Parse a `via` entry.
+ *
+ * @param {string} entry
+ * @param {string} line the line it was read from, named in the error
+ * @returns {string} normalised name, extras dropped
+ */
+function viaName(entry, line) {
+  if (!RE_NAME.test(entry)) {
+    throw new Error(`unsupported \`via\` entry: ${line.trim()}`);
+  }
+  return normalise(entry.replace(/\[.*$/, ""));
+}
+
+/**
+ * Record *parent* against *pkg*, a marker repeat naming it only once.
+ *
+ * @param {{ via: string[] }} pkg
+ * @param {string} parent
+ */
+function addVia(pkg, parent) {
+  if (!pkg.via.includes(parent)) {
+    pkg.via.push(parent);
+  }
+}
+
+/**
+ * Parse `uv export --format requirements-txt --no-hashes` output.
+ *
+ * Two shapes are read, a pin and the `# via` beneath it holding a name or list.
+ *
+ * A resolution fork states one package once per marker, so an entry is keyed by
+ * name and version and a repeat merges its `via` into the entry already held.
+ *
+ * @param {string} text
+ * @returns {Map<string, { name: string, version: string, via: string[] }>}
+ */
+function parseExport(text) {
+  /** @type {Map<string, { name: string, version: string, via: string[] }>} */
+  const packages = new Map();
+  /** @type {{ name: string, version: string, via: string[] } | null} */
+  let current = null;
+  let listing = false;
+
+  for (const raw of text.split("\n")) {
+    const line = raw.replace(/\r$/, "");
+
+    if (line.trim() === "" || RE_HEADER.test(line)) {
+      current = null;
+      listing = false;
+      continue;
+    }
+
+    if (current !== null && RE_OWNED.test(line)) {
+      const via = RE_VIA.exec(line);
+      if (via) {
+        const rest = via[1].trim();
+        listing = rest === "";
+        if (!listing) {
+          addVia(current, viaName(rest, line));
+        }
+        continue;
+      }
+
+      const listed = RE_VIA_ITEM.exec(line);
+      if (listed && listing) {
+        addVia(current, viaName(listed[1].trim(), line));
+      }
+      continue;
+    }
+
+    // Extras are matched so they cannot hide a pin.
+    const pin = RE_PIN.exec(line);
+    if (pin === null) {
+      throw new Error(`unsupported requirement: ${line.trim()}`);
+    }
+
+    const name = normalise(pin[1]);
+    const key = `${name}@${pin[2]}`;
+    let held = packages.get(key);
+    if (held === undefined) {
+      held = { name, version: pin[2], via: [] };
+      packages.set(key, held);
+    }
+
+    current = held;
+    listing = false;
+  }
+
+  return packages;
+}
+
+/**
+ * Build the `resolved` map a snapshot carries, keyed and cross-referenced
+ * by the package URL.
+ *
+ * All entries are scoped `development`, since they make up the devshell.
+ *
+ * A fork can resolve one name to several versions and `via` names only the
+ * parent, so an edge is drawn to every version of it rather than guessed at.
+ *
+ * @param {Map<string, { name: string, version: string, via: string[] }>} packages
+ * @param {string} project normalised name of the workspace project
+ * @returns {Record<string, { package_url: string, relationship: string, scope: string, dependencies: string[] }>}
+ */
+function resolveGraph(packages, project) {
+  /** @type {Map<string, { name: string, version: string, via: string[] }[]>} */
+  const byName = new Map();
+  for (const pkg of packages.values()) {
+    const held = byName.get(pkg.name);
+    if (held === undefined) {
+      byName.set(pkg.name, [pkg]);
+    } else {
+      held.push(pkg);
+    }
+  }
+
+  /** @type {Record<string, { package_url: string, relationship: string, scope: string, dependencies: string[] }>} */
+  const resolved = {};
+
+  for (const pkg of packages.values()) {
+    if (pkg.via.length === 0) {
+      throw new Error(`${pkg.name} has no \`via\`; export --no-emit-project`);
+    }
+    for (const parent of pkg.via) {
+      if (parent !== project && !byName.has(parent)) {
+        throw new Error(
+          `${pkg.name} names ${parent}, not a pin nor ${project}`,
+        );
+      }
+    }
+
+    const purl = purlFor(pkg.name, pkg.version);
+    resolved[purl] = {
+      package_url: purl,
+      relationship: pkg.via.includes(project) ? "direct" : "indirect",
+      scope: "development",
+      dependencies: [],
+    };
+  }
+
+  // `via` names parents, a snapshot states children, so invert the edges.
+  for (const pkg of packages.values()) {
+    const child = purlFor(pkg.name, pkg.version);
+    for (const parent of pkg.via) {
+      for (const owner of byName.get(parent) ?? []) {
+        const deps = resolved[purlFor(owner.name, owner.version)].dependencies;
+        if (!deps.includes(child)) {
+          deps.push(child);
+        }
+      }
+    }
+  }
+
+  return resolved;
+}
+
+/**
+ * @param {{ sha: string, ref: string, resolved: Record<string, object> }} params
+ * @returns {object}
+ */
+function buildSnapshot({ sha, ref, resolved }) {
+  return {
+    version: 0,
+    job: {
+      id: process.env.GITHUB_RUN_ID,
+      correlator: `${process.env.GITHUB_WORKFLOW}-${process.env.GITHUB_JOB}`,
+    },
+    sha,
+    ref,
+    detector: DETECTOR_PROFILE,
+    scanned: new Date().toISOString(),
+    manifests: {
+      [PY_MANIFEST_KEY]: {
+        name: PY_MANIFEST_KEY,
+        file: { source_location: PY_MANIFEST_KEY },
+        resolved,
+      },
+    },
+  };
+}
+
+/**
+ * @param {object} params
+ * @param {ReturnType<typeof import("@actions/github").getOctokit>} params.github
+ * @param {typeof import("@actions/github").context} params.context
+ * @param {any} params.core
+ */
+module.exports = async ({ github, context, core }) => {
+  const source = process.env.REQUIREMENTS;
+  if (source === undefined) {
+    throw new Error("REQUIREMENTS names the export to submit; it is unset");
+  }
+
+  const project = process.env.PROJECT;
+  if (project === undefined) {
+    throw new Error("PROJECT names the workspace project; it is unset");
+  }
+
+  const packages = parseExport(fs.readFileSync(source, "utf8"));
+  if (packages.size === 0) {
+    throw new Error(`${source} states no pinned versions`);
+  }
+
+  const resolved = resolveGraph(packages, normalise(project));
+  const snapshot = buildSnapshot({
+    sha: context.sha,
+    ref: context.ref,
+    resolved,
+  });
+
+  const entries = Object.values(resolved);
+  const direct = entries.filter((e) => e.relationship === "direct").length;
+  core.info(`submitting ${entries.length} packages, ${direct} direct`);
+
+  const { data } = await github.request(
+    "POST /repos/{owner}/{repo}/dependency-graph/snapshots",
+    {
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      ...snapshot,
+    },
+  );
+  if (data.result === "INVALID") {
+    throw new Error(`snapshot refused: ${data.message}`);
+  }
+  core.info(`snapshot ${data.id}: ${data.message}`);
+};
+
+module.exports.parseExport = parseExport;
+module.exports.resolveGraph = resolveGraph;
+module.exports.buildSnapshot = buildSnapshot;

.github/workflows/build_msrv.yml

@@ -45,12 +45,22 @@ jobs:
           node-version: 24
 
       - name: Set up Python
+        id: python
         uses: actions/setup-python@v6
         with:
           python-version-file: pyproject.toml
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v10.0.1
+        with:
+          version: 0.12.9
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
       - name: Install Python dependencies
-        run: pip install ".[dev]"
+        run: |
+          uv sync --locked --extra dev --python '${{ steps.python.outputs.python-path }}'
+          echo "${PWD}/.venv/bin" >> "${GITHUB_PATH}"
 
       - name: Install CodeQL
         id: setup-codeql
@@ -82,22 +92,24 @@ jobs:
         uses: actions/cache@v5
         with:
           path: ~/.codeql
-          key: codeql-packs-${{ hashFiles('contrib/codeql/codeql-pack.lock.yml') }}
+          key: codeql-packs-${{ hashFiles('maint/codeql/*/codeql-pack.lock.yml') }}
 
       - name: Run linters
-        run: python3 contrib/lint_all.py --exclude lint_codeql
+        run: |
+          python3 maint/lint_all.py --exclude lint_codeql
+          python3 maint/lint/lint_codeql.py check
         env:
           RUSTUP_TOOLCHAIN: 1.85.0
 
       - name: Run CodeQL
-        run: python3 contrib/lint/lint_codeql.py --with-suite=rust-security-and-quality
+        run: python3 maint/lint/lint_codeql.py run --lang=rust --with-suite=rust-security-and-quality
         env:
           RUSTUP_TOOLCHAIN: 1.85.0
 
       - name: Check PR commit messages
         if: github.event_name == 'pull_request'
         run: >
-          python3 contrib/lint/lint_unconv.py
+          python3 maint/lint/lint_unconv.py
           -r "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
 
   build:

.github/workflows/build_nightly.yml

@@ -88,7 +88,7 @@ jobs:
 
       - name: Check formatting
         if: matrix.config.name == 'full'
-        run: python contrib/lint/lint_rust.py
+        run: python maint/lint/lint_rust.py
 
       - name: Test package (with coverage)
         if: matrix.config.name == 'full'

.github/workflows/pages.yml

@@ -39,12 +39,22 @@ jobs:
         run: cargo install wasm-pack@0.15.0
 
       - name: Set up Python
+        id: python
         uses: actions/setup-python@v6
         with:
           python-version-file: pyproject.toml
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v10.0.1
+        with:
+          version: 0.12.9
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
       - name: Install Python dependencies
-        run: pip install ".[dev]"
+        run: |
+          uv sync --locked --extra dev --python '${{ steps.python.outputs.python-path }}'
+          echo "${PWD}/.venv/bin" >> "${GITHUB_PATH}"
 
       - name: Test documentation tooling
         run: pytest