From 806c27174a698b946a2c1985c508de0d4c1e9200 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 13:49:09 -0700 Subject: [PATCH 01/28] feat(skills): ship copilot-otel-metrics as the opt-in telemetry path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capture stays off until the developer enables the editor setting, and the skill presents the settings diff before writing it. - fail-closed redaction allow-list across trace, metric, and log pipelines - carrier test measures what survives the allow-list instead of asserting it - STRIDE security model over seven trust boundaries with registered gaps - local, local-stack, org-distribution, and azure-capture modes Refs #2717 πŸ“Š - Generated by Copilot --- .../copilot-otel-metrics/SECURITY.md | 267 +++-- .../copilot-otel-metrics/SKILL.md | 37 +- .../copilot-otel-metrics/examples/README.md | 44 +- .../examples/_input_policy.py | 223 ++++ .../examples/azure/README.md | 72 +- .../examples/azure/deploy.sh | 17 +- .../examples/azure/main.bicep | 33 +- .../examples/azure/main.tf | 40 +- .../examples/azure/otel-collector-config.yaml | 71 +- .../examples/azure/outputs.tf | 2 +- .../examples/azure/variables.tf | 14 +- .../copilot-otel-metrics/examples/baseline.py | 39 +- .../examples/compose.yaml | 74 +- .../examples/inspect_metrics.py | 96 +- .../examples/otel-collector-local.yaml | 170 +++ .../examples/settings_upsert.py | 429 +++++++ .../examples/validate_dashboard.py | 354 ++++-- .../copilot-otel-metrics/examples/verify.py | 42 +- .../copilot-otel-metrics/pyproject.toml | 27 + .../references/azure-capture.md | 105 +- .../references/local-setup.md | 64 +- .../references/local-stack.md | 81 +- .../references/org-distribution.md | 5 +- .../references/verification.md | 17 +- .../tests/fuzz_harness.py | 104 ++ .../tests/test_azure_templates.py | 172 +++ .../tests/test_collector_carriers.py | 1011 +++++++++++++++++ .../tests/test_helpers.py | 512 +++++++++ .../tests/test_local_config.py | 580 ++++++++++ .../tests/test_settings_upsert.py | 383 +++++++ .../experimental/copilot-otel-metrics/uv.lock | 179 +++ 31 files changed, 4874 insertions(+), 390 deletions(-) create mode 100644 .github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml create mode 100644 .github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/pyproject.toml create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py create mode 100644 .github/skills/experimental/copilot-otel-metrics/uv.lock diff --git a/.github/skills/experimental/copilot-otel-metrics/SECURITY.md b/.github/skills/experimental/copilot-otel-metrics/SECURITY.md index aa674eb0c..94e149160 100644 --- a/.github/skills/experimental/copilot-otel-metrics/SECURITY.md +++ b/.github/skills/experimental/copilot-otel-metrics/SECURITY.md @@ -2,7 +2,7 @@ title: Copilot OTel Metrics Skill Security Model description: STRIDE threat model for the copilot-otel-metrics skill organized by assets, adversaries, and trust buckets (editor OTLP ingest, telemetry at rest, reference helper scripts, container image supply chain, editor-global configuration mutation, host process control, cloud control-plane artifact generation) with in-design mitigations and acknowledged enterprise readiness gaps author: microsoft/hve-core -ms.date: 2026-07-30 +ms.date: 2026-08-13 ms.topic: reference estimated_reading_time: 18 keywords: @@ -15,7 +15,7 @@ keywords: # Copilot OTel Metrics Skill Security Model -This document records the STRIDE threat model for the copilot-otel-metrics skill. The shipped runtime is `examples/compose.yaml` (the local stack definition), two Grafana dashboards under `examples/dashboards/`, four reference helper scripts under `examples/`, and the Azure templates under `examples/azure/` (collector configuration, Bicep, Terraform, and an Azure CLI script). The model is organized by trust bucket: editor OTLP ingest (B1), telemetry at rest and its query surfaces (B2), reference helper scripts to local service APIs (B3), container image supply chain (B4), editor-global configuration mutation (B5), host process control (B6), and cloud control-plane artifact generation (B7). Each bucket enumerates all six STRIDE categories. Assets and adversaries are enumerated first. Acknowledged enterprise readiness gaps are listed at the end. +This document records the STRIDE threat model for the copilot-otel-metrics skill. The shipped runtime is `examples/compose.yaml` (the local stack definition), `examples/otel-collector-local.yaml` (the local filtering pipeline), two Grafana dashboards under `examples/dashboards/`, five reference helper scripts and one shared policy module under `examples/`, and the Azure templates under `examples/azure/` (collector configuration, Bicep, Terraform, and an Azure CLI script). The model is organized by trust bucket: editor OTLP ingest (B1), telemetry at rest and its query surfaces (B2), reference helper scripts to local service APIs (B3), container image supply chain (B4), editor-global configuration mutation (B5), host process control (B6), and cloud control-plane artifact generation (B7). Each bucket enumerates all six STRIDE categories. Assets and adversaries are enumerated first. Acknowledged enterprise readiness gaps are listed at the end. The skill is an assistant rather than a reference pack, and that changes the model materially. It may write the user's global `settings.json` after presenting a diff and obtaining explicit approval, and it generates files intended for the user to execute. It never starts a service and never provisions infrastructure: `docker compose`, `az deployment`, and `terraform apply` are handed to the user, not run. Buckets B5 through B7 exist because generating and writing are themselves exposures, independent of who runs the result. @@ -25,22 +25,26 @@ The skill is an assistant rather than a reference pack, and that changes the mod The copilot-otel-metrics skill helps an operator turn on GitHub Copilot Chat's OTLP export and stand up somewhere for the data to land, locally or in Azure. Its highest-risk property is **not the code, it is the payload**. Spans emitted by the extension were directly observed carrying full prompt text, tool call arguments and results, and system instructions, on a configuration where content capture was left at its default. Anyone who follows this skill therefore accumulates a durable corpus of prompt content, in a Docker volume locally or in a billed Log Analytics workspace for an organization. -That exposure is **not harmless and is not accepted silently**. It is rated Medium residual for local single-user capture and is the reason organization capture ships with a collector processor that deletes the six observed content attributes before they reach billable storage. The skill states the same position everywhere it appears: content reaches the store regardless of the `captureContent` setting, so the endpoint and the backing store are sensitive either way, and the user owns the decision. +That exposure is **not harmless and is not accepted silently**. Both capture paths filter before storage. The local path routes every export through an OpenTelemetry Collector whose attribute policy is a fail-closed allow-list, so an attribute this skill has never seen is dropped rather than stored; the organization path deletes the seven observed content attributes before they reach billable storage. The direction matters: a delete-list removes the content attributes someone already knows about and passes through the one a future extension release adds. What filtering cannot change is what the extension emits, so content still leaves the editor and still crosses the loopback hop in plaintext. The skill states the same position everywhere it appears: the endpoint is sensitive regardless of the `captureContent` setting, and the user owns the decision. -The design bounds the local case by construction: every published port binds to `127.0.0.1`, the stack image tag is pinned, the documented settings block omits `captureContent`, and the skill supplies a verification command so a reader checks rather than assumes. Residual risk concentrates in three places the skill cannot close: the extension's own span content behavior, the absence of authentication on loopback service APIs, and the lack of encryption or expiry on the persistent volume. +**How far the local filtering reaches is measured, not asserted.** `tests/test_collector_carriers.py` starts the pinned Collector with the shipped configuration, sends payloads placing a distinct marker in each of 28 OTLP carriers, and records what survives. Every claim about local minimization in this document is that module's output. Its method matters as much as its result: a paired run with the content processors removed establishes that the instrument would have rendered each marker had it survived, so a carrier is never called governed merely because its marker was absent. The map is asserted, so a configuration or image change that opens a carrier fails the suite. -The assistant behaviors add three bounded exposures. The settings write mutates a user-owned JSONC file, which is contained by a mandatory backup, a per-key upsert that never reserializes, an approved diff, and a post-write parse with automatic restore. Generated compose and IaC files are inert until the user runs them, and the agent is prohibited from running them. Organization capture places a shared write-side credential on every workstation with no per-user binding and no documented in-place rotation, which is the single largest new exposure and is the reason the Azure path leads with that fact rather than with the architecture. +That measurement corrected this document. The allow-list governs attributes at every level and a map-valued log body; it does **not** reach span names, span status messages, span event names, trace state, non-map log bodies, severity text, log event names, metric metadata, span link attributes, or metric exemplar attributes. A content-scrub processor now closes the ones no shipped consumer reads. Three categories remain open and are registered as gaps rather than described as filtered: span names and metric metadata, because dashboard queries read them; span links and exemplars, because this Collector distribution provides no way to reach them; and the instrumentation scope and schema fields, which are left open on an expectation about emitter behavior rather than on a control. + +The design bounds the local case by construction: a filtering Collector is the only OTLP listener published on the host, its attribute policy is a fail-closed allow-list rather than a delete-list, every published port binds to `127.0.0.1`, both images are pinned by manifest digest, Grafana requires an operator-supplied credential with anonymous access disabled, the documented settings block omits `captureContent`, and the skill supplies a verification command so a reader checks rather than assumes. Residual risk concentrates in four places the skill cannot close: the extension's own span content behavior before the Collector sees it, the carriers the Collector governs incompletely or not at all, the absence of authentication on the Prometheus and Tempo query APIs, and the lack of encryption on the persistent volume. + +The assistant behaviors add three bounded exposures. The settings write mutates a user-owned JSONC file, which is contained by an executable that enforces a verified schema, splices only the target value span, backs up before writing without overwriting an earlier backup, restores on a parse failure, and refuses to write when unrelated settings would change. Generated compose and IaC files are inert until the user runs them, and the agent is prohibited from running them. Organization capture places a shared write-side credential on every workstation with no per-user binding and no documented in-place rotation, which is the single largest new exposure and is the reason the Azure path leads with that fact rather than with the architecture. ### Security Posture Overview -| Dimension | Value | -|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Runtime surface | Local compose stack, two Grafana dashboards, four standard-library Python helpers, collector configuration, Bicep, Terraform, and an Azure CLI script | -| Trust buckets | B1 editor OTLP ingest, B2 telemetry at rest, B3 helper scripts, B4 image supply chain, B5 editor-global configuration mutation, B6 host process control, B7 cloud control-plane artifact generation | -| Credentials | Grafana default `admin`/`admin` for the local stack; an Application Insights connection string for the Azure path, supplied by the operator and never written by the skill | -| Network egress | None locally after the image pull. The Azure path sends telemetry to a collector the operator runs, which forwards to Application Insights over TLS | -| Agent execution | Writes the user's global `settings.json` after an approved diff; writes generated artifacts to disk. Never starts a service and never provisions infrastructure | -| Open residual gaps | 16 registered, 5 open (highest: InfoDisc-High, prompt content present in spans despite the documented default) | +| Dimension | Value | +|--------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Runtime surface | Local compose stack with a filtering Collector, two Grafana dashboards, five standard-library Python helpers plus one shared policy module, fleet collector configuration, Bicep, Terraform, and an Azure CLI script | +| Trust buckets | B1 editor OTLP ingest, B2 telemetry at rest, B3 helper scripts, B4 image supply chain, B5 editor-global configuration mutation, B6 host process control, B7 cloud control-plane artifact generation | +| Credentials | Grafana admin credentials supplied by the operator through required environment variables, with no default and no committed value; an Application Insights connection string and a fleet ingest token for the Azure path, both operator-supplied and never written by the skill | +| Network egress | None locally after the image pull. The Azure path sends telemetry to a collector the operator runs, which forwards to Application Insights over TLS | +| Agent execution | Writes the user's global `settings.json` after an approved diff; writes generated artifacts to disk. Never starts a service and never provisions infrastructure | +| Open residual gaps | 25 registered, 10 open (highest: InfoDisc-High, prompt content present in spans before the Collector filters it) | ## Contents @@ -62,16 +66,20 @@ The assistant behaviors add three bounded exposures. The settings write mutates ### Components -1. `examples/compose.yaml` β€” declares one `grafana/otel-lgtm` container publishing five loopback-bound ports, mounting an external named volume at `/data`, and passing delta-to-cumulative conversion plus 120-day retention to Prometheus. -2. `examples/dashboards/copilot-otel.json` β€” Grafana dashboard referencing the pre-provisioned `prometheus` and `tempo` datasource uids. Contains PromQL and TraceQL queries only. -3. `examples/verify.py` β€” read-only. Queries Grafana, Prometheus, and Tempo health plus stored signal presence. Exits non-zero when the stack is unhealthy. -4. `examples/baseline.py` β€” read-mostly. Snapshots Prometheus label values and Tempo trace names, writes one JSON file under the user cache directory, and diffs a later store state against it. -5. `examples/inspect_metrics.py` β€” read-only. Enumerates `copilot_chat` and `gen_ai` series with labels and current values. -6. `examples/validate_dashboard.py` β€” the only writing helper. Imports a dashboard through the Grafana API with `overwrite: true`, then replays each panel query against Prometheus or Tempo. Endpoint and credentials come from the environment, and it refuses a non-loopback Grafana unless `COPILOT_OTEL_ALLOW_REMOTE=1` is set. -7. `examples/dashboards/copilot-otel-azure.json` β€” Grafana dashboard for the Azure path. Contains KQL queries against Log Analytics only. -8. `examples/azure/otel-collector-config.yaml` β€” OpenTelemetry Collector pipeline. Receives OTLP, deletes the six observed plaintext content attributes plus `copilot_chat.reasoning_content` defensively, and exports to Application Insights using a connection string read from the environment. -9. `examples/azure/main.bicep`, `main.tf`, `variables.tf`, `outputs.tf`, `versions.tf`, `deploy.sh` β€” templates that create a Log Analytics workspace, an Application Insights component, an Azure Monitor dashboard, and an optional `Monitoring Reader` role assignment. Inert until an operator deploys them. -10. `SKILL.md` and `references/` β€” the instructional surface. It authorizes exactly two write behaviors: a diff-approved per-key upsert into the user's global `settings.json`, and writing generated artifacts to disk. +1. `examples/compose.yaml` β€” declares a Collector container publishing the only loopback-bound OTLP ports and one `grafana/otel-lgtm` container with no host OTLP mapping, joined by an explicit bridge network, mounting an external named volume at `/data`, requiring operator-supplied Grafana credentials with anonymous access disabled, and passing delta-to-cumulative conversion plus 120-day retention to Prometheus. Both images are digest-pinned. +2. `examples/otel-collector-local.yaml` β€” the local Collector pipeline. Applies a fail-closed `redaction` allow-list across the trace, metric, and log pipelines, masks recognizable credential shapes in surviving values, scrubs the content carriers the allow-list cannot reach, and exports only to the LGTM service over the internal network. What each of its controls actually reaches is recorded by `tests/test_collector_carriers.py` rather than inferred from the configuration. +3. `examples/dashboards/copilot-otel.json` β€” Grafana dashboard referencing the pre-provisioned `prometheus` and `tempo` datasource uids. Contains PromQL and TraceQL queries only. +4. `examples/_input_policy.py` β€” shared, not run directly. Centralizes endpoint scheme, authority, port, remote opt-in, redirect revalidation, path containment, and required-credential checks for the helpers that take configurable input. +5. `examples/verify.py` β€” read-only. Queries Grafana, Prometheus, and Tempo health plus stored signal presence. Exits non-zero when the stack is unhealthy. Takes no configurable endpoint. +6. `examples/baseline.py` β€” read-mostly. Snapshots Prometheus label values and Tempo trace names, writes one JSON file contained to the user cache root, and diffs a later store state against it. +7. `examples/inspect_metrics.py` β€” read-only. Enumerates `copilot_chat` and `gen_ai` series with labels and current values. Takes no configurable endpoint. +8. `examples/validate_dashboard.py` β€” imports a dashboard through the Grafana API with `overwrite: true`, then replays each panel query against Prometheus or Tempo. Every configurable endpoint and the dashboard path pass the shared policy first, and Grafana credentials are required with no default. +9. `examples/settings_upsert.py` β€” the settings mutation executable. Enforces a schema verified against a named extension build, splices only target value spans, backs up, restores on failure, and appends a redacted audit record. +10. `examples/dashboards/copilot-otel-azure.json` β€” Grafana dashboard for the Azure path. Contains KQL queries against Log Analytics only. +11. `examples/azure/otel-collector-config.yaml` β€” fleet OpenTelemetry Collector pipeline. Requires active bearer authentication and TLS material from the environment, deletes the seven observed plaintext content attributes, labels the deployment environment, and exports to Application Insights using a connection string read from the environment. +12. `examples/azure/main.bicep`, `main.tf`, `variables.tf`, `outputs.tf`, `versions.tf`, `deploy.sh` β€” templates that create a per-environment Log Analytics workspace, Application Insights component, Azure Monitor dashboard, and an optional workspace-scoped `Monitoring Reader` role assignment. Inert until an operator deploys them. +13. `tests/` β€” static and behavioral tests over the local configuration, the shared policy, the settings executable, and the Azure templates, plus a fuzz harness. They parse committed files and temporary fixtures; they start no container and contact no cloud. +14. `SKILL.md` and `references/` β€” the instructional surface. It authorizes exactly two write behaviors: a diff-approved per-key upsert into the user's global `settings.json`, and writing generated artifacts to disk. ### Data Flow @@ -90,11 +98,12 @@ flowchart TD HELPERS["verify.py / baseline.py
inspect_metrics.py / validate_dashboard.py"] SNAP["~/.cache/copilot-otel/
pre-enable-baseline.json"] end - subgraph STACK["otel-lgtm container (trust zone)"] - OTLP["OTLP receiver
:4318 HTTP / :4317 gRPC"] + subgraph STACK["Local stack (trust zone)"] + LCOLL["OTel Collector
:4318 / :4317 loopback
fail-closed allow-list"] + OTLP["LGTM OTLP receiver
internal network only"] PROM["Prometheus :9090"] TEMPO["Tempo :3200"] - GRAF["Grafana :3000
(default credentials)"] + GRAF["Grafana :3000
(credential required)"] VOL[("copilot-otel-data
volume /data")] end subgraph REGISTRY["Public container registry (external)"] @@ -112,7 +121,8 @@ flowchart TD GEN -.->|"written to disk, never executed by the agent"| HOST GEN -.->|"deployed by the operator"| AZURE SETTINGS -->|"configures endpoint"| EXT - EXT -->|"OTLP/HTTP plaintext, spans carrying prompt content"| OTLP + EXT -->|"OTLP/HTTP plaintext, spans carrying prompt content"| LCOLL + LCOLL -->|"allow-listed attributes only, internal network"| OTLP EXT -->|"OTLP over TLS, fleet path"| COLL OTLP -->|"metrics"| PROM OTLP -->|"traces"| TEMPO @@ -127,7 +137,7 @@ flowchart TD COLL -->|"connection string, content attributes stripped"| AI AI -->|"persists"| LAW AMD -->|"KQL, current-user auth"| LAW - IMG -.->|"docker pull, tag-pinned"| STACK + IMG -.->|"docker pull, digest-pinned"| STACK ``` ## Trust Boundaries @@ -153,21 +163,26 @@ flowchart TD β”‚ β”‚ prompt-bearing spans β”‚ queries β”‚ β”‚ ─────────┼─────────────────────────────────┼─────────────── β”‚ β”‚ β”‚ 127.0.0.1 ONLY (no off-host listener) β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ OTel Collector β€” the only published OTLP listener β”‚ β”‚ +β”‚ β”‚ fail-closed allow-list; unknown attributes dropped β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ internal network, no host mapping β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ TRUST BOUNDARY: otel-lgtm container β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ β”‚ β”‚ OTLP recvβ”‚ β”‚Prometheusβ”‚ β”‚ Tempo β”‚ β”‚ Grafana β”‚ β”‚ β”‚ -β”‚ β”‚ β”‚ :4317/18 β”‚ β”‚ :9090 β”‚ β”‚ :3200 β”‚ β”‚ :3000 admin β”‚ β”‚ β”‚ +β”‚ β”‚ β”‚ internal β”‚ β”‚ :9090 β”‚ β”‚ :3200 β”‚ β”‚ :3000 auth'd β”‚ β”‚ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β”‚ β”‚ persists β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ β”‚ β”‚ copilot-otel-data vol β”‚ prompt content at rest β”‚ β”‚ -β”‚ β”‚ β”‚ (unencrypted, 120d) β”‚ no expiry on traces β”‚ β”‚ +β”‚ β”‚ β”‚ (unencrypted, 120d) β”‚ traces expire at 336h β”‚ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ image pull (tag-pinned, not digest-pinned) + β”‚ image pull (digest-pinned) β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ TRUST BOUNDARY: public β”‚ β”‚ container registry β”‚ @@ -185,40 +200,40 @@ flowchart TD ### Boundary Descriptions -| Boundary | Assets Protected | Controls Enforced | -|----------------------|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Copilot agent | User configuration, host process state, cloud spend | Mandatory backup, per-key upsert, approved diff, post-write parse with restore; generation-only boundary on Docker and infrastructure commands | -| Operator workstation | Prompt content in transit, snapshot file | Loopback-only port publishing; `captureContent` omitted from the documented settings block; stdlib-only helpers | -| otel-lgtm container | Stored metrics and traces, Grafana configuration | Container isolation; single named volume; default credentials paired with no off-host listener | -| Public registry | Image integrity, stack availability | Tag-pinned image reference; no build step and no third-party plugin installation | -| Azure subscription | Fleet telemetry, ingestion spend, ingest credential | Collector strips content attributes before ingestion; connection string supplied from the operator's secret store and never written by the skill; daily ingestion cap defaulted in every template | +| Boundary | Assets Protected | Controls Enforced | +|----------------------|-----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Copilot agent | User configuration, host process state, cloud spend | Mandatory backup, per-key upsert, approved diff, post-write parse with restore; generation-only boundary on Docker and infrastructure commands | +| Operator workstation | Prompt content in transit, snapshot file | Loopback-only port publishing; `captureContent` omitted from the documented settings block; stdlib-only helpers | +| otel-lgtm container | Stored metrics and traces, Grafana configuration | Container isolation; single named volume; no host OTLP mapping, so the Collector allow-list cannot be bypassed by a local process; anonymous Grafana access disabled and admin credentials required from the environment | +| Public registry | Image integrity, stack availability | Both images pinned by multi-architecture manifest digest; no build step and no third-party plugin installation | +| Azure subscription | Fleet telemetry, ingestion spend, ingest credential | Collector strips content attributes before ingestion; connection string supplied from the operator's secret store and never written by the skill; daily ingestion cap defaulted in every template | ## Assets -| Id | Asset | Lifetime | Notes | -|----|----------------------------------------|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------| -| A1 | Prompt and tool-call content in spans | Persisted in the volume | Observed present despite content capture being left at its default. Highest-value asset here. | -| A2 | Usage and cost metrics | Persisted, 120-day retention | Token counts, AIU billing proxy, tool call counts. Commercially sensitive in aggregate. | -| A3 | `copilot-otel-data` Docker volume | Persistent until explicitly removed | Unencrypted at rest. Survives `docker compose down` by design. | -| A4 | Grafana instance and dashboards | Persistent | Default `admin`/`admin` credentials; reachable on loopback only. | -| A5 | Baseline snapshot file | Persistent under the user cache | Contains metric and service names plus session ids, not content. | -| A6 | Stack container image | External, pulled on first run | `grafana/otel-lgtm:0.29.2`, tag-pinned rather than digest-pinned. | -| A7 | Global `settings.json` | Persistent, user-owned | JSONC with the user's own comments and formatting. Resolves from the default profile regardless of the active profile. | -| A8 | Application Insights connection string | Persistent until the component is recreated | Fleet-wide write credential. Supplied by the operator, never written into a generated file. | -| A9 | Generated infrastructure templates | Persistent in the user's workspace | Inert until deployed. Deployment creates billable resources and an optional role assignment. | +| Id | Asset | Lifetime | Notes | +|----|----------------------------------------|---------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| +| A1 | Prompt and tool-call content in spans | Persisted in the volume | Observed present despite content capture being left at its default. Highest-value asset here. | +| A2 | Usage and cost metrics | Persisted, 120-day retention | Token counts, AIU billing proxy, tool call counts. Commercially sensitive in aggregate. | +| A3 | `copilot-otel-data` Docker volume | Persistent until explicitly removed | Unencrypted at rest. Survives `docker compose down` by design. | +| A4 | Grafana instance and dashboards | Persistent | Anonymous access disabled; admin credentials required from the operator's environment. Reachable on loopback only. | +| A5 | Baseline snapshot file | Persistent under the user cache | Contains metric and service names plus session ids, not content. | +| A6 | Stack container images | External, pulled on first run | `grafana/otel-lgtm` and `otel/opentelemetry-collector-contrib`, both pinned by manifest digest. | +| A7 | Global `settings.json` | Persistent, user-owned | JSONC with the user's own comments and formatting. Which file resolves depends on the scope the installed build declares. | +| A8 | Application Insights connection string | Persistent until the component is recreated | Fleet-wide write credential. Supplied by the operator, never written into a generated file. | +| A9 | Generated infrastructure templates | Persistent in the user's workspace | Inert until deployed. Deployment creates billable resources and an optional role assignment. | ## Adversaries -| Id | Adversary | In-scope mitigations | -|-------|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ADV-a | Off-host network attacker | Every published port binds `127.0.0.1`, so no listener is reachable off the host. Default Grafana credentials are never exposed to a network. | -| ADV-b | Malicious or compromised process on the same host | Not mitigated. Loopback services are unauthenticated and any local process can read or write them (G-SPF-1). | -| ADV-c | Another user on a shared workstation | Bounded by Docker socket access and filesystem permissions on the volume. Not otherwise mitigated (G-INF-2). | -| ADV-d | Upstream image or registry compromise | Tag pinning limits drift; no digest verification or signature check is performed (G-SUP-1). | -| ADV-e | Operator error redirecting the exporter off-host | Documentation states the endpoint carries prompt content and must be treated as sensitive regardless of the capture setting. | -| ADV-f | The agent itself, writing the wrong thing | Backup before write, per-key upsert that never reserializes, exact diff, explicit approval, post-write parse with automatic restore (G-TAM-2). | -| ADV-g | A fleet member misusing the shared ingest credential | Not mitigated. The credential is write-side only and identical on every workstation, with no per-user binding and no in-place rotation (G-INF-3, G-DOS-2). | -| ADV-h | An operator deploying generated templates carelessly | Every template requires named inputs with no defaults for subscription, region, and naming; the role assignment is opt-in; a daily ingestion cap is set by default (G-EOP-3). | +| Id | Adversary | In-scope mitigations | +|-------|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ADV-a | Off-host network attacker | Every published port binds `127.0.0.1`, so no listener is reachable off the host. Default Grafana credentials are never exposed to a network. | +| ADV-b | Malicious or compromised process on the same host | Not mitigated. Loopback services are unauthenticated and any local process can read or write them (G-SPF-1). | +| ADV-c | Another user on a shared workstation | Bounded by Docker socket access and filesystem permissions on the volume. Not otherwise mitigated (G-INF-2). | +| ADV-d | Upstream image or registry compromise | Tag pinning limits drift; no digest verification or signature check is performed (G-SUP-1). | +| ADV-e | Operator error redirecting the exporter off-host | Documentation states the endpoint carries prompt content and must be treated as sensitive regardless of the capture setting. | +| ADV-f | The agent itself, writing the wrong thing | Backup before write, per-key upsert that never reserializes, exact diff, explicit approval, post-write parse with automatic restore (G-TAM-2). | +| ADV-g | A fleet member misusing the shared ingest credential | Not mitigated. The credential is write-side only and identical on every workstation, with no per-user binding and no in-place rotation (G-INF-3, G-DOS-2). A workstation relay narrows where the credential is readable without binding it to a person (G-INF-7). | +| ADV-h | An operator deploying generated templates carelessly | Every template requires named inputs with no defaults for subscription, region, and naming; the role assignment is opt-in; a daily ingestion cap is set by default (G-EOP-3). | ## Bucket B1: Editor OTLP ingest @@ -234,12 +249,34 @@ An unauthenticated writer can also poison existing series by submitting conflict ### Repudiation -The receiver records no provenance for accepted payloads beyond the resource attributes the sender chooses to supply, so a submitting process cannot be identified after ingest. `service_version` and `session_id` label values are attacker-controlled in the injection case. `baseline.py` diffing provides a coarse before-and-after record rather than per-payload attribution. +The receiver records no provenance for accepted payloads beyond the resource attributes the sender chooses to supply, so a submitting process cannot be identified after ingest. `service_version` and `session_id` label values are attacker-controlled in the injection case. `baseline.py` diffing provides a coarse before-and-after record rather than per-payload attribution, and it depends on `session.id` surviving the allow-list: the attribute is allow-listed for exactly that reason, and the runtime harness asserts per key that the running Collector keeps it. ### Information Disclosure This is the material risk in the entire model. Spans emitted with content capture left at its documented default were directly observed carrying `copilot_chat.user_request` (full prompt text), `gen_ai.input.messages`, `gen_ai.output.messages`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, and `gen_ai.system_instructions` in plaintext. A seventh, `copilot_chat.reasoning_content`, was present but marked `[encrypted]`. The transport is plaintext HTTP. On loopback this is contained; the moment `otlpEndpoint` is redirected to a shared or hosted collector, prompt content leaves the machine in clear text. The skill documents this discrepancy explicitly and supplies a `curl` check so a reader verifies rather than trusts the documented default. Tracked as G-INF-1 and G-TLS-1. +What the Collector does about it downstream is measured by `tests/test_collector_carriers.py` against the pinned image, with a paired control run establishing that each marker is renderable before any of them is called governed: + +| Carrier | Handling | Mechanism | +|--------------------------------------------------------------------------------|------------------------------------|------------------------------| +| Attributes on resource, scope, span, span event, log record, metric datapoint | Dropped unless allow-listed | `redaction` | +| Map-valued log body | Reduced to the allow-listed subset | `redaction` | +| An allow-listed value matching a credential shape | Replaced with `****` | `redaction` `blocked_values` | +| Span status message, span event name, log severity text, log record event name | Replaced with `[redacted]` | `transform/scrub` | +| Log body β€” scalar, array, and bytes | Replaced with `[redacted]` | `transform/scrub` | +| Span trace state | Cleared | `transform/scrub` | +| **Span name** | **Passes through** | none β€” G-INF-8 | +| **Metric name, description, unit** | **Passes through** | none β€” G-INF-8 | +| **Span link attributes and link trace state** | **Passes through** | none β€” G-INF-9 | +| **Metric exemplar filtered attributes** | **Passes through** | none β€” G-INF-9 | +| Instrumentation scope name and version, resource and scope schema URLs | **Pass through** | none β€” G-INF-11 | + +The open rows differ in kind, and the difference decides whether any of them can be closed. Span names and metric metadata are preserved **by choice**: three dashboard TraceQL queries match on span name, `baseline.py` reads Tempo's `rootTraceName` as its injection discriminator, and every Prometheus query in the shipped dashboard selects by metric name. Span name is the more serious of the two, because a span name is emitter-composed and can carry request text. Span links and metric exemplars are open **by capability**: verified against the pinned image, OTTL has no `spanlink` context and refuses to index `links`, and assigning to `datapoint.exemplars` parses without effect, so no processor in this distribution reaches them. + +The scope and schema fields are open **by low expected content**, which is the weakest of the three justifications and is recorded as G-INF-11 rather than waved past. In normal operation they carry a library name, a version, and a specification URL. That is a statement about what a well-behaved emitter sends, not a control: the receiver is unauthenticated, these fields are attacker-settable by any local process that can reach it, and the harness shows no processor filters them. The same reasoning shape β€” this field carries identity, not content β€” is what the harness disproved for span links, so it is registered as a gap on the same footing as the others instead of resting on the argument alone. + +One further boundary: the harness probes the OTLP/HTTP receiver. The shipped configuration also publishes OTLP/gRPC, which is unprobed and recorded as G-INF-10. + ### Denial of Service Prometheus drops delta-temporality metrics by default, and a dropped delta metric was observed failing an entire batched write, discarding co-batched cumulative metrics. `compose.yaml` sets `--enable-feature=otlp-deltatocumulative` so conversion replaces dropping. The conversion state is held in memory and resets when the container restarts, producing a bounded gap rather than a persistent failure. Unauthenticated ingest also permits volumetric flooding of the local store by a local process. Tracked as G-DOS-1. @@ -250,12 +287,15 @@ Not applicable. The receiver executes no submitted content; OTLP payloads are pa ### Risk Rating -| Threat | Likelihood | Impact | Residual Risk | Status | -|------------------------------------------------|------------|--------|---------------|-------------------------------------------------------| -| Local process injects synthetic Copilot series | Low | Medium | Medium | Detectable via `baseline.py`; not prevented (G-SPF-1) | -| Prompt content traverses plaintext OTLP | High | High | Medium | Contained by loopback binding only (G-INF-1, G-TLS-1) | -| Batched write loss from delta temporality | Low | Low | Low | Mitigated by delta-to-cumulative conversion (G-DOS-1) | -| Local flooding of the ingest endpoint | Low | Low | Low | Accepted for a single-machine demonstration stack | +| Threat | Likelihood | Impact | Residual Risk | Status | +|---------------------------------------------------------------------------|------------|--------|---------------|---------------------------------------------------------------------| +| Local process injects synthetic Copilot series | Low | Medium | Medium | Detectable via `baseline.py`; not prevented (G-SPF-1) | +| Prompt content traverses plaintext OTLP | High | High | Medium | Contained by loopback binding only (G-INF-1, G-TLS-1) | +| Content survives in a carrier the filter preserves for a shipped consumer | Medium | High | Medium | Span names and metric metadata pass through by choice (G-INF-8) | +| Content survives in a carrier no processor can reach | Low | Medium | Medium | Span links and exemplars unreachable in this distribution (G-INF-9) | +| Content placed in a carrier assumed to hold only library identity | Low | Medium | Low | Scope and schema fields unfiltered and attacker-settable (G-INF-11) | +| Batched write loss from delta temporality | Low | Low | Low | Mitigated by delta-to-cumulative conversion (G-DOS-1) | +| Local flooding of the ingest endpoint | Low | Low | Low | Accepted for a single-machine demonstration stack | ## Bucket B2: Telemetry at rest and query surfaces @@ -263,7 +303,7 @@ Covers the persistent volume, Prometheus, Tempo, and the Grafana UI. ### Spoofing -Grafana ships with `admin`/`admin` and the skill does not change them. Any local process or local user can authenticate as the Grafana administrator. The compensating control is that Grafana is published on `127.0.0.1:3000` only, so the weak credential is never presented to a network. Prometheus and Tempo expose no authentication at all. Tracked as G-SPF-1. +Grafana no longer runs on a published default. The compose file disables anonymous access, restores the login form, and requires `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD` through the `${VAR:?message}` form, so the stack fails to start rather than coming up on a credential everyone knows. This matters more than it appears: the pinned image enables anonymous access with the Admin role by default, so the previous configuration exposed every panel and every stored prompt fragment to anything that could reach port 3000 without presenting a credential at all. Prometheus and Tempo still expose no authentication, and loopback binding remains their only control. Tracked as G-SPF-1. ### Tampering @@ -283,16 +323,16 @@ The volume grows without bound for traces, because no Tempo retention limit is s ### Elevation of Privilege -Grafana's administrator role is the highest privilege in this bucket and is reachable with published default credentials from any local process. That is privilege escalation within the stack, though not beyond the host user's existing authority, since the same actor could read the volume directly. Tracked as G-EOP-1. +Grafana's administrator role is the highest privilege in this bucket. It was previously reachable with published default credentials, and on this image with anonymous Admin access, from any local process. The compose file now disables anonymous access and requires an operator-supplied credential. Even before that change this was privilege escalation within the stack rather than beyond the host user's existing authority, since the same actor could read the volume directly. Tracked as G-EOP-1. ### Risk Rating -| Threat | Likelihood | Impact | Residual Risk | Status | -|---------------------------------------------------|------------|--------|---------------|---------------------------------------------------| -| Prompt content readable at rest by any local user | Medium | High | Medium | Unencrypted volume; no expiry on traces (G-INF-2) | -| Default Grafana credentials accepted | High | Medium | Low | Loopback-only publishing (G-SPF-1, G-EOP-1) | -| Dashboard overwritten by the validation helper | Low | Low | Low | Non-loopback targets refused by default (G-TAM-1) | -| Unbounded trace growth exhausts disk | Low | Medium | Low | Accepted; operator removes the volume to reclaim | +| Threat | Likelihood | Impact | Residual Risk | Status | +|---------------------------------------------------|------------|--------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Prompt content readable at rest by any local user | Low | High | Medium | Collector drops or scrubs content in every carrier it reaches; span names, metric metadata, links, and exemplars survive, and the volume remains unencrypted (G-INF-2, G-INF-8, G-INF-9) | +| Default Grafana credentials accepted | High | Medium | Low | Loopback-only publishing (G-SPF-1, G-EOP-1) | +| Dashboard overwritten by the validation helper | Low | Low | Low | Non-loopback targets refused by default (G-TAM-1) | +| Unbounded trace growth exhausts disk | Low | Medium | Low | Accepted; operator removes the volume to reclaim | ## Bucket B3: Reference helper scripts to local service APIs @@ -362,11 +402,11 @@ The container runs under the Docker daemon with whatever default privileges the ### Risk Rating -| Threat | Likelihood | Impact | Residual Risk | Status | -|------------------------------------------|------------|--------|---------------|-------------------------------------------------| -| Malicious image substitution under a tag | Low | High | Medium | Tag-pinned, not digest-pinned (G-SUP-1) | -| Compromised image gains daemon authority | Low | High | Medium | No added capabilities or host mounts (G-EOP-2) | -| Registry outage blocks first run | Low | Low | Low | Accepted; cached layers make later runs offline | +| Threat | Likelihood | Impact | Residual Risk | Status | +|------------------------------------------|------------|--------|---------------|-------------------------------------------------------------------------------------------| +| Malicious image substitution under a tag | Low | High | Low | Both images digest-pinned; publisher verification is a separate documented step (G-SUP-1) | +| Compromised image gains daemon authority | Low | High | Medium | No added capabilities or host mounts (G-EOP-2) | +| Registry outage blocks first run | Low | Low | Low | Accepted; cached layers make later runs offline | ## Bucket B5: Editor-global configuration mutation @@ -424,7 +464,7 @@ The generated file is the record of what was proposed. Because the agent does no ### Information Disclosure -The generated compose file contains no credentials. Grafana's default credentials are the image's, not values the skill writes. What the generated stack subsequently collects is covered by B1 and B2. +The generated compose file contains no credentials. Grafana's admin credentials are read from operator-supplied environment variables that the file requires and does not default, so a generated stack carries no secret and refuses to start without one. What the generated stack subsequently collects is covered by B1 and B2. ### Denial of Service @@ -450,6 +490,10 @@ Covers the collector configuration, Bicep, Terraform, and Azure CLI templates un Fleet telemetry authenticates to the collector with whatever static credential the managed settings distribute, because Copilot's exporter can only send a fixed header set. Every workstation therefore presents the same credential and no request is bound to a particular user or device. Anything holding that value can submit telemetry indistinguishable from a real developer's. Tracked as G-INF-3. +That credential does not reach every sender. Managed `telemetry.headers` are applied to the extension exporter directly and deliberately not through the environment, because an environment variable is inherited by the tool subprocesses the agent spawns and would place a fleet-wide write credential inside every model-directed subprocess. The agent host therefore exports without a credential, a receiver that requires one answers HTTP 401 or gRPC `UNAUTHENTICATED`, and OTLP treats neither as retryable, so that telemetry is dropped rather than queued. The two obvious remedies are both worse than the problem: exporting the token defeats the separation, and dropping receiver authentication reopens ingestion on a billed backend. + +The documented remedy is a per-workstation loopback relay that accepts the unauthenticated local export and adds the fleet credential upstream, keeping that credential in the relay's own configuration rather than in the editor's environment. It moves the exposure rather than removing it: the relay's listener is unauthenticated, so any local process reaching loopback can inject through it, and loopback binding constrains reachability without establishing provenance. Mutual TLS would bind a workstation rather than the fleet, and the receiver half is available in this Collector, but whether the agent host can present a client certificate is unestablished, so it is recorded as preferred and blocked rather than offered. Tracked as G-INF-7. + ### Tampering The consequence of the shared credential is that dashboards can be made to say anything. An actor holding it can inject fabricated spans carrying real service names and attribute values, so token totals, tool counts, and per-team breakdowns are all forgeable. The blast radius is write-side only: the credential does not grant read access to the workspace, which is governed separately by Azure RBAC. @@ -472,34 +516,45 @@ Deployment creates billable Azure resources and, when a principal is supplied, a ### Risk Rating -| Threat | Likelihood | Impact | Residual Risk | Status | -|-----------------------------------------------------------|------------|--------|---------------|-------------------------------------------------------------------------------------------| -| Shared fleet credential enables telemetry forgery | Low | Medium | Medium | Inherent to static-header export; stated before generation (G-INF-3) | -| Credential cannot be rotated without a fleet redeployment | Medium | Medium | Medium | No documented in-place rotation; disclosed rather than mitigated (G-INF-3) | -| Prompt content reaches a shared billed workspace | Medium | High | Low | Collector deletes six observed attributes plus one defensively (G-INF-1) | -| Ingestion cost inflation, accidental or deliberate | Medium | Medium | Low | Daily cap defaulted in every template; `captureContent` named as the multiplier (G-DOS-2) | -| Generated template over-grants access on deployment | Low | Medium | Low | Role assignment opt-in; agent never holds Azure credentials (G-EOP-3) | +| Threat | Likelihood | Impact | Residual Risk | Status | +|-------------------------------------------------------------|------------|--------|---------------|-------------------------------------------------------------------------------------------| +| Shared fleet credential enables telemetry forgery | Low | Medium | Medium | Inherent to static-header export; stated before generation (G-INF-3) | +| Agent-host telemetry silently dropped fleet-wide | High | Medium | Low | Loopback relay documented as the remedy; both obvious alternatives rejected (G-INF-7) | +| Local process injects through the relay's loopback listener | Low | Medium | Medium | Unauthenticated by construction; mTLS preferred and blocked on sender evidence (G-INF-7) | +| Credential cannot be rotated without a fleet redeployment | Medium | Medium | Medium | No documented in-place rotation; disclosed rather than mitigated (G-INF-3) | +| Prompt content reaches a shared billed workspace | Medium | High | Low | Collector deletes six observed attributes plus one defensively (G-INF-1) | +| Ingestion cost inflation, accidental or deliberate | Medium | Medium | Low | Daily cap defaulted in every template; `captureContent` named as the multiplier (G-DOS-2) | +| Generated template over-grants access on deployment | Low | Medium | Low | Role assignment opt-in; agent never holds Azure credentials (G-EOP-3) | ## Enterprise Readiness Gaps -| Id | Severity | Gap | Status | -|---------|-----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| G-INF-1 | InfoDisc-High | Spans carry full prompt text, tool call arguments and results, and system instructions on a configuration where content capture was left at its documented default. The skill cannot change extension behavior. | Local: documented prominently with a verification command, contained only by loopback binding. Organization: the generated collector deletes seven content attributes before export, and the Azure dashboard counts them so a leak is visible. | -| G-INF-2 | InfoDisc-Med | The `copilot-otel-data` volume stores captured content unencrypted, with no Tempo retention limit and no expiry, and survives `docker compose down` by design. | Open, and stated as a real exposure rather than a harmless local condition. `SKILL.md` and this model agree that the store holds prompt content regardless of the capture setting; the user owns the decision. Teardown documentation states both variants explicitly. | -| G-INF-3 | InfoDisc-Med | Organization capture distributes one static write-side credential to every workstation, with no per-user binding and no documented in-place rotation. Anything holding it can inject telemetry indistinguishable from a real developer's. | Open. Inherent to static-header export against a credentialed backend. Disclosed before any Azure artifact is generated, with the write-side-only blast radius stated plainly. | -| G-INF-4 | InfoDisc-Low | The assisted settings write reads the whole global `settings.json`, so unrelated values in that file enter model context. | Accepted. Reading the whole document is required to preserve it; only the changed lines are echoed in the presented diff. | -| G-SPF-1 | Spoofing-Med | OTLP ingest, Prometheus, and Tempo are unauthenticated, and Grafana accepts published default credentials. Any local process can read or write the store. | Mitigated only by `127.0.0.1` port binding. `baseline.py` provides after-the-fact detection of injected series. | -| G-SUP-1 | SupplyChain-Med | The stack image is pinned by tag rather than by digest, and no signature or provenance verification is performed before the container runs. | Open. Digest pinning would close the substitution vector at the cost of manual updates. | -| G-EOP-1 | EoP-Low | Grafana administrator access is reachable from any local process using published default credentials. | Accepted. The same actor can already read the volume directly, so the escalation does not cross the host user boundary. | -| G-EOP-2 | EoP-Med | A compromised stack image would execute with Docker daemon authority, which is root-equivalent on a typical developer workstation. | Bounded by the tag pin, by adding no capabilities, and by mounting no host paths beyond the named volume. | -| G-TAM-1 | Tampering-Low | `validate_dashboard.py` imports with `overwrite: true`, so it would replace an unrelated dashboard sharing the same uid. | Constrained. The helper refuses a non-loopback Grafana unless `COPILOT_OTEL_ALLOW_REMOTE=1` is set, and endpoint and credentials come from the environment rather than being hard-coded. | -| G-TAM-2 | Tampering-Med | The assisted settings write mutates a user-owned JSONC file that may hold comments, formatting, and unrelated configuration the skill did not create. | Mitigated structurally: timestamped backup, per-key upsert that never reserializes, exact diff, explicit approval, post-write parse with automatic restore. Concurrent VS Code writes remain unpreventable. | -| G-REP-1 | Repudiation-Low | Settings Sync conflict behavior for a write made outside the running VS Code instance was never confirmed. | Open. Disclosed to the user as a caveat before the write rather than asserted in either direction. | -| G-DOS-1 | DoS-Low | Delta-to-cumulative conversion state is held in memory and resets on container restart, producing a bounded gap in converted series. | Accepted. Without the flag the failure mode is worse, because a dropped delta metric can fail an entire batched write. | -| G-DOS-2 | DoS-Low | The shared fleet credential permits unbounded ingestion against a billed backend, so the practical denial of service is financial. | Mitigated by a 5 GB daily cap defaulted in every generated template, and by naming `captureContent` as the dominant volume multiplier wherever cost is discussed. | -| G-EOP-3 | EoP-Med | Generated infrastructure templates provision billable Azure resources and can create a `Monitoring Reader` role assignment when deployed. | Bounded. The agent never holds Azure credentials and never deploys; the role assignment is opt-in through an empty-by-default parameter; required inputs have no defaults. | -| G-EOP-4 | EoP-Med | The prohibition on the agent running `docker compose`, `az deployment`, or `terraform apply` is advisory prose in `SKILL.md` rather than an enforced control. | Open. Stated in the constraints and the stop rules and exercised by the behavior gate. A hook would make it enforced. | -| G-TLS-1 | InfoDisc-Low | OTLP ingest and all service queries use plaintext HTTP with no transport security. | Acceptable on loopback. Material the moment `otlpEndpoint` targets a remote collector, which the skill states explicitly. | +| Id | Severity | Gap | Status | +|----------|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| G-INF-1 | InfoDisc-High | Spans carry full prompt text, tool call arguments and results, and system instructions on a configuration where content capture was left at its documented default. The skill cannot change extension behavior. | Partially mitigated. Content still leaves the editor and crosses the loopback hop in plaintext, which remains outside the skill's control. Both the local and organization paths now filter before storage: local uses a fail-closed allow-list at the Collector, so an attribute added by a future extension release is dropped rather than stored, and the Azure path deletes the seven observed content attributes before export. | +| G-INF-2 | InfoDisc-Med | The `copilot-otel-data` volume stores whatever survives filtering, unencrypted, and survives `docker compose down` by design. | Open on encryption, reduced on content. The allow-list keeps observed and unknown content attributes out of the volume, so what remains is usage and cost data rather than prompt text. Volume encryption is a Docker and host-platform property the skill does not control. Traces inherit the image's 336h Tempo expiry rather than persisting indefinitely; metrics retain for 120 days by configuration. Teardown documentation states both variants explicitly. | +| G-INF-3 | InfoDisc-Med | Organization capture distributes one static write-side credential to every workstation, with no per-user binding and no documented in-place rotation. Anything holding it can inject telemetry indistinguishable from a real developer's. | Open and now active rather than optional. The receiver requires the token instead of leaving authentication commented out, so the gap is a bounded shared-credential weakness rather than an unauthenticated endpoint. Disclosed before any Azure artifact is generated, with the write-side-only blast radius stated plainly. | +| G-INF-4 | InfoDisc-Low | The assisted settings write reads the whole global `settings.json`, so unrelated values in that file enter model context. | Accepted. Reading the whole document is required to preserve it; only the changed lines are echoed in the presented diff. | +| G-SPF-1 | Spoofing-Med | The Prometheus and Tempo query APIs are unauthenticated, and the local OTLP ingress accepts spans from any local process. | Reduced. Grafana now requires an operator-supplied credential with anonymous access disabled. Prometheus and Tempo remain unauthenticated behind `127.0.0.1` binding, and injected spans are still accepted, though the allow-list bounds what an injector can store. `baseline.py` provides after-the-fact detection of injected series. | +| G-SUP-1 | SupplyChain-Med | Image identity was previously pinned by mutable tag, and no signature or provenance verification is performed before the container runs. | Partially closed. Both images are now pinned by multi-architecture manifest digest, which removes the substitution-under-a-tag vector. Signature and provenance verification remains a documented operator step rather than an enforced one; the update procedure states that digest pinning, publisher verification, and vulnerability review are three separate checks. | +| G-EOP-1 | EoP-Low | Grafana administrator access was previously reachable from any local process using published default credentials. | Closed. Anonymous Admin access is disabled and the admin credential is operator-supplied with no default. The residual observation stands that the same local actor can read the volume directly, so this was never a host-user boundary crossing. | +| G-EOP-2 | EoP-Med | A compromised stack image would execute with Docker daemon authority, which is root-equivalent on a typical developer workstation. | Bounded by digest pinning, by adding no capabilities, and by mounting no host paths beyond the named volume and the read-only Collector configuration. | +| G-TAM-1 | Tampering-Low | `validate_dashboard.py` imports with `overwrite: true`, so it would replace an unrelated dashboard sharing the same uid. | Constrained. All configurable endpoints pass a shared policy that refuses non-http schemes, credentials in the authority, unrelated local ports, and non-loopback hosts without an explicit opt-in, and re-applies the same rules to every redirect target. The dashboard path is contained to the examples directory. | +| G-TAM-2 | Tampering-Med | The assisted settings write mutates a user-owned JSONC file that may hold comments, formatting, and unrelated configuration the skill did not create. | Mitigated structurally and now executably. `examples/settings_upsert.py` enforces the contract the prose described: schema-checked keys and types, value-span splicing that never reserializes, a refusal to write when unrelated settings would change or the result would not parse, backup before write, restore on failure, and a redacted audit record. Concurrent VS Code writes remain unpreventable. | +| G-REP-1 | Repudiation-Low | Settings Sync conflict behavior for a write made outside the running VS Code instance was never confirmed. | Open. Disclosed to the user as a caveat before the write rather than asserted in either direction. | +| G-DOS-1 | DoS-Low | Delta-to-cumulative conversion state is held in memory and resets on container restart, producing a bounded gap in converted series. | Accepted. Without the flag the failure mode is worse, because a dropped delta metric can fail an entire batched write. | +| G-DOS-2 | DoS-Low | The shared fleet credential permits unbounded ingestion against a billed backend, so the practical denial of service is financial. | Mitigated by a 5 GB daily cap defaulted in every generated template, and by naming `captureContent` as the dominant volume multiplier wherever cost is discussed. | +| G-EOP-3 | EoP-Med | Generated infrastructure templates provision billable Azure resources and can create a `Monitoring Reader` role assignment when deployed. | Bounded. The agent never holds Azure credentials and never deploys; the role assignment is opt-in through an empty-by-default parameter and is scoped to a single workspace rather than to the resource group or subscription; required inputs have no defaults. | +| G-TLS-2 | Spoofing-Med | The generated fleet receiver now terminates TLS from operator-supplied material, but how Copilot's exporter validates that certificate is exporter behavior the skill does not control. | Open by ownership. The receiver requires TLS material or an explicitly recorded terminating ingress that owns certificate lifecycle and cipher policy. No claim is made about exporter-side validation or custom CA support, because none was verified. | +| G-INF-5 | InfoDisc-Med | A single shared Log Analytics workspace gives every authorized reader visibility across all environments, and a resource attribute cannot restrict that after the fact. | Mitigated by default. One collector endpoint, ingest token, Application Insights component, and workspace per environment is now the shipped topology; environment is a required input with no default in Bicep, Terraform, and the CLI script. Reader assignments are scoped to a single workspace. Attributes are documented as grouping keys, never as isolation controls. This is environment separation, not customer multi-tenancy, and is stated as such. | +| G-INF-6 | InfoDisc-Low | Retention is the only deletion mechanism. Neither the local stack nor the Azure templates purge a specific person's data on request. | Open and operator-owned. Retention is documented as the deletion boundary in both templates and their guidance, and an erasure obligation is presented as an operator procedure against the workspace rather than a template capability. No purge automation is provided. | +| G-SUP-2 | SupplyChain-Low | Terraform state for the fleet templates contains the Application Insights connection string, a fleet-wide write credential, and `sensitive = true` does not remove it from state. | Documented rather than solved. The configuration, its README, and the Azure reference all state that sensitivity hides CLI display only, and recommend an encrypted remote backend with restricted access and treating the state file as a secret. | +| G-EOP-4 | EoP-Med | The prohibition on the agent running `docker compose`, `az deployment`, or `terraform apply` is advisory prose in `SKILL.md` rather than an enforced control. | Open. Stated in the constraints and the stop rules and exercised by the behavior gate. A hook would make it enforced. | +| G-TLS-1 | InfoDisc-Low | OTLP ingest and all service queries use plaintext HTTP with no transport security. | Acceptable on loopback. Material the moment `otlpEndpoint` targets a remote collector, which the skill states explicitly. | +| G-INF-7 | InfoDisc-Med | The per-workstation relay that restores fleet agent-host telemetry listens without authentication, so any local process able to reach loopback can inject spans that the relay then forwards under the fleet credential. | Open, and the least-bad of three options: the alternatives are delivering the ingest token through an environment agent-spawned subprocesses inherit, or removing receiver authentication entirely. Loopback binding is necessary and not sufficient for provenance. The relay authenticates the relay, not the workstation and not the developer. mTLS is the preferred replacement and is blocked on unestablished agent-host client-certificate support. | +| G-INF-8 | InfoDisc-Med | Span names, metric names, metric descriptions, and metric units reach storage unfiltered. A span name is emitter-composed and can carry request text. | Open by choice, not by oversight. Three dashboard TraceQL queries match on span name, `baseline.py` reads Tempo `rootTraceName` as its injection discriminator, and every shipped Prometheus query selects by metric name; scrubbing any of them breaks shipped functionality. Recorded by `tests/test_collector_carriers.py`, which fails if a scrub rule is later added that targets them. | +| G-INF-9 | InfoDisc-Med | Span link attributes, span link trace state, and metric exemplar filtered attributes reach storage unfiltered. | Open by capability. Verified against the pinned Collector: the `redaction` processor does not traverse links or exemplars, OTTL has no `spanlink` context and refuses to index `links`, and assigning to `datapoint.exemplars` parses without effect. No processor in this distribution reaches them. Whether Copilot populates links or exemplars in practice is unmeasured, so the exposure is recorded unbounded rather than narrowed by an assumption; settling it needs a live capture. | +| G-INF-10 | InfoDisc-Low | The carrier map is established against the OTLP/HTTP receiver. The shipped configuration also publishes OTLP/gRPC, whose content handling is unprobed. | Open. Both receivers feed the same processor chain, so divergence is unlikely rather than excluded. Closing it means extending the harness to the gRPC path. | +| G-INF-11 | InfoDisc-Low | Instrumentation scope name and version and the resource and scope schema URLs reach storage unfiltered. They normally carry library identity, but the receiver is unauthenticated and no processor filters them, so a local process can place arbitrary text there. | Open by low expected content, the weakest of this document's three justifications for an open carrier. Recorded as a gap rather than dismissed by rationale, because "identity, not content" is the same argument the harness disproved for span links. Closing it means extending `transform/scrub` to these fields, which is possible for the schema URLs and would need a consumer check for scope name. | ## References diff --git a/.github/skills/experimental/copilot-otel-metrics/SKILL.md b/.github/skills/experimental/copilot-otel-metrics/SKILL.md index 409543a4d..a0d23131c 100644 --- a/.github/skills/experimental/copilot-otel-metrics/SKILL.md +++ b/.github/skills/experimental/copilot-otel-metrics/SKILL.md @@ -64,7 +64,8 @@ Every setting name, metric name, and API version here is a snapshot of one build * **The installed extension manifest settles settings.** Read `contributes.configuration` in the Copilot Chat extension's `package.json`, or open the Settings UI and filter on `otel`. Do not settle a settings question from documentation, including this document. * **The store settles metric names.** Enumerate what the build emits before writing a query against it. -* **Verified for this revision:** extension `copilot-chat` `0.59.2026072702` declares 11 `github.copilot.chat.otel.*` settings, all `scope: application`. Re-check the count before relying on it. +* **A runtime test settles what the filter reaches.** A claim about how a running Collector behaves needs runtime evidence. `tests/test_collector_carriers.py` starts the pinned image and records what each OTLP carrier does; a static test that reads the configuration and confirms it says the right thing does not establish that the Collector does the right thing. Every unbacked control claim this skill has had to retract came from treating the first as the second. +* **Verified for this revision:** extension `copilot-chat` `0.52.0` declares 7 `github.copilot.chat.otel.*` settings, none declaring a `scope`. An earlier revision verified `0.59.2026072702` as declaring 11, all `scope: application`. Both observations stand for their own build, which is exactly why the count and the scope must be re-checked rather than carried forward. Treat any statement about how many settings exist, or which file they resolve from, as build-specific. ## Inputs @@ -108,22 +109,24 @@ Every setting name, metric name, and API version here is a snapshot of one build Read the references. Copy or adapt the seeds. Offer the helpers to the user with the exact command, resolved to an absolute path so it works from whatever directory their shell is in. -| Path | Use | -|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| -| `references/local-setup.md` | Read for the settings procedure, the profile pitfall, the diff-and-confirm write, and the paste alternative | -| `references/local-stack.md` | Read for the local backend, the generated compose contract, and the helper inventory | -| `references/org-distribution.md` | Read for managed-settings channels, precedence, and the agent-host split | -| `references/azure-capture.md` | Read for the Azure data path, the product and cost comparison, and the generated templates | -| `references/verification.md` | Read for proving data landed, the false-positive pitfalls, and metric enumeration | -| `examples/compose.yaml` | Copy as the seed for a generated local stack | -| `examples/dashboards/copilot-otel.json` | Copy as the seed for a generated local PromQL dashboard | -| `examples/dashboards/copilot-otel-azure.json` | Copy as the seed for a generated Azure KQL dashboard | -| `examples/azure/` | Copy as the seed for the collector configuration and infrastructure templates | -| `examples/verify.py` | Offer to the user as a stored-signal check they run themselves | -| `examples/inspect_metrics.py` | Offer to the user to enumerate the metric surface their build emits | -| `examples/baseline.py` | Offer to the user to separate real telemetry from residue | -| `examples/validate_dashboard.py` | Offer to the user for a **local** PromQL or TraceQL dashboard only; it has no Azure Monitor path and it overwrites by uid | -| `examples/README.md`, `examples/azure/README.md` | Written for the user, not part of the agent's reading path. Point the user at them rather than summarizing them | +| Path | Use | +|--------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| `references/local-setup.md` | Read for the settings procedure, the profile pitfall, the diff-and-confirm write, and the paste alternative | +| `references/local-stack.md` | Read for the local backend, the generated compose contract, and the helper inventory | +| `references/org-distribution.md` | Read for managed-settings channels, precedence, and the agent-host split | +| `references/azure-capture.md` | Read for the Azure data path, the product and cost comparison, and the generated templates | +| `references/verification.md` | Read for proving data landed, the false-positive pitfalls, and metric enumeration | +| `examples/compose.yaml` | Copy as the seed for a generated local stack | +| `examples/otel-collector-local.yaml` | Copy alongside `compose.yaml`; carries the fail-closed local attribute allow-list and the content scrub for the carriers it cannot reach | +| `examples/dashboards/copilot-otel.json` | Copy as the seed for a generated local PromQL dashboard | +| `examples/dashboards/copilot-otel-azure.json` | Copy as the seed for a generated Azure KQL dashboard | +| `examples/azure/` | Copy as the seed for the collector configuration and infrastructure templates | +| `examples/verify.py` | Offer to the user as a stored-signal check they run themselves | +| `examples/inspect_metrics.py` | Offer to the user to enumerate the metric surface their build emits | +| `examples/baseline.py` | Offer to the user to separate real telemetry from residue | +| `examples/validate_dashboard.py` | Offer to the user for a **local** PromQL or TraceQL dashboard only; it has no Azure Monitor path and it overwrites by uid | +| `examples/settings_upsert.py` | Offer to the user for the assisted settings write; it enforces the schema, backs up, and restores on a parse failure | +| `examples/README.md`, `examples/azure/README.md` | Written for the user, not part of the agent's reading path. Point the user at them rather than summarizing them | ## References diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/README.md b/.github/skills/experimental/copilot-otel-metrics/examples/README.md index 45736235c..9e2d18361 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/README.md +++ b/.github/skills/experimental/copilot-otel-metrics/examples/README.md @@ -2,7 +2,7 @@ title: Copilot OTel Metrics Examples description: Reference stack definition, dashboard, and helper scripts for capturing GitHub Copilot OpenTelemetry output locally author: Microsoft -ms.date: 2026-07-27 +ms.date: 2026-08-13 ms.topic: reference keywords: - opentelemetry @@ -15,11 +15,12 @@ estimated_reading_time: 4 ## What is here -Two kinds of file. `compose.yaml` and `dashboards/` are **seeds**: the skill copies and adapts them to produce artifacts for your situation, and you deploy the result. The Python files are **helpers you run yourself**; the skill offers them and shows the command, and you decide when they run and against which endpoint. +Two kinds of file. `compose.yaml`, `otel-collector-local.yaml`, and `dashboards/` are **seeds**: the skill copies and adapts them to produce artifacts for your situation, and you deploy the result. The Python files are **helpers you run yourself**; the skill offers them and shows the command, and you decide when they run and against which endpoint. | File | Kind | Purpose | |--------------------------------------|--------|-----------------------------------------------------------------------------| -| `compose.yaml` | Seed | Pinned single-container stack with the external data volume | +| `compose.yaml` | Seed | Digest-pinned Collector plus LGTM stack with the external data volume | +| `otel-collector-local.yaml` | Seed | Local Collector pipeline with the fail-closed attribute allow-list | | `dashboards/copilot-otel.json` | Seed | Local Grafana dashboard: tokens, tools, latency, and agents | | `dashboards/copilot-otel-azure.json` | Seed | Azure dashboard querying Log Analytics with KQL | | `azure/` | Seed | Collector config, Bicep, Terraform, and Azure CLI for the organization path | @@ -27,6 +28,8 @@ Two kinds of file. `compose.yaml` and `dashboards/` are **seeds**: the skill cop | `baseline.py` | Helper | Snapshot and diff, to separate real telemetry from residue | | `inspect_metrics.py` | Helper | Enumerate the metric surface the installed build actually emits | | `validate_dashboard.py` | Helper | Import a dashboard and check every panel query returns data | +| `settings_upsert.py` | Helper | Reversible, audited settings edit that refuses anything outside its schema | +| `_input_policy.py` | Module | Shared endpoint, redirect, path, and credential policy; imported, not run | The two dashboards are not interchangeable. The local one queries Prometheus and Tempo; the Azure one queries Log Analytics. See `azure/README.md` for the organization path. @@ -37,38 +40,53 @@ The helpers use only the Python standard library, so there is nothing to install Run these yourself, from this directory. Nothing here runs automatically, and the skill will not run any of it for you. ```bash -# 1. Create the volume once, then start the stack. +# 1. Choose Grafana credentials. The stack will not start without them, and +# the helpers read the same two variables. Reading the password rather than +# writing it inline keeps it out of the terminal scrollback and out of the +# shell history file. +export COPILOT_OTEL_GRAFANA_USER=admin +read -rs -p 'Grafana password: ' COPILOT_OTEL_GRAFANA_PASSWORD; echo +export COPILOT_OTEL_GRAFANA_PASSWORD + +# 2. Create the volume once, then start the stack. docker volume create copilot-otel-data docker compose -f compose.yaml up -d -# 2. Optional but recommended if this store has ever held test payloads. +# 3. Optional but recommended if this store has ever held test payloads. python3 baseline.py capture -# 3. Enable export in your VS Code user settings, then reload the window. +# 4. Enable export in your VS Code user settings, then reload the window. -# 4. Confirm signals are actually stored, not merely accepted. +# 5. Confirm signals are actually stored, not merely accepted. python3 verify.py -# 5. If step 4 is ambiguous, prove the telemetry is genuine. +# 6. If step 5 is ambiguous, prove the telemetry is genuine. python3 baseline.py diff -# 6. See what your build really emits before trusting any metric name. +# 7. See what your build really emits before trusting any metric name. python3 inspect_metrics.py -# 7. Import a dashboard and check every panel resolves. +# 8. Import a dashboard and check every panel resolves. python3 validate_dashboard.py # the bundled seed python3 validate_dashboard.py my-dash.json # a generated dashboard ``` ## Notes on the stack definition -`compose.yaml` carries four deliberate choices worth preserving if you adapt it. +`compose.yaml` and `otel-collector-local.yaml` carry deliberate choices worth preserving if you adapt them. -* The image tag is pinned. The all-in-one image changes its bundled component versions between tags. +* **Copilot exports to the Collector, not to LGTM.** LGTM's OTLP ports are not published to the host at all. That is what makes the filter unavoidable: if LGTM listened on the host too, any local process could write straight past it, and loopback binding would not help, because every process on the machine is already on loopback. +* **The attribute policy is an allow-list.** `allow_all_keys: false` drops any key the config has not named, so an attribute added by a future extension release is dropped rather than stored. A delete-list would pass it through. Every allowed key is there because a shipped dashboard panel reads it. +* **Grafana requires a credential you supply.** The image otherwise enables anonymous Admin access and hides the login form. The compose file turns that off and requires `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD`; an unset variable fails the `up` with a message rather than starting on a guessable credential. +* **Both images are pinned by digest, with the tag in a comment.** A tag is mutable, so a tag-pinned stack can change under a configuration you already reviewed. +* Every port publishes to `127.0.0.1` only. The OTLP endpoint accepts writes from anything that can reach it. * The data volume is declared `external`, so Compose binds the existing `copilot-otel-data` volume instead of creating a project-prefixed duplicate that would orphan your history. -* Every port publishes to `127.0.0.1` only. Grafana ships with default credentials, and this stack is single-machine. * `PROMETHEUS_EXTRA_ARGS` enables delta-to-cumulative conversion and raises retention to 120 days. Prometheus otherwise drops delta metrics, and a dropped delta metric was observed failing an entire batched write. The default 15-day retention silently truncates monthly token totals. +The filter governs what reaches the store. It does not change what the extension emits, so content attributes still leave the editor and still cross the loopback hop in plaintext. + +To move to a newer image, resolve the multi-architecture manifest digest for the tag you want from the publisher's own registry listing, verify the publisher's signature or attestation separately, and review the new version for known vulnerabilities. Those are three different checks and none of them implies the others. + ## Notes on the helpers `verify.py` exits non-zero when the stack is unhealthy or when no Copilot signals are stored. Health and the delta flag are treated as required; signal presence depends on the editor having exported something. diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py new file mode 100644 index 000000000..0e280e628 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Shared input policy for the bundled helper scripts. + +Every helper takes a configurable endpoint or path from the environment or the +command line. Each one previously guarded itself, or did not, and the guards +disagreed: the dashboard helper checked a hostname while the metric and trace +endpoints were unchecked, and no helper rechecked the target after a redirect. + +This module is the single place those decisions are made, so a helper cannot be +hardened by accident in one place and left open in another. + +Nothing here makes an arbitrary PromQL or TraceQL query safe. It constrains +where a request may go and where a file may be written, not what is asked for. +""" + +from __future__ import annotations + +import ipaddress +import os +import pathlib +import urllib.parse +import urllib.request + +__all__ = [ + "DEFAULT_ALLOWED_PORTS", + "PolicyError", + "check_url", + "contain_path", + "open_url", + "origin_of", + "require_credentials", +] + + +class PolicyError(ValueError): + """Raised when an input is refused. No request and no write has occurred.""" + + +# The local stack's published surfaces. A helper pointed somewhere else on the +# machine is more likely a mistake or a redirect than an intent. +DEFAULT_ALLOWED_PORTS = frozenset({3000, 3200, 4317, 4318, 9090}) + +_ALLOWED_SCHEMES = frozenset({"http", "https"}) +_LOOPBACK_NAMES = frozenset({"localhost", "ip6-localhost", "ip6-loopback"}) + +# Headers that identify the caller to the origin they were addressed to. They +# are meaningless to a different origin and dangerous there. +_CREDENTIAL_HEADERS = frozenset({"authorization", "proxy-authorization", "cookie"}) + +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +def origin_of(url: str) -> tuple[str, str, int]: + """Return the normalized (scheme, host, port) triple for a URL. + + Normalization is the whole point. `http://h/x` and `http://h:80/x` are the + same origin, and comparing raw parsed ports would call them different and + strip credentials from a redirect that never left the origin that was + given them. + """ + parsed = urllib.parse.urlparse(url) + scheme = parsed.scheme.lower() + host = (parsed.hostname or "").lower() + try: + port = parsed.port + except ValueError: + port = None + return scheme, host, port if port is not None else _DEFAULT_PORTS.get(scheme, -1) + + +def _is_loopback(host: str) -> bool: + """Whether a hostname or address refers to this machine.""" + if host.lower() in _LOOPBACK_NAMES: + return True + try: + return ipaddress.ip_address(host.strip("[]")).is_loopback + except ValueError: + return False + + +def check_url( + url: str, + *, + allow_remote: bool = False, + allowed_ports: frozenset[int] = DEFAULT_ALLOWED_PORTS, +) -> urllib.parse.ParseResult: + """Return the parsed URL if policy permits it, otherwise raise PolicyError. + + Remote targets require both an explicit opt-in and HTTPS. A loopback target + may stay on HTTP because it does not leave the machine. + """ + try: + parsed = urllib.parse.urlparse(url) + except ValueError as exc: + raise PolicyError(f"unparseable URL: {exc}") from exc + + if parsed.scheme not in _ALLOWED_SCHEMES: + raise PolicyError( + f"refusing scheme '{parsed.scheme or '(none)'}': only http and https are allowed" + ) + + # Credentials in the authority are refused rather than stripped, because + # stripping them silently changes which identity the request is made as. + if parsed.username is not None or parsed.password is not None: + raise PolicyError("refusing a URL that carries credentials in its authority") + + try: + host = parsed.hostname + port = parsed.port + except ValueError as exc: + raise PolicyError(f"malformed authority: {exc}") from exc + + if not host: + raise PolicyError("refusing a URL with no host") + + resolved_port = port if port is not None else (443 if parsed.scheme == "https" else 80) + + if _is_loopback(host): + if resolved_port not in allowed_ports: + raise PolicyError( + f"refusing local port {resolved_port}: expected one of " + f"{', '.join(str(value) for value in sorted(allowed_ports))}" + ) + return parsed + + if not allow_remote: + raise PolicyError( + f"refusing non-loopback host '{host}'. " + "Set COPILOT_OTEL_ALLOW_REMOTE=1 only if that target is disposable." + ) + if parsed.scheme != "https": + raise PolicyError(f"refusing plaintext http to remote host '{host}': use https") + return parsed + + +class _PolicyRedirectHandler(urllib.request.HTTPRedirectHandler): + """Re-applies policy to every redirect target. + + A server that answers a permitted loopback request with a redirect to + somewhere else would otherwise carry the request straight past the check + that was made on the original URL. + + Rechecking the target is not enough on its own. The standard library + carries `Authorization`, `Proxy-Authorization`, and `Cookie` onto the + redirected request, so a permitted redirect to a different origin would + hand that origin a credential it was never issued. Those headers are + removed after the redirect request is built, and only when the normalized + origin actually changed. + """ + + def __init__(self, *, allow_remote: bool, allowed_ports: frozenset[int]) -> None: + self._allow_remote = allow_remote + self._allowed_ports = allowed_ports + + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ANN201 + check_url(newurl, allow_remote=self._allow_remote, allowed_ports=self._allowed_ports) + redirected = super().redirect_request(req, fp, code, msg, headers, newurl) + if redirected is None: + return None + if origin_of(req.full_url) != origin_of(redirected.full_url): + _strip_credential_headers(redirected) + return redirected + + +def _strip_credential_headers(request: urllib.request.Request) -> None: + """Remove credential headers from both stores urllib keeps them in. + + Keys are matched case-insensitively rather than by reproducing urllib's + `str.capitalize()` convention, so this does not depend on how the header + was spelled when it was added. + """ + for store in (request.headers, request.unredirected_hdrs): + for key in [name for name in store if name.lower() in _CREDENTIAL_HEADERS]: + del store[key] + + +def open_url( + request: urllib.request.Request | str, + *, + allow_remote: bool = False, + allowed_ports: frozenset[int] = DEFAULT_ALLOWED_PORTS, + timeout: int = 25, +): # noqa: ANN201 + """Open a URL after checking it, and re-check every redirect it follows.""" + url = request if isinstance(request, str) else request.full_url + check_url(url, allow_remote=allow_remote, allowed_ports=allowed_ports) + opener = urllib.request.build_opener( + _PolicyRedirectHandler(allow_remote=allow_remote, allowed_ports=allowed_ports) + ) + return opener.open(request, timeout=timeout) + + +def contain_path(candidate: str | os.PathLike[str], root: str | os.PathLike[str]) -> pathlib.Path: + """Return the resolved candidate if it stays inside root, else raise. + + Both sides are fully resolved first, so `..` segments and symlinks that + point outside the root are caught rather than normalized away. + """ + resolved_root = pathlib.Path(root).expanduser().resolve() + resolved = pathlib.Path(candidate).expanduser() + if not resolved.is_absolute(): + resolved = resolved_root / resolved + resolved = resolved.resolve() + + if resolved != resolved_root and resolved_root not in resolved.parents: + raise PolicyError(f"refusing a path outside {resolved_root}: {resolved}") + return resolved + + +def require_credentials(*names: str) -> tuple[str, ...]: + """Return the named environment values, or raise naming every missing one. + + Missing configuration and a rejected credential are different problems, and + a helper that sends a request with an empty password reports the second + when it means the first. + """ + values = tuple(os.environ.get(name, "") for name in names) + missing = [name for name, value in zip(names, values, strict=True) if not value] + if missing: + raise PolicyError(f"missing required environment variable(s): {', '.join(missing)}") + return values diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md b/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md index 00f95b6f4..83953e5c0 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md @@ -56,28 +56,86 @@ Azure Managed Grafana, if you decide its triggers apply, needs `az extension add ## Deploying +Deploy once per environment. Every resource name carries the environment, so `prod` and `staging` produce separate workspaces rather than colliding. + ```bash # Bicep -az deployment group create -g -f main.bicep -p namePrefix= +az deployment group create -g -f main.bicep \ + -p namePrefix= environment= # Terraform terraform init -terraform plan -var name_prefix= -var resource_group_name= -var location= -terraform apply -var name_prefix= -var resource_group_name= -var location= +terraform plan -var name_prefix= -var environment= -var resource_group_name= -var location= +terraform apply -var name_prefix= -var environment= -var resource_group_name= -var location= # Azure CLI -RESOURCE_GROUP= LOCATION= NAME_PREFIX= ./deploy.sh +RESOURCE_GROUP= LOCATION= NAME_PREFIX= ENVIRONMENT= ./deploy.sh ``` `backend.tf` is intentionally absent from the Terraform configuration. Remote state belongs to the repository that consumes these files, not to the template. +The Terraform state for this configuration contains the Application Insights connection string, which is a fleet-wide write credential. The output is marked `sensitive`, which keeps it out of CLI display but does **not** keep it out of state. Use a remote backend with encryption at rest and restricted access, and treat the state file as a secret in its own right. + +## Why one deployment per environment + +A resource attribute such as `service.namespace` or `deployment.environment.name` is a grouping key. It lets a reader filter; it does not stop them removing the filter. Anyone with `Monitoring Reader` on a workspace can read everything in it. + +Separation therefore comes from separate workspaces and separate collector endpoints, each with its own ingest token, and from scoping every reader assignment to a single workspace. The templates scope the assignment to the workspace for this reason; assigning at subscription or resource-group scope hands the principal every environment at once and undoes the split. + +This is environment separation, not customer multi-tenancy. Copilot's exporter sends the same static headers from every workstation, so nothing in the payload identifies a tenant in a way you could trust for isolation. + +## Collector authentication and TLS + +`otel-collector-config.yaml` requires both, actively rather than as a suggestion: + +```bash +export COPILOT_OTEL_INGEST_TOKEN="" +export COPILOT_OTEL_ENVIRONMENT="prod" +export COPILOT_OTEL_TLS_CERT_FILE="/etc/otel/tls/tls.crt" +export COPILOT_OTEL_TLS_KEY_FILE="/etc/otel/tls/tls.key" +``` + +Be clear about what the token does. Every workstation holds the same value, so it authenticates the fleet rather than a person, it cannot be rotated for one user, and anyone who extracts it from a workstation can write spans as the fleet. It is still worth having: without it the receiver accepts spans from anything that can route to it, and every accepted span is billed and stored. + +If TLS is terminated by an ingress controller or load balancer in front of the collector, delete the `tls` blocks and record that the terminating hop owns certificate lifecycle and cipher policy. Deleting them because certificates are inconvenient leaves fleet telemetry in plaintext on the wire. + +What the collector configuration cannot control is the sender. Copilot's exporter decides how it validates the server certificate, and no setting here changes that. + +## Agent-host telemetry needs a relay + +Requiring authentication on the receiver has a consequence that is easy to miss because its symptom is an absence. + +Managed `telemetry.headers` are applied to the extension exporter directly and are deliberately **not** exported into the environment, because an environment variable would be inherited by the tool subprocesses the agent spawns, putting this fleet-wide write credential inside every model-directed subprocess. So the extension authenticates and the agent host does not. This receiver rejects the agent host's export with HTTP 401 or gRPC `UNAUTHENTICATED`, and neither is retryable under the OTLP specification: the data is dropped, not queued. + +Do not fix this by exporting the token into the environment, and do not fix it by removing authentication from the receiver. The first defeats the separation that keeps the credential away from agent-spawned subprocesses; the second reopens ingestion to anything that can route to this endpoint, on a billed backend. + +Run a per-workstation relay instead: a Collector bound to `127.0.0.1:4318` that accepts the agent host's unauthenticated export and adds the fleet credential on its own upstream hop. `../otel-collector-local.yaml` is the same Collector configuration, so its fail-closed attribute allow-list applies to agent-host telemetry too. + +Two conditions decide whether this actually keeps the credential out of the editor's reach: + +* Launch the relay **independently** of VS Code, as a service or user daemon. A relay started from the same shell as VS Code, or as its child, can share the environment this arrangement exists to keep clean. +* Store its credential where the relay reads it and the editor does not: its configuration file or a secret store, not an exported variable. + +**What the relay does not do.** Its listener is unauthenticated, so any local process that can reach loopback can inject telemetry into this backend through it. Binding to `127.0.0.1` keeps the listener off the network; it says nothing about which local process connected. The relay authenticates the relay to this endpoint. It does not authenticate the workstation and it does not authenticate the developer. + +Mutual TLS would be better, and this receiver already supports the server half: adding `client_ca_file` beside `cert_file` and `key_file` makes a valid client certificate mandatory on both protocols. Whether the VS Code agent host can present one is unknown, so it is not offered here as an available option. + +**Verify positively.** After a reload, query the backend for a span category only the agent host emits. If extension telemetry arrives and that category does not, the split is present. The symptom is a whole category of spans missing while everything else looks healthy, not an error in a log. + +## Retention and deletion + +`retentionInDays` and `retention_in_days` are the deletion policy. Data ages out at that boundary and not before, and nothing in these templates purges on request. An erasure obligation for a specific person is an operator procedure run against the workspace; treat it as owned work rather than as something the template handles. + +The daily cap is the only spend guardrail. Setting it to `-1` removes it. + ## After you deploy 1. Retrieve the Application Insights connection string and put it in a secret store. Do not commit it. -2. Run the collector somewhere the fleet can reach, with `APPLICATIONINSIGHTS_CONNECTION_STRING` supplied from that secret store. Where it runs is your platform decision; Container Apps, AKS, and a VM behind a load balancer all work. +2. Run the collector somewhere the fleet can reach, with `APPLICATIONINSIGHTS_CONNECTION_STRING`, `COPILOT_OTEL_INGEST_TOKEN`, `COPILOT_OTEL_ENVIRONMENT`, and the TLS paths supplied from that secret store. Where it runs is your platform decision; Container Apps, AKS, and a VM behind a load balancer all work. 3. Distribute the endpoint through Copilot managed settings. See `references/org-distribution.md`. -4. Import `../dashboards/copilot-otel-azure.json` into Azure Monitor dashboards with Grafana and set the workspace variable to your Log Analytics resource ID. The templates provision the dashboard resource empty; this import is what fills it. -5. Confirm data landed by querying Log Analytics, not by checking that the collector reports success. +4. Deploy the per-workstation relay described above, or accept that agent-host telemetry is dropped fleet-wide. +5. Import `../dashboards/copilot-otel-azure.json` into Azure Monitor dashboards with Grafana and set the workspace variable to your Log Analytics resource ID. The templates provision the dashboard resource empty; this import is what fills it. +6. Confirm data landed by querying Log Analytics, not by checking that the collector reports success. Query for an agent-host span category specifically; extension telemetry arriving does not establish that agent-host telemetry is. ## API versions diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/deploy.sh b/.github/skills/experimental/copilot-otel-metrics/examples/azure/deploy.sh index 435960d06..8e138e672 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/deploy.sh +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/deploy.sh @@ -14,27 +14,32 @@ set -euo pipefail RESOURCE_GROUP="${RESOURCE_GROUP:-}" LOCATION="${LOCATION:-}" NAME_PREFIX="${NAME_PREFIX:-}" +ENVIRONMENT="${ENVIRONMENT:-}" RETENTION_DAYS="${RETENTION_DAYS:-90}" DAILY_QUOTA_GB="${DAILY_QUOTA_GB:-5}" usage() { cat <<'EOF' -Usage: RESOURCE_GROUP=rg LOCATION=eastus NAME_PREFIX=contoso ./deploy.sh +Usage: RESOURCE_GROUP=rg LOCATION=eastus NAME_PREFIX=contoso ENVIRONMENT=prod ./deploy.sh Required environment variables: RESOURCE_GROUP Existing resource group for the telemetry resources LOCATION Region; the dashboard must match the workspace region NAME_PREFIX 3-16 lowercase letters, digits, or hyphens + ENVIRONMENT 2-12 lowercase letters, digits, or hyphens; run this once + per environment, because one shared workspace gives every + reader every environment at once Optional: - RETENTION_DAYS Log Analytics retention, 30-730 (default 90) + RETENTION_DAYS Log Analytics retention, 30-730 (default 90). This is also + the deletion boundary; nothing purges before it DAILY_QUOTA_GB Ingestion cap in GB; -1 disables the cap (default 5) EOF } require_inputs() { local missing=0 - for var in RESOURCE_GROUP LOCATION NAME_PREFIX; do + for var in RESOURCE_GROUP LOCATION NAME_PREFIX ENVIRONMENT; do if [[ -z "${!var}" ]]; then echo "error: $var is not set" >&2 missing=1 @@ -64,9 +69,9 @@ main() { require_inputs require_tooling - local workspace_name="${NAME_PREFIX}-copilot-logs" - local insights_name="${NAME_PREFIX}-copilot-insights" - local dashboard_name="${NAME_PREFIX}-copilot-dashboard" + local workspace_name="${NAME_PREFIX}-${ENVIRONMENT}-copilot-logs" + local insights_name="${NAME_PREFIX}-${ENVIRONMENT}-copilot-insights" + local dashboard_name="${NAME_PREFIX}-${ENVIRONMENT}-copilot-dashboard" echo "Creating Log Analytics workspace ${workspace_name}" local workspace_id diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.bicep b/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.bicep index f525e89fe..f9e4557c9 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.bicep +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.bicep @@ -1,7 +1,13 @@ // Backing store and dashboard surface for GitHub Copilot fleet telemetry. // // Deploy with: -// az deployment group create -g -f main.bicep -p namePrefix= +// az deployment group create -g -f main.bicep \ +// -p namePrefix= environment= +// +// Deploy this once per environment. One workspace shared across environments +// gives every authorized reader visibility into all of them, and a resource +// attribute cannot take that back: attributes are grouping keys, not access +// boundaries. That is why environment is required rather than defaulted. // // This template does not deploy the collector. Where the collector runs // (Container Apps, AKS, a VM) depends on the organization's existing platform, @@ -17,6 +23,11 @@ @maxLength(16) param namePrefix string +@description('Environment this deployment serves. Deploy one set of resources per environment.') +@minLength(2) +@maxLength(12) +param environment string + @description('Region for all resources. The dashboard must sit in the same region as the workspace.') param location string = resourceGroup().location @@ -31,9 +42,9 @@ param dailyQuotaGb int = 5 @description('Object ID of the principal that will read telemetry. Leave empty to skip the role assignment.') param readerPrincipalId string = '' -var workspaceName = '${namePrefix}-copilot-logs' -var appInsightsName = '${namePrefix}-copilot-insights' -var dashboardName = '${namePrefix}-copilot-dashboard' +var workspaceName = '${namePrefix}-${environment}-copilot-logs' +var appInsightsName = '${namePrefix}-${environment}-copilot-insights' +var dashboardName = '${namePrefix}-${environment}-copilot-dashboard' // Monitoring Reader. Required to query Azure Monitor data from Grafana. var monitoringReaderRoleId = '43d0d8ad-25c7-4714-9337-8ba259a9fe05' @@ -41,10 +52,17 @@ var monitoringReaderRoleId = '43d0d8ad-25c7-4714-9337-8ba259a9fe05' resource workspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { name: workspaceName location: location + tags: { + environment: environment + } properties: { sku: { name: 'PerGB2018' } + // Retention is the deletion policy. Data ages out at this boundary and not + // before; nothing here purges on request. An erasure obligation for a + // specific person is an operator procedure against this workspace, not a + // property of this template. retentionInDays: retentionInDays workspaceCapping: { dailyQuotaGb: dailyQuotaGb @@ -58,6 +76,9 @@ resource appInsights 'Microsoft.Insights/components@2020-02-02' = { name: appInsightsName location: location kind: 'other' + tags: { + environment: environment + } properties: { Application_Type: 'other' WorkspaceResourceId: workspace.id @@ -82,10 +103,14 @@ resource dashboard 'Microsoft.Dashboard/dashboards@2025-08-01' = { location: location tags: { GrafanaDashboardTags: 'copilot,telemetry' + environment: environment } properties: {} } +// Opt-in and scoped to this environment's workspace only. Assigning a reader at +// subscription or resource-group scope would hand them every environment at +// once, which is the outcome the per-environment split exists to prevent. resource readerAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(readerPrincipalId)) { name: guid(workspace.id, readerPrincipalId, monitoringReaderRoleId) scope: workspace diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.tf b/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.tf index be3c1d6f9..1c0ac8a9a 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.tf +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/main.tf @@ -1,5 +1,16 @@ # Backing store and dashboard surface for GitHub Copilot fleet telemetry. # +# Apply this once per environment. One workspace shared across environments +# gives every authorized reader visibility into all of them, and a resource +# attribute cannot take that back: attributes are grouping keys, not access +# boundaries. That is why environment is a required variable with no default. +# +# The state file for this configuration contains the Application Insights +# connection string, which is a fleet-wide write credential. Marking the output +# sensitive hides it from CLI display; it does not remove it from state. Use a +# remote backend with encryption at rest and restricted access, and treat the +# state file itself as a secret. +# # This configuration does not deploy the collector. Where the collector runs # depends on the organization's existing platform, so that choice is left to # the operator. @@ -8,18 +19,27 @@ data "azurerm_resource_group" "this" { name = var.resource_group_name } +locals { + resource_prefix = "${var.name_prefix}-${var.environment}" + common_tags = merge(var.tags, { environment = var.environment }) +} + resource "azurerm_log_analytics_workspace" "this" { - name = "${var.name_prefix}-copilot-logs" + name = "${local.resource_prefix}-copilot-logs" resource_group_name = data.azurerm_resource_group.this.name location = var.location sku = "PerGB2018" - retention_in_days = var.retention_in_days - daily_quota_gb = var.daily_quota_gb - tags = var.tags + + # Retention is the deletion policy. Data ages out here and not before; + # nothing in this configuration purges on request. An erasure obligation for + # a specific person is an operator procedure against this workspace. + retention_in_days = var.retention_in_days + daily_quota_gb = var.daily_quota_gb + tags = local.common_tags } resource "azurerm_application_insights" "this" { - name = "${var.name_prefix}-copilot-insights" + name = "${local.resource_prefix}-copilot-insights" resource_group_name = data.azurerm_resource_group.this.name location = var.location workspace_id = azurerm_log_analytics_workspace.this.id @@ -29,7 +49,7 @@ resource "azurerm_application_insights" "this" { # Set this to true only when the collector runs with a managed identity. local_authentication_disabled = false - tags = var.tags + tags = local.common_tags } # Azure Monitor dashboards with Grafana. Free, in-portal, and deployable as an @@ -40,11 +60,11 @@ resource "azurerm_application_insights" "this" { # populate it; see README.md step 4. resource "azapi_resource" "dashboard" { type = "Microsoft.Dashboard/dashboards@2025-08-01" - name = "${var.name_prefix}-copilot-dashboard" + name = "${local.resource_prefix}-copilot-dashboard" parent_id = data.azurerm_resource_group.this.id location = var.location - tags = merge(var.tags, { + tags = merge(local.common_tags, { GrafanaDashboardTags = "copilot,telemetry" }) @@ -54,6 +74,10 @@ resource "azapi_resource" "dashboard" { } # Monitoring Reader. Required to query Azure Monitor data from Grafana. +# +# Opt-in and scoped to this environment's workspace only. Assigning at +# subscription or resource-group scope would hand the principal every +# environment at once, which is what the per-environment split prevents. resource "azurerm_role_assignment" "reader" { count = var.reader_principal_id == "" ? 0 : 1 diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml index da006cebc..0537aed3e 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml @@ -3,8 +3,13 @@ # Copilot's exporter sends static headers; Azure Monitor requires rotating Entra # credentials. This collector is what bridges the two, and it is not optional. # -# Supply the connection string through the environment, never in this file: +# Supply every credential and certificate path through the environment, never in +# this file: # export APPLICATIONINSIGHTS_CONNECTION_STRING="InstrumentationKey=...;IngestionEndpoint=..." +# export COPILOT_OTEL_INGEST_TOKEN="" +# export COPILOT_OTEL_ENVIRONMENT="prod" # one deployment per environment +# export COPILOT_OTEL_TLS_CERT_FILE="/etc/otel/tls/tls.crt" +# export COPILOT_OTEL_TLS_KEY_FILE="/etc/otel/tls/tls.key" # # Requires the OpenTelemetry Collector Contrib distribution; the azuremonitor # exporter is not in the core distribution. @@ -12,10 +17,28 @@ receivers: otlp: protocols: + # Authentication and TLS are active, not suggested. An unauthenticated + # plaintext receiver reachable by a fleet accepts spans from anything that + # can route to it, and every span it accepts is billed and stored. + # + # If TLS is terminated by an ingress controller or load balancer in front + # of this collector, delete the tls blocks below and record that the + # terminating hop owns certificate lifecycle and cipher policy. Do not + # delete them merely because supplying certificates is inconvenient. http: endpoint: 0.0.0.0:4318 + auth: + authenticator: bearertokenauth + tls: + cert_file: ${env:COPILOT_OTEL_TLS_CERT_FILE} + key_file: ${env:COPILOT_OTEL_TLS_KEY_FILE} grpc: endpoint: 0.0.0.0:4317 + auth: + authenticator: bearertokenauth + tls: + cert_file: ${env:COPILOT_OTEL_TLS_CERT_FILE} + key_file: ${env:COPILOT_OTEL_TLS_KEY_FILE} processors: # Bounds collector memory so a traffic spike degrades rather than crashes it. @@ -49,11 +72,20 @@ processors: # Fleet dimensions for grouping. Keep these to org structure; anything that # identifies an individual lands on every span they produce. + # + # These are labels, not boundaries. An attribute set by the collector is a + # grouping key that anyone with read access to this workspace can filter on; + # it does not stop them reading any other environment's data. Separation comes + # from deploying one collector and one workspace per environment, which is why + # the environment name is a required input rather than a default. resource/fleet: attributes: - key: service.namespace value: github-copilot action: upsert + - key: deployment.environment.name + value: ${env:COPILOT_OTEL_ENVIRONMENT} + action: upsert batch: timeout: 10s @@ -68,16 +100,39 @@ extensions: endpoint: 0.0.0.0:13133 # Copilot can only send a fixed header set, so a static bearer token is the - # available fleet authentication mechanism. Every workstation holds the same - # value: it is write-side only, but it cannot be rotated per user. Uncomment - # and add `auth: { authenticator: bearertokenauth }` to the otlp receiver - # after deciding that trade-off deliberately. + # available fleet authentication mechanism, and it is active above. Be clear + # about what it is and is not: every workstation holds the same value, so it + # authenticates the fleet rather than a person, it cannot be rotated for one + # user, and anyone who extracts it from a workstation can write spans as the + # fleet. It is write-side only, and it is still worth having, because without + # it the receiver accepts spans from anything that can reach it. + # + # This authentication is mandatory, and that has a consequence worth stating + # here rather than only in the README. Managed telemetry.headers reach the + # extension exporter but deliberately not the agent host, because delivering + # them through the environment would put this token inside every tool + # subprocess the agent spawns. So the agent host exports without a + # credential, this receiver answers 401 or UNAUTHENTICATED, and OTLP treats + # neither as retryable: that telemetry is dropped, not queued. + # + # Do not resolve that by exporting the token into the environment, and do not + # resolve it by removing this authenticator. Run a loopback relay on each + # workstation that accepts the unauthenticated local export and adds this + # credential upstream. See README.md, "Agent-host telemetry needs a relay". # - # bearertokenauth: - # token: ${env:COPILOT_OTEL_INGEST_TOKEN} + # Mutual TLS would authenticate each workstation rather than the fleet, and + # this receiver can already require it: adding client_ca_file beside + # cert_file and key_file in the otlp receiver selects + # RequireAndVerifyClientCert on both protocols. It is not enabled because + # whether the VS Code agent host can present a client certificate is + # unestablished, and configuring a receiver to require one the sender cannot + # supply would replace a documented gap with fleet-wide data loss. + bearertokenauth: + scheme: Bearer + token: ${env:COPILOT_OTEL_INGEST_TOKEN} service: - extensions: [health_check] + extensions: [health_check, bearertokenauth] pipelines: traces: receivers: [otlp] diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/outputs.tf b/.github/skills/experimental/copilot-otel-metrics/examples/azure/outputs.tf index e973b6792..545ee5c21 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/outputs.tf +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/outputs.tf @@ -14,7 +14,7 @@ output "dashboard_id" { } output "connection_string" { - description = "Connection string the collector needs. Treat as a fleet-wide write credential." + description = "Connection string the collector needs. Treat as a fleet-wide write credential. Marking this sensitive hides it from CLI output but does not remove it from the state file; protect the state accordingly." value = azurerm_application_insights.this.connection_string sensitive = true } diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/variables.tf b/.github/skills/experimental/copilot-otel-metrics/examples/azure/variables.tf index e3d8dd912..207acd328 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/variables.tf +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/variables.tf @@ -8,6 +8,16 @@ variable "name_prefix" { } } +variable "environment" { + description = "Environment this deployment serves. Apply one set of resources per environment; a shared workspace gives every reader all environments at once." + type = string + + validation { + condition = can(regex("^[a-z0-9-]{2,12}$", var.environment)) + error_message = "The environment value must be 2 to 12 lowercase letters, digits, or hyphens." + } +} + variable "resource_group_name" { description = "Existing resource group that will hold the telemetry resources." type = string @@ -19,7 +29,7 @@ variable "location" { } variable "retention_in_days" { - description = "Log Analytics retention in days. This is a cost decision, not a default." + description = "Log Analytics retention in days. This is a cost decision, not a default, and it is also the deletion boundary: nothing purges before it." type = number default = 90 @@ -36,7 +46,7 @@ variable "daily_quota_gb" { } variable "reader_principal_id" { - description = "Object ID of the principal that will read telemetry. Empty skips the role assignment." + description = "Object ID of the principal that will read telemetry. Empty skips the role assignment. The assignment is scoped to this environment's workspace only." type = string default = "" } diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/baseline.py b/.github/skills/experimental/copilot-otel-metrics/examples/baseline.py index 4dbbc99fc..b9c711536 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/baseline.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/baseline.py @@ -11,7 +11,8 @@ baseline.py capture write a snapshot baseline.py diff compare the store against the snapshot -The snapshot defaults to a user cache path. Set COPILOT_OTEL_BASELINE to override. +The snapshot defaults to a user cache path. Set COPILOT_OTEL_BASELINE to override +it with another path inside that same cache directory. """ from __future__ import annotations @@ -25,12 +26,23 @@ import urllib.parse import urllib.request +from _input_policy import PolicyError, contain_path, open_url + PROM = "http://localhost:9090" TEMPO = "http://localhost:3200" -SNAPSHOT = pathlib.Path( - os.environ.get("COPILOT_OTEL_BASELINE") - or pathlib.Path.home() / ".cache" / "copilot-otel" / "pre-enable-baseline.json" -) + +# The snapshot is written, so the override is contained to the cache root. +# An unconstrained path here turns a read-only diagnostic into an arbitrary +# file write driven by an environment variable. +CACHE_ROOT = pathlib.Path.home() / ".cache" / "copilot-otel" +try: + SNAPSHOT = ( + contain_path(os.environ["COPILOT_OTEL_BASELINE"], CACHE_ROOT) + if os.environ.get("COPILOT_OTEL_BASELINE") + else CACHE_ROOT / "pre-enable-baseline.json" + ) +except PolicyError as exc: + sys.exit(str(exc)) # Copilot metrics that a synthetic sender is unlikely to fabricate, because they # require real editor activity to exist at all. Any of these appearing after the @@ -59,7 +71,7 @@ def api(base: str, path: str, params: dict | None = None) -> dict: url = f"{base}{path}" if params: url += "?" + urllib.parse.urlencode(params, doseq=True) - with urllib.request.urlopen(url, timeout=20) as resp: + with open_url(url, timeout=20) as resp: return json.load(resp) @@ -72,7 +84,11 @@ def label_values(label: str) -> list[str]: def tempo_trace_names() -> list[str]: try: - res = api(TEMPO, "/api/search", {"q": '{resource.service.name="copilot-chat"}', "limit": "50"}) + res = api( + TEMPO, + "/api/search", + {"q": '{resource.service.name="copilot-chat"}', "limit": "50"}, + ) return sorted({t.get("rootTraceName", "") for t in (res.get("traces") or [])}) except Exception: # noqa: BLE001 return [] @@ -147,7 +163,10 @@ def do_diff() -> int: print(f" New service_version not used by synthetic payloads: {real_version}") return 0 if new_metrics or new_sessions or new_traces: - print(" INCONCLUSIVE: something new arrived, but nothing uniquely attributable to the extension.") + print( + " INCONCLUSIVE: something new arrived, but nothing uniquely " + "attributable to the extension." + ) print(" Treat as not yet confirmed and re-check after more editor activity.") return 1 print(" NOT CONFIRMED: nothing new since baseline.") @@ -156,7 +175,9 @@ def do_diff() -> int: def main() -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.add_argument("action", choices=["capture", "diff"]) args = parser.parse_args() return do_capture() if args.action == "capture" else do_diff() diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml index 651fbf325..4e13e5cb1 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml @@ -1,4 +1,27 @@ -# Local Copilot OTel stack: Grafana, Prometheus, Tempo, Loki, Pyroscope in one container. +# Local Copilot OTel stack: a filtering Collector in front of Grafana, +# Prometheus, Tempo, Loki, and Pyroscope. +# +# Copilot exports to the Collector, never to LGTM directly. The Collector +# applies the fail-closed attribute allow-list in otel-collector-local.yaml and +# is the only OTLP listener published on the host, so nothing reaches storage +# without passing that policy. LGTM's own OTLP ports are deliberately not +# mapped: publishing them would let any local process bypass the filter, which +# loopback binding alone does not prevent. +# +# Both images are pinned by manifest digest rather than tag. A tag is mutable, +# so a tag-pinned stack can change underneath a reviewed configuration. +# +# Grafana credentials are required, not defaulted. Set them before the first +# up; the stack refuses to start otherwise. Read the password rather than +# typing it inline, so it does not land in the terminal scrollback or the +# shell history file: +# export COPILOT_OTEL_GRAFANA_USER=admin +# read -rs -p 'Grafana password: ' COPILOT_OTEL_GRAFANA_PASSWORD; echo +# export COPILOT_OTEL_GRAFANA_PASSWORD +# +# In PowerShell, use Read-Host -AsSecureString for the same reason. A .env file +# reaches these containers but not the bundled Python helpers, which read the +# same two variables from the environment of the shell that runs them. # # The data volume is declared external so this file binds the existing # copilot-otel-data volume rather than creating a project-prefixed duplicate, @@ -8,19 +31,52 @@ # docker volume create copilot-otel-data services: + otel-collector: + # otel/opentelemetry-collector-contrib:0.158.0 + image: otel/opentelemetry-collector-contrib@sha256:c5918f78992ee73b0d6f0e599423ac5ec52dd5d9726733114d6eca53d5a32ed5 + container_name: copilot-otel-collector + restart: unless-stopped + command: ["--config=/etc/otelcol-contrib/config.yaml"] + ports: + # The only OTLP ingress for the stack, loopback only. + - "127.0.0.1:4317:4317" # OTLP gRPC + - "127.0.0.1:4318:4318" # OTLP HTTP <- Copilot exports here + volumes: + - ./otel-collector-local.yaml:/etc/otelcol-contrib/config.yaml:ro + networks: + - copilot-otel + depends_on: + lgtm: + condition: service_healthy + lgtm: - image: grafana/otel-lgtm:0.29.2 + # grafana/otel-lgtm:0.29.2 + image: grafana/otel-lgtm@sha256:af7242c1a9608faf6d26e6f235392fd0c32b67258228f9a3cfc96e724974930c container_name: copilot-otel-lgtm restart: unless-stopped ports: - # Loopback only. Grafana ships with default admin credentials and this - # stack is single-machine, so nothing needs to be reachable off-host. + # Read surfaces only, loopback only. There is deliberately no 4317 or + # 4318 mapping here; OTLP arrives from the Collector over the internal + # network instead. - "127.0.0.1:3000:3000" # Grafana UI - - "127.0.0.1:4317:4317" # OTLP gRPC - - "127.0.0.1:4318:4318" # OTLP HTTP <- Copilot exports here - "127.0.0.1:9090:9090" # Prometheus - "127.0.0.1:3200:3200" # Tempo + networks: + - copilot-otel environment: + # This image enables anonymous Grafana access with the Admin role and + # hides the login form. Left alone, every panel and every stored prompt + # fragment is readable by anything that can reach port 3000 without + # presenting a credential. All three flags are needed: disabling + # anonymous access without restoring the login form leaves no way in. + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_AUTH_DISABLE_LOGIN_FORM: "false" + GF_AUTH_BASIC_ENABLED: "true" + # Required, with no default. The :? form fails the up with this message + # rather than silently starting on a guessable credential, and it is the + # same pair validate_dashboard.py reads. + GF_SECURITY_ADMIN_USER: ${COPILOT_OTEL_GRAFANA_USER:?set COPILOT_OTEL_GRAFANA_USER before starting the stack} + GF_SECURITY_ADMIN_PASSWORD: ${COPILOT_OTEL_GRAFANA_PASSWORD:?set COPILOT_OTEL_GRAFANA_PASSWORD before starting the stack} # Two flags, space separated, appended to the Prometheus command line. # # otlp-deltatocumulative is applied defensively. Prometheus drops @@ -47,6 +103,12 @@ services: retries: 10 start_period: 30s +networks: + # Explicit rather than implicit so the LGTM OTLP listener is reachable from + # the Collector and from nothing else. + copilot-otel: + driver: bridge + volumes: copilot-otel-data: external: true diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py b/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py index 4ea66cad9..ec1e09f94 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py @@ -9,58 +9,80 @@ Reports every copilot_chat and gen_ai series with its labels and current value, the resource identity, and the Copilot services currently present in Tempo. + +Every request goes through the shared policy module rather than raw urllib. The +endpoints below are loopback constants today, so nothing is currently refused +that was previously allowed; routing them through the chokepoint means a later +endpoint change inherits the scheme, authority, port, redirect, and +remote-opt-in rules instead of quietly escaping them. """ from __future__ import annotations import json -import time import urllib.parse -import urllib.request + +from _input_policy import open_url PROM = "http://localhost:9090" TEMPO = "http://localhost:3200" +COPILOT_SERVICES = ("copilot-chat", "github-copilot", "claude-code") + def api(base: str, path: str, params: dict | None = None) -> dict: url = f"{base}{path}" if params: url += "?" + urllib.parse.urlencode(params, doseq=True) - with urllib.request.urlopen(url, timeout=20) as resp: + with open_url(url, timeout=20) as resp: return json.load(resp) -names = api(PROM, "/api/v1/label/__name__/values")["data"] -copilot = sorted(n for n in names if n.startswith(("copilot_chat", "gen_ai"))) -now = int(time.time()) - -print("=== real Copilot metrics: series, labels, values ===\n") -for n in copilot: - res = api(PROM, "/api/v1/query", {"query": n}).get("data", {}).get("result", []) - if not res: - print(f"{n}\n (no current samples)") - continue - print(f"{n} [{len(res)} series]") - for r in res[:4]: - labels = {k: v for k, v in r["metric"].items() if k not in ("__name__", "job", "instance")} - print(f" {r['value'][1]:>12} {labels}") - if len(res) > 4: - print(f" ... {len(res) - 4} more") - print() - -print("=== resource identity (target_info) ===") -for r in api(PROM, "/api/v1/query", {"query": "target_info"}).get("data", {}).get("result", []): - m = r["metric"] - if "copilot" in m.get("service_name", ""): - print(f" {dict(sorted((k, v) for k, v in m.items() if k != '__name__'))}") - -print("\n=== tempo: copilot services ===") -for svc in ("copilot-chat", "github-copilot", "claude-code"): - try: - res = api(TEMPO, "/api/search", {"q": f'{{resource.service.name="{svc}"}}', "limit": "5"}) - traces = res.get("traces") or [] - print(f" {svc:<16} {len(traces)} traces") - for t in traces[:3]: - print(f" {t.get('rootTraceName')} {t.get('durationMs')}ms {t.get('traceID')[:16]}") - except Exception as exc: # noqa: BLE001 - print(f" {svc:<16} error: {exc}") +def main() -> int: + names = api(PROM, "/api/v1/label/__name__/values")["data"] + copilot = sorted(n for n in names if n.startswith(("copilot_chat", "gen_ai"))) + + print("=== real Copilot metrics: series, labels, values ===\n") + for n in copilot: + res = api(PROM, "/api/v1/query", {"query": n}).get("data", {}).get("result", []) + if not res: + print(f"{n}\n (no current samples)") + continue + print(f"{n} [{len(res)} series]") + for r in res[:4]: + labels = { + k: v for k, v in r["metric"].items() if k not in ("__name__", "job", "instance") + } + print(f" {r['value'][1]:>12} {labels}") + if len(res) > 4: + print(f" ... {len(res) - 4} more") + print() + + print("=== resource identity (target_info) ===") + for r in api(PROM, "/api/v1/query", {"query": "target_info"}).get("data", {}).get("result", []): + m = r["metric"] + if "copilot" in m.get("service_name", ""): + print(f" {dict(sorted((k, v) for k, v in m.items() if k != '__name__'))}") + + print("\n=== tempo: copilot services ===") + for svc in COPILOT_SERVICES: + try: + res = api( + TEMPO, "/api/search", {"q": f'{{resource.service.name="{svc}"}}', "limit": "5"} + ) + traces = res.get("traces") or [] + print(f" {svc:<16} {len(traces)} traces") + for t in traces[:3]: + print( + f" {t.get('rootTraceName')} {t.get('durationMs')}ms " + f"{t.get('traceID')[:16]}" + ) + except Exception as exc: # noqa: BLE001 + print(f" {svc:<16} error: {exc}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml new file mode 100644 index 000000000..7c1779db0 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml @@ -0,0 +1,170 @@ +# OpenTelemetry Collector configuration for the local Copilot stack. +# +# This Collector is the only OTLP listener published on the host. It sits in +# front of the LGTM container, which has no host OTLP mapping, so nothing +# reaches storage without passing through the policy declared here. +# +# The policy is fail-closed: allowed_keys is an allow-list, so an attribute +# this configuration has never heard of is dropped rather than stored. That +# matters because the emitter's attribute set is not a stable contract. A +# future extension release can add a new prompt-bearing attribute, and a +# delete-list would silently pass it through while this allow-list will not. +# +# Requires the OpenTelemetry Collector Contrib distribution; the redaction +# processor is not in the core distribution. + +receivers: + otlp: + protocols: + # Bound to the container interface. Host exposure is decided by the + # loopback port mappings in compose.yaml, not here. + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + # Bounds Collector memory so a traffic spike degrades rather than crashes it. + memory_limiter: + check_interval: 1s + limit_percentage: 80 + spike_limit_percentage: 20 + + # The entire minimization control for the local stack. + # + # allow_all_keys: false makes allowed_keys authoritative. Every key below + # exists because a shipped dashboard panel or a datasource lookup reads it. + # Adding a key here is a deliberate decision to store that attribute; the + # content attributes that were observed carrying plaintext are absent by + # construction rather than by enumeration. + # + # blocked_values is a second, independent pass over the values that survive. + # An allowed key can still carry a credential pasted into a model name or a + # tool argument, so recognizable secret shapes are masked rather than stored. + redaction: + allow_all_keys: false + allowed_keys: + # Resource identity. Dropping these would orphan every span. + - service.name + - service.namespace + - service.version + - telemetry.sdk.language + - telemetry.sdk.name + - telemetry.sdk.version + # Model and token accounting. The token panels read these directly. + - gen_ai.provider.name + - gen_ai.operation.name + - gen_ai.request.model + - gen_ai.response.model + - gen_ai.response.finish_reasons + - gen_ai.usage.input_tokens + - gen_ai.usage.output_tokens + - gen_ai.usage.cache_read.input_tokens + # The token panels group by this. Prometheus stores it as the + # gen_ai_token_type label; dropping it collapsed input and output tokens + # into one indistinguishable series. + - gen_ai.token.type + # Read by baseline.py through the session_id label. It is the + # discriminator that distinguishes injected series from real ones, so + # dropping it disabled a detection control the threat model credits. + - session.id + # Agent and tool identity. Names only; arguments and results are not + # allowed and therefore cannot reach storage. + - gen_ai.agent.name + - gen_ai.tool.name + - gen_ai.tool.type + - copilot_chat.mode_name + - copilot_chat.turn_count + - copilot_chat.copilot_usage_nano_aiu + - github.copilot.agent.type + # The edit-acceptance panel groups by the copilot_chat_edit_source label. + # Which OTLP key produces that label is not established: normalization to + # a Prometheus label is not reversible, and both spellings below collapse + # to it. Allow-listing both means a wrong single guess cannot silently + # empty the panel. Settling this needs live emitter output, and the gap + # is recorded in SECURITY.md rather than guessed away here. + - copilot_chat.edit_source + - copilot_chat.edit.source + # Failure triage without failure content. + - error.type + # Masked, not dropped, so a panel keeps its shape while the value is gone. + blocked_values: + - "gh[pousr]_[0-9A-Za-z]{16,}" + - "github_pat_[0-9A-Za-z_]{20,}" + - "xox[baprs]-[0-9A-Za-z-]{10,}" + - "AKIA[0-9A-Z]{16}" + - "eyJ[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]{10,}" + - "(?i)-----BEGIN [A-Z ]*PRIVATE KEY-----" + - "(?i)\\b(?:bearer|authorization|api[_-]?key|secret|password)\\b\\s*[:=]\\s*\\S+" + # Silent so the processor does not annotate telemetry with the names of the + # keys it removed, which would reintroduce the drift it exists to stop. + summary: silent + + # Everything the redaction processor cannot reach. + # + # redaction traverses attribute maps. Running the pinned Collector against + # payloads that placed a distinct marker in every OTLP carrier showed which + # carriers that leaves open: span status messages, span event names, trace + # state, and every log body shape except a map all reached the exporter + # verbatim. An allow-list that never sees a field cannot fail closed on it, + # so those carriers are overwritten here. + # + # The rule is: a carrier the allow-list does not reach is scrubbed unless a + # shipped consumer reads it. That is why span names are absent from this + # list even though they are the carrier most likely to hold prompt text. + # Three dashboard queries match on span name and baseline.py reads Tempo's + # rootTraceName, so scrubbing them would break shipped functionality. Metric + # names, descriptions, and units are absent for the same reason. Those + # residual exposures are recorded in SECURITY.md rather than hidden here. + # + # Span link attributes, span link trace state, and metric exemplar filtered + # attributes are also open and are also absent, but for a different reason: + # this distribution provides no way to reach them. OTTL has no `spanlink` + # context and refuses to index `links`, and assigning to `datapoint.exemplars` + # parses but does nothing. Both are recorded as gaps. + transform/scrub: + error_mode: ignore + trace_statements: + - set(span.status.message, "[redacted]") where span.status.message != "" + - set(span.trace_state, "") where span.trace_state != "" + - set(spanevent.name, "[redacted]") + log_statements: + - set(log.body, "[redacted]") where log.body != nil + - set(log.severity_text, "[redacted]") where log.severity_text != "" + - set(log.event_name, "[redacted]") where log.event_name != "" + + batch: + timeout: 10s + send_batch_size: 1024 + +exporters: + # The LGTM container is reachable only on the internal Compose network. TLS + # is not used on that hop and the insecure flag says so explicitly rather + # than leaving it to a default. + # + # `otlp_grpc` rather than `otlp`: the pinned Collector logs a deprecation + # warning for the `otlp` exporter alias once per signal at every start. + otlp_grpc/lgtm: + endpoint: lgtm:4317 + tls: + insecure: true + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, redaction, transform/scrub, batch] + exporters: [otlp_grpc/lgtm] + metrics: + receivers: [otlp] + processors: [memory_limiter, redaction, transform/scrub, batch] + exporters: [otlp_grpc/lgtm] + logs: + receivers: [otlp] + processors: [memory_limiter, redaction, transform/scrub, batch] + exporters: [otlp_grpc/lgtm] diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py new file mode 100644 index 000000000..045e53377 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Reversible, audited upsert of Copilot OTel settings into a JSONC settings file. + +The skill's written rules for this edit were strong and entirely advisory: an +agent could describe a careful diff-and-confirm write and then perform any edit +at all. This makes the rules executable. Every key, type, and value combination +is checked against a schema before a byte is written, non-target bytes are +preserved exactly, the previous file is backed up, and a failure to parse the +result restores the backup. + + settings_upsert.py --settings --set key=value [--set ...] # dry run + settings_upsert.py --settings --set key=value --apply # write + +Schema provenance is recorded in SCHEMA_SOURCE. Re-verify it against the +installed build before trusting this tool on a newer extension; the settings +surface follows the extension, not this file. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import pathlib +import shutil +import sys +import urllib.parse + +from _input_policy import DEFAULT_ALLOWED_PORTS, PolicyError, check_url, contain_path + +# Provenance for the schema below. This is the authoritative manifest that was +# read, not a recollection of one. +SCHEMA_SOURCE = { + "artifact": "GitHub Copilot Chat extension manifest (package.json)", + "extension": "GitHub.copilot-chat", + "version": "0.52.0", + "retrieved_from": "contributes.configuration[].properties of the installed build", + "verified": "2026-08-13", +} + +# The OTel settings this build declares, with their declared types. Seven, not +# the eleven earlier revisions of this skill documented: protocol, headers, +# serviceName, and resourceAttributes are absent from this manifest. A key that +# is not here is refused rather than written, because writing a key the build +# does not declare produces a setting that is silently inert. +SCHEMA: dict[str, type | tuple[type, ...]] = { + "github.copilot.chat.otel.enabled": bool, + "github.copilot.chat.otel.exporterType": str, + "github.copilot.chat.otel.otlpEndpoint": str, + "github.copilot.chat.otel.captureContent": bool, + "github.copilot.chat.otel.maxAttributeSizeChars": int, + "github.copilot.chat.otel.outfile": str, + "github.copilot.chat.otel.dbSpanExporter.enabled": bool, +} + +EXPORTER_TYPES = frozenset({"otlp-grpc", "otlp-http", "console", "file"}) + +# Keys whose value can never be summarized safely in an audit record. +SENSITIVE_KEYS = frozenset({"github.copilot.chat.otel.outfile"}) + +# Keys whose value is a URL. The audit record keeps the authority so an +# operator can tell which destination was configured, and drops the path, +# query, and fragment, which is where a token or an identifier would sit. +ENDPOINT_KEYS = frozenset({"github.copilot.chat.otel.otlpEndpoint"}) + + +class SettingsError(ValueError): + """Raised when a requested edit is refused. Nothing has been written.""" + + +def parse_assignment(raw: str) -> tuple[str, object]: + """Split a key=value argument and coerce the value to its declared type.""" + key, separator, text = raw.partition("=") + if not separator: + raise SettingsError(f"expected key=value, got '{raw}'") + key = key.strip() + if key not in SCHEMA: + raise SettingsError( + f"refusing unknown key '{key}'. This build declares: {', '.join(sorted(SCHEMA))}" + ) + + expected = SCHEMA[key] + text = text.strip() + if expected is bool: + if text not in ("true", "false"): + raise SettingsError(f"{key} is boolean; expected true or false, got '{text}'") + return key, text == "true" + if expected is int: + try: + return key, int(text) + except ValueError as exc: + raise SettingsError(f"{key} is integer; got '{text}'") from exc + return key, text + + +def check_policy(values: dict[str, object]) -> None: + """Apply the local-mode rules that the schema alone cannot express.""" + if values.get("github.copilot.chat.otel.captureContent") is True: + raise SettingsError( + "refusing to enable captureContent. It puts prompt text, responses, system " + "instructions, and tool arguments onto spans. Set it by hand, deliberately." + ) + + exporter = values.get("github.copilot.chat.otel.exporterType") + if exporter is not None and exporter not in EXPORTER_TYPES: + raise SettingsError( + f"exporterType must be one of {', '.join(sorted(EXPORTER_TYPES))}, got '{exporter}'" + ) + + endpoint = values.get("github.copilot.chat.otel.otlpEndpoint") + if endpoint: + try: + check_url(str(endpoint), allow_remote=False, allowed_ports=DEFAULT_ALLOWED_PORTS) + except PolicyError as exc: + raise SettingsError(f"otlpEndpoint rejected: {exc}") from exc + + outfile = values.get("github.copilot.chat.otel.outfile") + if outfile and exporter is not None and exporter.startswith("otlp-"): + raise SettingsError( + "outfile forces the file exporter, which contradicts the requested " + f"exporterType '{exporter}'. Set one or the other." + ) + + size = values.get("github.copilot.chat.otel.maxAttributeSizeChars") + if size is not None and int(size) < 0: + raise SettingsError("maxAttributeSizeChars cannot be negative") + + +def strip_jsonc(text: str) -> str: + """Return the text with comments and trailing commas blanked out. + + Offsets are preserved so the result can be parsed for validation while the + spans found by the scanner still line up with the original bytes. Nothing + is removed; every blanked byte becomes a space in the same position. + + Trailing commas are handled here rather than by a regex over the whole + document. A regex cannot tell a comma inside a string literal from a + structural one, and the untouched-key comparison that guards this edit + could not catch the difference, because both sides of that comparison would + be produced by the same corrupting pass. + """ + out = list(text) + index, length = 0, len(text) + # Position of the most recent structural comma with only blank output + # between it and the current cursor. A trailing comma is one that is still + # a candidate when a closing bracket arrives. + comma_candidate: int | None = None + while index < length: + char = text[index] + if char == '"': + comma_candidate = None + index += 1 + while index < length: + if text[index] == "\\": + index += 2 + continue + if text[index] == '"': + break + index += 1 + index += 1 + continue + if char == "/" and index + 1 < length: + if text[index + 1] == "/": + while index < length and text[index] != "\n": + out[index] = " " + index += 1 + continue + if text[index + 1] == "*": + while index < length and not ( + text[index] == "*" and text[index + 1 : index + 2] == "/" + ): + if text[index] != "\n": + out[index] = " " + index += 1 + for offset in range(index, min(index + 2, length)): + out[offset] = " " + index += 2 + continue + if char == ",": + comma_candidate = index + elif char in "}]": + if comma_candidate is not None: + out[comma_candidate] = " " + comma_candidate = None + elif not char.isspace(): + comma_candidate = None + index += 1 + return "".join(out) + + +def top_level_spans(text: str) -> dict[str, tuple[int, int]]: + """Map each top-level key to the (start, end) offsets of its value.""" + scan = strip_jsonc(text) + spans: dict[str, tuple[int, int]] = {} + index, length, depth = 0, len(scan), 0 + while index < length: + char = scan[index] + if char == '"': + start = index + 1 + index += 1 + while index < length: + if scan[index] == "\\": + index += 2 + continue + if scan[index] == '"': + break + index += 1 + key = scan[start:index] + index += 1 + if depth == 1: + cursor = index + while cursor < length and scan[cursor] in " \t\r\n": + cursor += 1 + if cursor < length and scan[cursor] == ":": + cursor += 1 + while cursor < length and scan[cursor] in " \t\r\n": + cursor += 1 + value_start = cursor + value_depth = 0 + while cursor < length: + current = scan[cursor] + if current == '"': + cursor += 1 + while cursor < length: + if scan[cursor] == "\\": + cursor += 2 + continue + if scan[cursor] == '"': + break + cursor += 1 + elif current in "{[": + value_depth += 1 + elif current in "}]": + if value_depth == 0: + break + value_depth -= 1 + elif current == "," and value_depth == 0: + break + cursor += 1 + spans[key] = (value_start, len(scan[:cursor].rstrip())) + index = cursor + continue + if char in "{[": + depth += 1 + elif char in "}]": + depth -= 1 + index += 1 + return spans + + +def upsert(text: str, values: dict[str, object]) -> str: + """Return the text with each key set, changing nothing else.""" + result = text + for key, value in values.items(): + encoded = json.dumps(value) + spans = top_level_spans(result) + if key in spans: + start, end = spans[key] + result = result[:start] + encoded + result[end:] + continue + closing = result.rstrip().rfind("}") + if closing == -1: + raise SettingsError("settings file has no top-level object to edit") + head = result[:closing].rstrip() + # A document that already ends its last entry with a trailing comma + # needs no second one; emitting both produces `,,`, which no amount of + # trailing-comma tolerance makes valid. + separator = "" if head.endswith(("{", ",")) else "," + result = f'{head}{separator}\n "{key}": {encoded}\n{result[closing:]}' + return result + + +def summarize(key: str, value: object) -> object: + """Return an audit-safe rendering of a value. + + The record exists to show what changed, not to become a second copy of the + settings. A path is reduced to its shape and an endpoint to its authority, + so the record stays useful without retaining more than it needs. + """ + if value is None: + return None + if key in SENSITIVE_KEYS: + return f"<{type(value).__name__} of length {len(str(value))}>" + if key in ENDPOINT_KEYS and isinstance(value, str): + return summarize_endpoint(value) + return value + + +def summarize_endpoint(value: str) -> str: + """Reduce a URL to scheme and authority. + + An OTLP endpoint's path and query are operator-supplied and can carry a + tenant identifier or a token. Neither is needed to answer the question the + record is kept for, which is where telemetry was pointed. + """ + parsed = urllib.parse.urlparse(value) + if not parsed.scheme or not parsed.netloc: + return f"" + return f"{parsed.scheme}://{parsed.netloc}" + + +def write_audit(audit_path: pathlib.Path, record: dict[str, object]) -> None: + """Append one audit line. Values are summarized, never copied verbatim.""" + audit_path.parent.mkdir(parents=True, exist_ok=True) + with audit_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + + +def backup_path_for( + settings_path: pathlib.Path, *, now: datetime.datetime | None = None +) -> pathlib.Path: + """Return a backup path that cannot overwrite an existing backup. + + A fixed `.bak` name makes the second run destroy the only copy of the + original, which is the opposite of what a backup is for. The timestamp is + to the second and a counter resolves same-second collisions, so repeated + runs accumulate rather than overwrite. + + Retention is the operator's: this tool never deletes a backup it wrote. + """ + stamp = (now or datetime.datetime.now(datetime.UTC)).strftime("%Y%m%dT%H%M%SZ") + candidate = settings_path.with_name(f"{settings_path.name}.{stamp}.bak") + collision = 1 + while candidate.exists(): + candidate = settings_path.with_name(f"{settings_path.name}.{stamp}-{collision}.bak") + collision += 1 + return candidate + + +def apply_changes( + settings_path: pathlib.Path, + values: dict[str, object], + *, + apply: bool, + audit_path: pathlib.Path | None = None, +) -> str: + """Validate, then optionally write, restoring the backup if the result is invalid.""" + check_policy(values) + + original = settings_path.read_text(encoding="utf-8") if settings_path.is_file() else "{}\n" + try: + before = json.loads(strip_jsonc(original) or "{}") + except json.JSONDecodeError as exc: + raise SettingsError(f"existing settings file is not valid JSONC: {exc}") from exc + + updated = upsert(original, values) + + try: + after = json.loads(strip_jsonc(updated) or "{}") + except json.JSONDecodeError as exc: + raise SettingsError(f"refusing to write: the result would not parse ({exc})") from exc + + untouched_before = {k: v for k, v in before.items() if k not in values} + untouched_after = {k: v for k, v in after.items() if k not in values} + if untouched_before != untouched_after: + raise SettingsError("refusing to write: the edit would change unrelated settings") + for key, value in values.items(): + if after.get(key) != value: + raise SettingsError(f"refusing to write: {key} did not take the requested value") + + if not apply: + return updated + + backup_path = backup_path_for(settings_path) + if settings_path.is_file(): + shutil.copy2(settings_path, backup_path) + try: + settings_path.write_text(updated, encoding="utf-8") + json.loads(strip_jsonc(settings_path.read_text(encoding="utf-8")) or "{}") + except Exception: + if backup_path.is_file(): + shutil.copy2(backup_path, settings_path) + raise + + if audit_path is not None: + write_audit( + audit_path, + { + "timestamp": datetime.datetime.now(datetime.UTC).isoformat(), + "settings_file": str(settings_path), + "backup_file": str(backup_path), + "schema_version": SCHEMA_SOURCE["version"], + "changes": { + key: { + "from": summarize(key, before.get(key)), + "to": summarize(key, value), + } + for key, value in values.items() + }, + }, + ) + return updated + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--settings", required=True, help="path to the settings.json to edit") + parser.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + parser.add_argument("--apply", action="store_true", help="write; omit for a dry run") + parser.add_argument("--audit", help="append an audit record to this path") + args = parser.parse_args(argv) + + try: + values: dict[str, object] = {} + for assignment in args.set: + key, value = parse_assignment(assignment) + if key in values: + raise SettingsError(f"duplicate key '{key}'") + values[key] = value + if not values: + raise SettingsError("nothing to do: pass at least one --set key=value") + + settings_path = pathlib.Path(args.settings).expanduser() + audit_path = contain_path(args.audit, settings_path.parent) if args.audit else None + updated = apply_changes(settings_path, values, apply=args.apply, audit_path=audit_path) + except (SettingsError, PolicyError) as exc: + print(str(exc), file=sys.stderr) + return 1 + + if not args.apply: + print(updated) + print("\n(dry run; pass --apply to write)", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py b/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py index 6d4189544..df49671d4 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py @@ -16,8 +16,17 @@ uid. It therefore refuses a non-loopback Grafana unless the target is confirmed disposable via COPILOT_OTEL_ALLOW_REMOTE=1. -Environment overrides: COPILOT_OTEL_GRAFANA, COPILOT_OTEL_GRAFANA_USER, -COPILOT_OTEL_GRAFANA_PASSWORD, COPILOT_OTEL_PROMETHEUS, COPILOT_OTEL_TEMPO. +Environment overrides: COPILOT_OTEL_GRAFANA, COPILOT_OTEL_PROMETHEUS, +COPILOT_OTEL_TEMPO. + +Required: COPILOT_OTEL_GRAFANA_USER, COPILOT_OTEL_GRAFANA_PASSWORD. These are +the same variables compose.yaml requires; there is no default credential. + +The Grafana credential is passed to the one request that needs it rather than +held in a module global. Five of this tool's six requests go to Prometheus or +Tempo, neither of which authenticates against Grafana, and a shared builder +that attaches the header unconditionally sends the credential to all six. +Making it a parameter means a call site can only send it by naming it. """ from __future__ import annotations @@ -32,123 +41,248 @@ import urllib.error import urllib.parse import urllib.request +from collections.abc import Callable + +from _input_policy import PolicyError, check_url, contain_path, open_url, require_credentials + +EXAMPLES_DIR = pathlib.Path(__file__).resolve().parent +DEFAULT_GRAFANA = "http://localhost:3000" +DEFAULT_PROMETHEUS = "http://localhost:9090" +DEFAULT_TEMPO = "http://localhost:3200" -GRAFANA = os.environ.get("COPILOT_OTEL_GRAFANA", "http://localhost:3000") -PROM = os.environ.get("COPILOT_OTEL_PROMETHEUS", "http://localhost:9090") -TEMPO = os.environ.get("COPILOT_OTEL_TEMPO", "http://localhost:3200") -USER = os.environ.get("COPILOT_OTEL_GRAFANA_USER", "admin") -PASSWORD = os.environ.get("COPILOT_OTEL_GRAFANA_PASSWORD", "admin") -AUTH = "Basic " + base64.b64encode(f"{USER}:{PASSWORD}".encode()).decode() -DASH = ( - pathlib.Path(sys.argv[1]) - if len(sys.argv) > 1 - else pathlib.Path(__file__).parent / "dashboards" / "copilot-otel.json" +TRACEQL_METRICS_FUNCTIONS = ( + "rate()", + "_over_time(", + "quantile_over_time", + "histogram_over_time", ) -_host = (urllib.parse.urlparse(GRAFANA).hostname or "").lower() -if _host not in ("localhost", "127.0.0.1", "::1") and os.environ.get("COPILOT_OTEL_ALLOW_REMOTE") != "1": - sys.exit( - f"refusing to import into {GRAFANA}: this overwrites any dashboard with the same uid.\n" - "Point COPILOT_OTEL_GRAFANA at a local stack, or set COPILOT_OTEL_ALLOW_REMOTE=1 if the target is disposable." - ) + +def basic_auth(user: str, password: str) -> str: + """Render a Basic credential for the Grafana API.""" + return "Basic " + base64.b64encode(f"{user}:{password}".encode()).decode() -def req(url: str, data: bytes | None = None, method: str = "GET") -> dict: - r = urllib.request.Request(url, data=data, method=method) - r.add_header("Authorization", AUTH) +def build_request( + url: str, + data: bytes | None = None, + method: str = "GET", + *, + authorization: str | None = None, +) -> urllib.request.Request: + """Build a request, attaching a credential only when one is supplied. + + Building is separate from sending so the header set of every call site is + inspectable without a network. + """ + request = urllib.request.Request(url, data=data, method=method) + if authorization is not None: + request.add_header("Authorization", authorization) if data: - r.add_header("Content-Type", "application/json") - with urllib.request.urlopen(r, timeout=25) as resp: - return json.load(resp) + request.add_header("Content-Type", "application/json") + return request -dash = json.loads(DASH.read_text()) +def fetch( + url: str, + data: bytes | None = None, + method: str = "GET", + *, + authorization: str | None = None, + allow_remote: bool = False, +) -> dict: + """Send a built request through the shared policy boundary and decode JSON.""" + request = build_request(url, data, method, authorization=authorization) + with open_url(request, allow_remote=allow_remote, timeout=25) as response: + return json.load(response) -# Silently falling through to Tempo for an unknown datasource would produce empty -# results indistinguishable from a genuinely empty store, which is the exact -# failure this tool exists to detect. -_types = {p["datasource"]["type"] for p in dash["panels"] if p.get("type") != "row"} -if not _types <= {"prometheus", "tempo"}: - sys.exit( - f"unsupported datasource types: {', '.join(sorted(_types - {'prometheus', 'tempo'}))}.\n" - "This tool validates Prometheus and Tempo panels only. The Azure dashboard has no supported path here." - ) -body = {"dashboard": dash, "overwrite": True, "folderId": 0, "message": "validated against live telemetry"} -res = req(f"{GRAFANA}/api/dashboards/db", json.dumps(body).encode(), "POST") -print(f"import: {res.get('status')} -> {GRAFANA}{res.get('url')}\n") - -now = int(time.time()) -print("=== panel query validation ===\n") -ok_count = empty_count = 0 - -for panel in dash["panels"]: - if panel.get("type") == "row": - print(f"-- {panel['title']}") - continue - ds = panel["datasource"]["type"] - title = panel["title"] - - for target in panel.get("targets", []): - expr = target.get("expr") or target.get("query", "") - # substitute dashboard variables with match-all for validation - expr_r = expr.replace("$model", ".*").replace("[$__range]", "[1h]") - label = target.get("legendFormat", target["refId"]) - - if ds == "prometheus": - try: - if target.get("instant"): - rows = req(f"{PROM}/api/v1/query?" + urllib.parse.urlencode({"query": expr_r}))["data"]["result"] - else: - rows = req( - f"{PROM}/api/v1/query_range?" - + urllib.parse.urlencode({"query": expr_r, "start": now - 7200, "end": now, "step": "60"}) - )["data"]["result"] - n = len(rows) - except urllib.error.HTTPError as exc: - print(f" [ERROR] {title:<44} {label:<28} {exc.read().decode()[:80]}") - continue - else: - # A TraceQL metrics query aggregates span attributes over time and - # uses a different endpoint than a search query. - is_metrics = any(fn in expr_r for fn in ("rate()", "_over_time(", "quantile_over_time", "histogram_over_time")) - try: - if is_metrics: - res = req( - f"{TEMPO}/api/metrics/query_range?" - + urllib.parse.urlencode( - {"q": expr_r, "start": (now - 7200) * 10**9, "end": now * 10**9, "step": "5m"} +def validate_panels( + dash: dict, + *, + prometheus: str, + tempo: str, + store: Callable[[str], dict], +) -> tuple[int, int]: + """Run every panel query through the credential-free transport.""" + now = int(time.time()) + print("=== panel query validation ===\n") + ok_count = empty_count = 0 + + for panel in dash["panels"]: + if panel.get("type") == "row": + print(f"-- {panel['title']}") + continue + ds = panel["datasource"]["type"] + title = panel["title"] + + for target in panel.get("targets", []): + expr = target.get("expr") or target.get("query", "") + # substitute dashboard variables with match-all for validation + expr_r = expr.replace("$model", ".*").replace("[$__range]", "[1h]") + label = target.get("legendFormat", target["refId"]) + + if ds == "prometheus": + try: + if target.get("instant"): + rows = store( + f"{prometheus}/api/v1/query?" + + urllib.parse.urlencode({"query": expr_r}) + )["data"]["result"] + else: + rows = store( + f"{prometheus}/api/v1/query_range?" + + urllib.parse.urlencode( + {"query": expr_r, "start": now - 7200, "end": now, "step": "60"} + ) + )["data"]["result"] + n = len(rows) + except urllib.error.HTTPError as exc: + print(f" [ERROR] {title:<44} {label:<28} {exc.read().decode()[:80]}") + continue + else: + # A TraceQL metrics query aggregates span attributes over time + # and uses a different endpoint than a search query. + is_metrics = any(fn in expr_r for fn in TRACEQL_METRICS_FUNCTIONS) + try: + if is_metrics: + result = store( + f"{tempo}/api/metrics/query_range?" + + urllib.parse.urlencode( + { + "q": expr_r, + "start": (now - 7200) * 10**9, + "end": now * 10**9, + "step": "5m", + } + ) ) - ) - rows = [s for s in (res.get("series") or []) if s.get("samples")] - else: - # Tempo search returns nothing without explicit bounds; Grafana always sends them. - rows = req( - f"{TEMPO}/api/search?" - + urllib.parse.urlencode( - {"q": expr_r, "limit": "20", "start": now - 7200, "end": now} + rows = [s for s in (result.get("series") or []) if s.get("samples")] + else: + # Tempo search returns nothing without explicit bounds; + # Grafana always sends them. + rows = ( + store( + f"{tempo}/api/search?" + + urllib.parse.urlencode( + {"q": expr_r, "limit": "20", "start": now - 7200, "end": now} + ) + ).get("traces") + or [] ) - ).get("traces") or [] - n = len(rows) - except urllib.error.HTTPError as exc: - print(f" [ERROR] {title:<44} {label:<28} {exc.code} {exc.read().decode()[:70]}") - continue - except Exception as exc: # noqa: BLE001 - print(f" [ERROR] {title:<44} {label:<28} {exc}") - continue - - if n: - ok_count += 1 - print(f" [DATA ] {title:<44} {label:<28} {n} series/traces") - else: - empty_count += 1 - # distinguish "metric absent" from "metric present but no samples in window" - metrics = set(re.findall(r"\b([a-z_][a-z0-9_]*(?:_total|_sum|_count|_bucket))\b", expr_r)) - missing = [] - if ds == "prometheus" and metrics: - names = req(f"{PROM}/api/v1/label/__name__/values")["data"] - missing = sorted(m for m in metrics if m not in names) - reason = f"metric name absent: {missing}" if missing else "no samples in window" - print(f" [EMPTY] {title:<44} {label:<28} {reason}") - -print(f"\nqueries with data: {ok_count} empty: {empty_count}") + n = len(rows) + except urllib.error.HTTPError as exc: + print( + f" [ERROR] {title:<44} {label:<28} " + f"{exc.code} {exc.read().decode()[:70]}" + ) + continue + except Exception as exc: # noqa: BLE001 + print(f" [ERROR] {title:<44} {label:<28} {exc}") + continue + + if n: + ok_count += 1 + print(f" [DATA ] {title:<44} {label:<28} {n} series/traces") + else: + empty_count += 1 + # distinguish "metric absent" from "metric present but no + # samples in window" + metrics = set( + re.findall(r"\b([a-z_][a-z0-9_]*(?:_total|_sum|_count|_bucket))\b", expr_r) + ) + missing = [] + if ds == "prometheus" and metrics: + names = store(f"{prometheus}/api/v1/label/__name__/values")["data"] + missing = sorted(m for m in metrics if m not in names) + reason = f"metric name absent: {missing}" if missing else "no samples in window" + print(f" [EMPTY] {title:<44} {label:<28} {reason}") + + return ok_count, empty_count + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + grafana_url = os.environ.get("COPILOT_OTEL_GRAFANA", DEFAULT_GRAFANA) + prometheus_url = os.environ.get("COPILOT_OTEL_PROMETHEUS", DEFAULT_PROMETHEUS) + tempo_url = os.environ.get("COPILOT_OTEL_TEMPO", DEFAULT_TEMPO) + allow_remote = os.environ.get("COPILOT_OTEL_ALLOW_REMOTE") == "1" + + # All three endpoints go through the same check. Previously only Grafana + # was guarded, so a redirected Prometheus or Tempo override could send + # queries anywhere while the import still looked contained. + try: + for _name, _url in ( + ("Grafana", grafana_url), + ("Prometheus", prometheus_url), + ("Tempo", tempo_url), + ): + check_url(_url, allow_remote=allow_remote) + user, password = require_credentials( + "COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD" + ) + # The import uses overwrite semantics, so a dashboard path outside this + # directory is refused rather than read. + dash_path = ( + contain_path(args[0], EXAMPLES_DIR) + if args + else EXAMPLES_DIR / "dashboards" / "copilot-otel.json" + ) + except PolicyError as exc: + print(str(exc), file=sys.stderr) + return 1 + + if not check_url(grafana_url, allow_remote=allow_remote).hostname.startswith( + ("localhost", "127.", "::1") + ): + print( + f"warning: importing into {grafana_url} overwrites any dashboard " + "sharing the same uid.", + file=sys.stderr, + ) + + authorization = basic_auth(user, password) + + def grafana(url: str, data: bytes | None = None, method: str = "GET") -> dict: + """The only transport that carries the Grafana credential.""" + return fetch(url, data, method, authorization=authorization, allow_remote=allow_remote) + + def store(url: str) -> dict: + """Prometheus and Tempo. No credential is in scope for this closure.""" + return fetch(url, allow_remote=allow_remote) + + dash = json.loads(dash_path.read_text()) + + # Silently falling through to Tempo for an unknown datasource would produce + # empty results indistinguishable from a genuinely empty store, which is the + # exact failure this tool exists to detect. + types = {p["datasource"]["type"] for p in dash["panels"] if p.get("type") != "row"} + if not types <= {"prometheus", "tempo"}: + unsupported = ", ".join(sorted(types - {"prometheus", "tempo"})) + print( + f"unsupported datasource types: {unsupported}.\n" + "This tool validates Prometheus and Tempo panels only. " + "The Azure dashboard has no supported path here.", + file=sys.stderr, + ) + return 1 + + body = { + "dashboard": dash, + "overwrite": True, + "folderId": 0, + "message": "validated against live telemetry", + } + imported = grafana(f"{grafana_url}/api/dashboards/db", json.dumps(body).encode(), "POST") + print(f"import: {imported.get('status')} -> {grafana_url}{imported.get('url')}\n") + + ok_count, empty_count = validate_panels( + dash, prometheus=prometheus_url, tempo=tempo_url, store=store + ) + print(f"\nqueries with data: {ok_count} empty: {empty_count}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/verify.py b/.github/skills/experimental/copilot-otel-metrics/examples/verify.py index 7c0ba070f..78dfa499d 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/verify.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/verify.py @@ -7,6 +7,12 @@ successful export call proves nothing. Every check here queries the store. Exits non-zero if any required check fails. + +Every request goes through the shared policy module rather than raw urllib. +The endpoints below are loopback constants today, so nothing is currently +refused that was previously allowed; routing them through the chokepoint means +a later endpoint change inherits the scheme, authority, port, redirect, and +remote-opt-in rules instead of quietly escaping them. """ from __future__ import annotations @@ -14,7 +20,8 @@ import json import time import urllib.parse -import urllib.request + +from _input_policy import open_url GRAFANA = "http://localhost:3000" PROM = "http://localhost:9090" @@ -33,7 +40,7 @@ def api(base: str, path: str, params: dict | None = None, timeout: int = 15) -> url = f"{base}{path}" if params: url += "?" + urllib.parse.urlencode(params, doseq=True) - with urllib.request.urlopen(url, timeout=timeout) as resp: + with open_url(url, timeout=timeout) as resp: return json.load(resp) @@ -44,7 +51,7 @@ def check_health() -> None: ("tempo", f"{TEMPO}/ready"), ): try: - with urllib.request.urlopen(url, timeout=10) as resp: + with open_url(url, timeout=10) as resp: record(f"{name} reachable", resp.status == 200, f"HTTP {resp.status}") except Exception as exc: # noqa: BLE001 - report, do not raise record(f"{name} reachable", False, str(exc)) @@ -65,19 +72,31 @@ def check_copilot_metrics() -> None: try: names = api(PROM, "/api/v1/label/__name__/values")["data"] copilot = [n for n in names if n.startswith(("copilot_chat", "gen_ai"))] - record("copilot metric names present", bool(copilot), f"{len(copilot)} names: {copilot[:6]}") + record( + "copilot metric names present", + bool(copilot), + f"{len(copilot)} names: {copilot[:6]}", + ) now = int(time.time()) stored = [] for n in copilot: res = ( - api(PROM, "/api/v1/query_range", {"query": n, "start": now - 3600, "end": now, "step": "60"}) + api( + PROM, + "/api/v1/query_range", + {"query": n, "start": now - 3600, "end": now, "step": "60"}, + ) .get("data", {}) .get("result", []) ) if res: stored.append(n) - record("copilot metrics have samples", bool(stored), f"{len(stored)} of {len(copilot)} with data in last hour") + record( + "copilot metrics have samples", + bool(stored), + f"{len(stored)} of {len(copilot)} with data in last hour", + ) except Exception as exc: # noqa: BLE001 record("copilot metric names present", False, str(exc)) @@ -86,7 +105,11 @@ def check_copilot_traces() -> None: found = {} for svc in COPILOT_SERVICES: try: - res = api(TEMPO, "/api/search", {"q": f'{{resource.service.name="{svc}"}}', "limit": "5"}) + res = api( + TEMPO, + "/api/search", + {"q": f'{{resource.service.name="{svc}"}}', "limit": "5"}, + ) n = len(res.get("traces") or []) if n: found[svc] = n @@ -122,7 +145,10 @@ def main() -> int: return 1 if not any(signals): print("RESULT: stack healthy, but no Copilot signals stored yet.") - print(" Confirm export is enabled in user settings.json, then reload the VS Code window.") + print( + " Confirm export is enabled in user settings.json, " + "then reload the VS Code window." + ) return 1 print("RESULT: stack healthy and storing Copilot signals.") return 0 diff --git a/.github/skills/experimental/copilot-otel-metrics/pyproject.toml b/.github/skills/experimental/copilot-otel-metrics/pyproject.toml new file mode 100644 index 000000000..10932a0fc --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "copilot-otel-metrics-skill" +version = "0.0.0" +requires-python = ">=3.11" +dependencies = [] + +[dependency-groups] +dev = [ + "pytest>=9.0", + "pyyaml>=6.0", + "ruff>=0.15", +] +fuzz = [ + "atheris>=3.0", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["tests", "examples"] +python_files = ["test_*.py", "fuzz_harness.py"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] diff --git a/.github/skills/experimental/copilot-otel-metrics/references/azure-capture.md b/.github/skills/experimental/copilot-otel-metrics/references/azure-capture.md index 2597a0028..2dc279d0f 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/azure-capture.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/azure-capture.md @@ -83,8 +83,111 @@ Every one of them requires values the agent must not invent. Ask for subscriptio * Subscription and tenant. * Region, which must be consistent across the resource group and the dashboard. * Resource naming, which usually follows an existing organizational convention. +* The environment this deployment serves. It has no default, because the templates are deployed once per environment. * The RBAC principal that will hold `Monitoring Reader`, and `Monitoring Data Reader` when Prometheus data is involved. -* Retention, which is a cost decision rather than a default. +* Retention, which is a cost decision rather than a default, and which is also the deletion boundary. +* The fleet ingest token and the TLS certificate and key paths for the collector, which come from the operator's secret store and are never generated here. + +### Separation is deployment topology, not an attribute + +The default is one collector endpoint, one ingest token, one Application Insights resource, and one Log Analytics workspace per environment. + +A resource attribute such as `service.namespace` or `deployment.environment.name` is a grouping key. It lets a reader filter; it does not stop a reader removing the filter. Anyone holding `Monitoring Reader` on a workspace can read everything in that workspace. Never describe an attribute as an isolation control, and scope each reader assignment to a single workspace rather than to the resource group or subscription. + +A shared workspace across environments is a legitimate cost trade-off, but say what it costs: every authorized reader gains fleet-wide visibility across all of them, and no attribute can take that back afterwards. + +This is environment separation, not customer multi-tenancy. Copilot's exporter sends the same static headers from every workstation, so nothing in the payload identifies a tenant in a way worth trusting for isolation. + +### The receiver is authenticated and encrypted by default + +The generated collector requires an active bearer authenticator and TLS material, both supplied from the environment. Neither is present as a commented-out suggestion, because a control that must be uncommented is a control that will not be. + +Be accurate about the token's limits when presenting it. Copilot sends a fixed header set, so a static shared token is the mechanism available: it authenticates the fleet rather than a person, cannot be rotated for one user, and is extractable from any workstation. It is still worth having, because the alternative is a receiver that accepts and bills spans from anything that can route to it. + +Where an ingress controller or load balancer terminates TLS, the `tls` blocks are removed and the terminating hop owns certificate lifecycle and cipher policy. Record that ownership rather than leaving it implied. + +One asymmetry to state plainly: this configuration governs the server. How Copilot's exporter validates the server certificate is the exporter's behavior, and nothing here changes it. + +### Mandatory authentication silently drops agent-host telemetry + +Raise this before an operator deploys, because it is a data-loss defect and its first symptom is an absence. + +Managed `telemetry.headers` are applied to the extension exporter directly rather than through environment variables. That is deliberate and correct: an environment variable is inherited by the tool subprocesses the agent spawns, so delivering the ingest token that way would put a fleet-wide write credential inside every model-directed subprocess. The consequence is a split. The extension's telemetry authenticates; the agent host's telemetry does not, and a receiver that requires authentication rejects it. + +A rejected OTLP export answers HTTP 401 or gRPC `UNAUTHENTICATED`. Neither is retryable under the OTLP specification, so the export is dropped rather than queued. Nothing is retried later and no partial data arrives. Whether VS Code surfaces the rejection to the developer is not established, so do not rely on someone reporting an error. + +Two fixes suggest themselves and both make the deployment less safe. Do not offer either: + +* **Putting the token in the environment** so the agent host inherits it. This is the exact path the extension-versus-agent-host split exists to prevent. +* **Removing authentication from the receiver** so the agent host is accepted. This reopens ingestion to anything that can route to the endpoint, on a billed backend. + +#### The relay + +The carrier that does not require either concession is a per-workstation Collector between the agent host and the fleet endpoint. + +```text +agent host ──unauthenticated──▢ local relay ──authenticated, TLS──▢ fleet receiver + (loopback) (own config) +``` + +The agent host exports to `http://127.0.0.1:4318` with no credential, which it can already do. The relay holds the fleet credential in its own configuration and adds it on the upstream hop. The credential therefore lives in a file the relay reads, not in the environment VS Code hands to its children. + +Two conditions decide whether that property actually holds. State both when offering this: + +* The relay is launched **independently** of VS Code, as a service or a user daemon. A relay started from the same shell as VS Code, or as its child, may share the environment the split exists to keep clean. +* The relay's credential is stored where the relay reads it and the editor does not, meaning its configuration file or a secret store, not an exported variable. + +The relay is the same Collector already used locally, so the fail-closed attribute allow-list applies to agent-host telemetry as well. + +#### What the relay does not do + +Be honest about what changed. The relay moves the exposure; it does not remove it. + +Its listener is unauthenticated by construction, so **any local process that can reach loopback can inject telemetry into the fleet backend** through it, carrying genuine service names. Binding to `127.0.0.1` is necessary and not sufficient: it keeps the listener off the network but says nothing about which local process connected. The relay authenticates the *workstation's relay* to the fleet endpoint. It does not authenticate the workstation, and it does not authenticate the developer. Any span arriving through it is attributable to the fleet credential and no further. + +The credential is also readable by whatever identity the relay runs as. That is a smaller blast radius than the environment of every agent-spawned subprocess, which is the point, but it is not zero. + +Tracked as `G-INF-7`. + +#### mTLS is preferred and not yet available + +The better answer is mutual TLS, where each workstation presents a client certificate and the receiver verifies it. + +The receiver half is settled. Supplying `tls.client_ca_file` alongside `cert_file` and `key_file` selects `RequireAndVerifyClientCert`, so the Collector rejects a connection with no client certificate or one signed by another authority, on both the gRPC and HTTP receivers: + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + cert_file: /etc/otel/server.crt + key_file: /etc/otel/server.key + # Presence of this field is what makes a client certificate + # mandatory rather than optional. + client_ca_file: /etc/otel/clients-ca.crt +``` + +The sender half is not settled. No primary source establishes that the VS Code agent host can present a client certificate, or that it reads an operating-system certificate store for one. Until that is known, this is a Collector capability rather than a reachable deployment, and offering it as available would be the same unbacked-claim pattern this skill has been correcting. + +The test that would settle it: stand up a receiver configured as above, point one workstation's agent host at it, and observe whether the handshake completes. A completed handshake settles it; a rejected one distinguishes "cannot present a certificate" from "presented the wrong certificate" in the Collector's own log. + +#### Make the failure detectable + +Do not let an operator infer success from an absence of errors. Verify positively that agent-host spans are arriving: + +1. Note which span categories only the agent host emits, so their absence is diagnostic rather than ambiguous. +2. After a reload, query the backend for a span from that category within the last few minutes. +3. If none arrives while extension telemetry does, the split is present and the relay is missing or misconfigured. + +The symptom to describe to an operator is a **whole category of spans missing while other telemetry looks healthy**, not an error in a log. + +### Lifecycle ownership + +Retention is the deletion policy. Data ages out at that boundary and not before, and the templates purge nothing on request. An erasure obligation for a named individual is an operator procedure run against the workspace; present it as owned work rather than implying the template handles it. + +Terraform state for these files contains the Application Insights connection string, a fleet-wide write credential. `sensitive = true` keeps it out of CLI display, not out of state. Say so, and recommend an encrypted remote backend with restricted access. Verify every API version and provider version against current documentation at generation time. Do not reuse a version because a template already contains it. diff --git a/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md b/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md index 23381cd1a..8a26c5afe 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md @@ -10,25 +10,27 @@ Read this before writing or advising on any `github.copilot.chat.otel.*` setting ## The settings -Eleven settings exist. Three of them turn export on; the remaining eight tune it. Enumerate the live set before quoting this table, because it describes one build. - -| Setting | Type | Default | Sets what | -|---------------------------------------------------|---------|---------------------------|----------------------------------------------------------------------------| -| `github.copilot.chat.otel.enabled` | boolean | `false` | Master switch for trace, metric, and log emission | -| `github.copilot.chat.otel.exporterType` | string | `"otlp-http"` | One of `otlp-grpc`, `otlp-http`, `console`, `file` | -| `github.copilot.chat.otel.otlpEndpoint` | string | `"http://localhost:4318"` | Where the data goes | -| `github.copilot.chat.otel.protocol` | string | `""` | One of `""`, `http/json`, `http/protobuf`, `grpc`; empty means `http/json` | -| `github.copilot.chat.otel.headers` | object | `{}` | Extra OTLP headers such as auth tokens, applied to the exporter directly | -| `github.copilot.chat.otel.serviceName` | string | `""` | The `service.name` resource attribute | -| `github.copilot.chat.otel.resourceAttributes` | object | `{}` | Extra resource attributes, merged per key with the environment | -| `github.copilot.chat.otel.captureContent` | boolean | `false` | Prompts, responses, system instructions, and tool definitions on spans | -| `github.copilot.chat.otel.maxAttributeSizeChars` | integer | `0` | Truncation limit in characters; `0` disables truncation | -| `github.copilot.chat.otel.outfile` | string | `""` | JSON-lines output path; setting it forces the `file` exporter | -| `github.copilot.chat.otel.dbSpanExporter.enabled` | boolean | `false` | Local SQLite span exporter; turning it on turns OTel on | - -Every one of them is `scope: application`. That single fact drives the rest of this procedure. - -A minimal local setup writes three keys and leaves the other eight at their defaults: +Seven settings exist in the build verified for this revision. Three of them turn export on; the remaining four tune it. Enumerate the live set before quoting this table, because it describes one build. + +Verified against the GitHub Copilot Chat extension manifest, version 0.52.0, reading `contributes.configuration[].properties`. + +| Setting | Type | Default | Sets what | +|---------------------------------------------------|---------|---------------------------|------------------------------------------------------------------------| +| `github.copilot.chat.otel.enabled` | boolean | `false` | Master switch for trace, metric, and log emission | +| `github.copilot.chat.otel.exporterType` | string | `"otlp-http"` | One of `otlp-grpc`, `otlp-http`, `console`, `file` | +| `github.copilot.chat.otel.otlpEndpoint` | string | `"http://localhost:4318"` | Where the data goes | +| `github.copilot.chat.otel.captureContent` | boolean | `false` | Prompts, responses, system instructions, and tool definitions on spans | +| `github.copilot.chat.otel.maxAttributeSizeChars` | integer | `0` | Truncation limit in characters; `0` disables truncation | +| `github.copilot.chat.otel.outfile` | string | `""` | JSON-lines output path; setting it forces the `file` exporter | +| `github.copilot.chat.otel.dbSpanExporter.enabled` | boolean | `false` | Local SQLite span exporter; turning it on turns OTel on | + +An earlier revision of this reference listed eleven settings, adding `protocol`, `headers`, `serviceName`, and `resourceAttributes`, verified against extension build `0.59.2026072702` where all eleven declared `scope: application`. Those four are **not declared in the build verified here**. Both observations are real, which is the point: the settings surface moves with the extension. Writing a key the installed build does not declare produces a setting that is silently inert, so confirm against the user's own build before offering any of them. + +`headers` in particular is absent from **user settings** in this build. It is not absent from the product: the managed `telemetry` block carries a `headers` field, which is how a fleet authenticates to a collector. The two are different configuration surfaces delivered through different channels, so an operator who cannot find `headers` in their settings UI has not found a bug. See `org-distribution.md` for the managed surface and the consequence that headers reach the extension exporter and not the agent host. + +None of these settings declares a `scope` in the verified manifest, which means they take VS Code's default `window` scope rather than `application`. Do not assert application scope without checking; see the profile section below for what that changes. + +A minimal local setup writes three keys and leaves the rest at their defaults: ```json { @@ -49,10 +51,6 @@ Enterprise policy beats environment variable beats user setting beats default. A | `enabled` | `COPILOT_OTEL_ENABLED` | | `captureContent` | `COPILOT_OTEL_CAPTURE_CONTENT` | | `otlpEndpoint` | `OTEL_EXPORTER_OTLP_ENDPOINT` | -| `protocol` | `OTEL_EXPORTER_OTLP_PROTOCOL` | -| `serviceName` | `OTEL_SERVICE_NAME` | -| `resourceAttributes` | `OTEL_RESOURCE_ATTRIBUTES` | -| `headers` | `OTEL_EXPORTER_OTLP_HEADERS` | | `maxAttributeSizeChars` | `COPILOT_OTEL_MAX_ATTRIBUTE_SIZE_CHARS` | `exporterType`, `outfile`, and `dbSpanExporter.enabled` declare no environment variable in the manifest inspected for this revision; they are set through user settings or enterprise policy. Confirm against the installed build before telling a user an environment variable will or will not apply, because this mapping moves with the extension. @@ -61,7 +59,12 @@ The VS Code documentation renders "This setting is managed at the organization l ## Find the file that actually resolves -Application-scoped settings live only in the global `settings.json`, and that file resolves from the **default profile no matter which profile is active**. Writing into `User/profiles//settings.json` therefore accomplishes nothing for these keys, and it fails silently: no error, no warning, no telemetry. +Which file resolves depends on the declared scope, and the scope is a property of the installed build rather than of this document. Check it before choosing a target file. + +* **If a setting declares `scope: application`,** it lives only in the global `settings.json`, and that file resolves from the **default profile no matter which profile is active**. Writing into `User/profiles//settings.json` accomplishes nothing for that key, and it fails silently: no error, no warning, no telemetry. +* **If it declares no scope,** as every OTel key does in the verified manifest, it takes the default `window` scope and the active profile's `settings.json` does apply. + +When unsure, write the global file: it is correct in both cases, because a window-scoped key still resolves there when no profile overrides it. Then have the user confirm the value took effect rather than assuming it did. | Platform | VS Code | VS Code Insiders | |----------|---------------------------------------------------------|--------------------------------------------------------------------| @@ -77,6 +80,19 @@ Stop and ask if the file cannot be identified with confidence. Do not write to a Follow every step. Steps 1, 4, and 5 are what make this reversible and visible. +`examples/settings_upsert.py` implements this contract rather than describing it. Prefer it over an ad-hoc edit: it refuses any key outside the verified schema, rejects a type mismatch, an unsafe endpoint, an `outfile` and `otlp-*` combination, and any attempt to enable `captureContent`; it splices only the target value span, backs the file up, restores the backup if the result does not parse, and refuses to write if unrelated settings would change. Run it without `--apply` to get the diff for step 4. + +```bash +python3 settings_upsert.py --settings \ + --set github.copilot.chat.otel.enabled=true \ + --set github.copilot.chat.otel.exporterType=otlp-http \ + --set github.copilot.chat.otel.otlpEndpoint=http://localhost:4318 +``` + +Its schema is frozen against one build and records which one. Re-verify against the installed extension before trusting it on a newer version. + +The steps below remain the contract, whether the tool performs them or a hand edit does. + 1. **Back up.** Copy the file alongside itself as `settings.json.bak-otel-`, for example `settings.json.bak-otel-20260727T142530Z`. Tell the user the backup path and that restoring it is a file copy in the other direction. That path sits beside `settings.json`, outside the workspace; name it when asking for the write rather than treating it as a separate approval. 2. **Read the file as text.** If it does not exist or is empty, treat the content as `{}`. It is JSONC: it may contain `//` and `/* */` comments and trailing commas, all of which are legal and all of which the user wrote deliberately. 3. **Compute the change per key. Do not write yet.** Work out the exact edit without applying it, so step 4 has something to show. diff --git a/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md b/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md index 7a24efdf8..1f2ba0b63 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md @@ -1,5 +1,5 @@ --- -description: "Contract for generating the local single-container Grafana capture stack and the local PromQL dashboard, plus the inventory and disposition of the bundled helper scripts" +description: "Contract for generating the local filtered Grafana capture stack and the local PromQL dashboard, plus the inventory and disposition of the bundled helper scripts" --- # Local stack: generate a backend on this machine @@ -10,35 +10,70 @@ Read this when the user needs somewhere for Copilot's telemetry to land on their ## The mode -Offer this mode when the user has enabled export, or is about to, and has nowhere for the data to go. The whole backend is one container: Grafana, Prometheus, and Tempo behind a single OTLP endpoint. +Offer this mode when the user has enabled export, or is about to, and has nowhere for the data to go. The backend is two containers: an OpenTelemetry Collector that filters, and one LGTM container holding Grafana, Prometheus, and Tempo behind it. -**Consent gate.** Say which files will be written and where, then write them on approval. Then print the commands and stop. Starting the stack is the user's action under the execution boundary in `SKILL.md`, because it pulls a multi-hundred-megabyte image, binds five ports, and creates a Docker volume that outlives the container. +**Consent gate.** Say which files will be written and where, then write them on approval. Then print the commands and stop. Starting the stack is the user's action under the execution boundary in `SKILL.md`, because it pulls a multi-hundred-megabyte image, binds ports, and creates a Docker volume that outlives the container. + +The stack will not start without Grafana credentials, so those come first. Offer the entry form that does not echo, because an inline assignment leaves the password in the terminal scrollback and in the shell history file, and both outlive the session. ```bash +export COPILOT_OTEL_GRAFANA_USER=admin +read -rs -p 'Grafana password: ' COPILOT_OTEL_GRAFANA_PASSWORD; echo +export COPILOT_OTEL_GRAFANA_PASSWORD docker volume create copilot-otel-data docker compose -f compose.yaml up -d ``` -Hand those over. Do not run them. +In PowerShell: + +```powershell +$env:COPILOT_OTEL_GRAFANA_USER = 'admin' +$env:COPILOT_OTEL_GRAFANA_PASSWORD = + [Runtime.InteropServices.Marshal]::PtrToStringBSTR( + [Runtime.InteropServices.Marshal]::SecureStringToBSTR( + (Read-Host 'Grafana password' -AsSecureString))) +docker volume create copilot-otel-data +docker compose -f compose.yaml up -d +``` + +A Compose `.env` file is not an equivalent substitute here. It would reach the containers, but `validate_dashboard.py` and the other bundled helpers read these variables from the environment of the shell that runs them, so an operator who only writes a `.env` file gets a running stack and helpers that fail on missing configuration. + +Hand those over. Do not run them. Do not invent the password or suggest one; the user chooses it. -| Surface | Where | For | -|------------|-------------------------|---------------------------------------------------| -| Grafana | `http://localhost:3000` | Dashboards; default credentials `admin` / `admin` | -| Prometheus | `http://localhost:9090` | Metrics store | -| Tempo | `http://localhost:3200` | Trace store | -| OTLP HTTP | `http://localhost:4318` | Where Copilot exports | -| OTLP gRPC | `localhost:4317` | Alternative transport | +| Surface | Where | For | +|------------|-------------------------|-----------------------------------------------------| +| Grafana | `http://localhost:3000` | Dashboards; credentials come from the two variables | +| Prometheus | `http://localhost:9090` | Metrics store | +| Tempo | `http://localhost:3200` | Trace store | +| OTLP HTTP | `http://localhost:4318` | Where Copilot exports; served by the Collector | +| OTLP gRPC | `localhost:4317` | Alternative transport; served by the Collector | ## The generated compose contract -Seed from `examples/compose.yaml`. Four choices in it are load-bearing. A user adapting the file may change anything else; tell them what these cost before they change one of these. +Seed from `examples/compose.yaml` and `examples/otel-collector-local.yaml`. Seven choices in them are load-bearing. A user adapting the files may change anything else; tell them what these cost before they change one of these. -* **Every port binds `127.0.0.1`.** Nothing in this stack authenticates. Grafana ships with `admin`/`admin`, and the OTLP endpoint accepts writes from anyone who can reach it. Loopback binding is what makes that acceptable. Publishing `4318` on all interfaces, as some published walkthroughs do, exposes an unauthenticated ingest endpoint on the network; OpenTelemetry's own security guidance flags that pattern under CWE-1327. +* **The Collector is the only OTLP listener published on the host.** LGTM's own `4317` and `4318` are deliberately not mapped. This is the difference between filtering telemetry and merely hoping it is filtered: if LGTM published OTLP itself, any local process could write straight past the allow-list. Loopback binding does not prevent that, because every process on the machine is already on loopback. +* **The attribute policy is an allow-list, not a delete-list.** `allow_all_keys: false` means an attribute the configuration has never seen is dropped. The emitter's attribute set is not a stable contract, so a delete-list is only correct until the next extension release adds a content-bearing key. Adding a key to `allowed_keys` is a deliberate decision to store it. +* **Grafana credentials are required with no default.** The image enables anonymous access with the Admin role and hides the login form, so left alone every stored prompt fragment is readable by anything that can reach port 3000. The compose file disables anonymous access, restores the login form, and requires `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD` through the `${VAR:?message}` form, which fails the `up` rather than starting on a guessable credential. +* **Both images are pinned by digest.** A tag is mutable, so a tag-pinned stack can change underneath a configuration that was reviewed. The tag is kept in a comment above each digest so the version stays legible. +* **Every port binds `127.0.0.1`.** The OTLP endpoint accepts writes from anyone who can reach it. Loopback binding is what makes that acceptable. Publishing `4318` on all interfaces, as some published walkthroughs do, exposes an unauthenticated ingest endpoint on the network; OpenTelemetry's own security guidance flags that pattern under CWE-1327. * **The data volume is declared `external`.** Compose namespaces volumes by project. A plain declaration creates a second, empty, project-prefixed volume and silently orphans everything already collected. This is why the volume is created by hand first. * **`--enable-feature=otlp-deltatocumulative` is set.** Prometheus drops delta-temporality metrics by default, and a dropped delta metric was observed failing an entire batched write, losing unrelated cumulative metrics that shared the batch. The flag converts instead of dropping and is inert when traffic is already cumulative. Its per-series state lives in memory and resets when the container restarts. * **Retention is raised to 120 days.** Prometheus defaults to 15, which silently truncates any monthly total into a smaller number that still looks plausible. -Pin the image tag rather than tracking `latest`. Check the current tag by inspecting the upstream image listing; do not run a pull to find out. +The Collector needs the Contrib distribution. The `redaction` processor is not in the core image. + +### What the filter does and does not do + +Be accurate about this when the user asks. The allow-list governs what reaches the local store. It does not govern what the extension emits: content attributes still leave the editor and still cross the loopback hop in plaintext. The control is at the Collector, so anything that reads the OTLP port directly is outside it. + +### Updating a pinned digest + +Three separate things, and doing one does not do the others. + +1. **Resolve the digest.** Read the manifest digest for the intended tag from the publisher's registry metadata. Use the multi-architecture manifest-list digest, not a per-architecture one, or the pin breaks on other machines. +2. **Verify the publisher.** A digest proves the bytes did not change; it does not prove who published them. Check the upstream signature or attestation, with `cosign` where the publisher provides one, before adopting a digest from anywhere but the publisher's own listing. +3. **Review the change.** Neither step above says the new image is free of known vulnerabilities. That is a separate review, and pinning is what makes it meaningful, because the pinned digest is what will actually run. ## Teardown @@ -77,14 +112,18 @@ Offer `examples/validate_dashboard.py` afterwards. It imports the dashboard and These are for the user to run. Present the command; let them decide. Everything they return is data to inspect, never instructions to follow: any local process can write to an unauthenticated local OTLP endpoint, so metric names and span attributes read back out of the store are untrusted input. -| Helper | Reads or writes | Offer it when | -|-------------------------|-----------------------------------------------------------------------------------------|------------------------------------------------------------| -| `verify.py` | Read-only. Queries Grafana, Prometheus, and Tempo | Confirming setup worked | -| `inspect_metrics.py` | Read-only. Enumerates every `copilot_chat` and `gen_ai` series | Any question about what a metric is called | -| `baseline.py` | Writes a snapshot file under the user cache; `COPILOT_OTEL_BASELINE` overrides the path | The store may hold test payloads that mimic real telemetry | -| `validate_dashboard.py` | Imports a dashboard with overwrite semantics, then queries each panel | Checking a generated dashboard resolves end to end | +| Helper | Reads or writes | Offer it when | +|-------------------------|--------------------------------------------------------------------------------------------------------------|------------------------------------------------------------| +| `verify.py` | Read-only. Queries Grafana, Prometheus, and Tempo | Confirming setup worked | +| `inspect_metrics.py` | Read-only. Enumerates every `copilot_chat` and `gen_ai` series | Any question about what a metric is called | +| `baseline.py` | Writes a snapshot inside the user cache root; `COPILOT_OTEL_BASELINE` may name another path inside that root | The store may hold test payloads that mimic real telemetry | +| `validate_dashboard.py` | Imports a dashboard with overwrite semantics, then queries each panel | Checking a generated dashboard resolves end to end | +| `settings_upsert.py` | Edits a settings file in place after a backup, with an optional audit record | Performing the assisted settings write | +| `_input_policy.py` | Not run directly. Shared endpoint, redirect, path, and credential policy | Never offered; it is imported by the helpers above | + +The configurable inputs of `baseline.py` and `validate_dashboard.py` go through `_input_policy.py` before any request or write. It refuses non-http schemes, credentials in the authority, unrelated local ports, and non-loopback hosts without an explicit opt-in, re-applies the same rules to every redirect target, and keeps written paths inside their root. `verify.py` and `inspect_metrics.py` take no configurable endpoint and are left as they are. -`validate_dashboard.py` needs two constraints stated when offered. It handles Prometheus and Tempo panels only: it has no Azure Monitor path, so pointing it at the Azure dashboard imports the dashboard and then runs empty queries that prove nothing. And it imports with `overwrite: true`, so it replaces any dashboard sharing the uid; it refuses a non-loopback Grafana unless `COPILOT_OTEL_ALLOW_REMOTE=1` is set. It accepts a dashboard path argument so a generated local dashboard can be checked instead of the bundled one. Endpoint and credentials come from `COPILOT_OTEL_GRAFANA`, `COPILOT_OTEL_GRAFANA_USER`, and `COPILOT_OTEL_GRAFANA_PASSWORD`, defaulting to the local stack. +`validate_dashboard.py` needs two constraints stated when offered. It handles Prometheus and Tempo panels only: it has no Azure Monitor path, so pointing it at the Azure dashboard imports the dashboard and then runs empty queries that prove nothing. And it imports with `overwrite: true`, so it replaces any dashboard sharing the uid; it refuses a non-loopback Grafana unless `COPILOT_OTEL_ALLOW_REMOTE=1` is set. It accepts a dashboard path argument so a generated local dashboard can be checked instead of the bundled one. The endpoint comes from `COPILOT_OTEL_GRAFANA` and defaults to the local stack. `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD` are required with no default: they are the same pair the compose file requires, so a running stack always has them, and the helper exits with a configuration error naming both rather than sending a request that fails as an auth error. The helpers use only the Python standard library, so there is nothing to install. diff --git a/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md b/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md index cd280dd48..e9a196c11 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md @@ -22,6 +22,7 @@ Administrators mandate OTel export through the `telemetry` block in Copilot mana "enabled": true, "endpoint": "https://collector.example.internal:4318", "protocol": "otlp-http", + "headers": { "Authorization": "Bearer " }, "captureContent": false, "lockCaptureContent": true, "serviceName": "copilot-chat", @@ -32,6 +33,8 @@ Administrators mandate OTel export through the `telemetry` block in Copilot mana `lockCaptureContent` is worth deciding deliberately rather than copying. It removes the developer's ability to turn content capture on, which is usually what an organization wants, and it is also the setting most likely to be missed when the block is adapted. +`headers` is a managed-settings field and has no user-settings counterpart in the builds this skill has verified. An operator who goes looking for it in the settings UI will not find it; that is the surface being different, not the field being missing. Its delivery is also what produces the agent-host split below. + `resourceAttributes` is where fleet-wide dimensions belong. Anything put here lands on every span from every developer, so keep it to team and org structure. Do not put anything that identifies an individual in it. ## The three channels @@ -49,7 +52,7 @@ Channel precedence enforcement begins in VS Code 1.128. Below that version the b ## Four things that decide a rollout * **A managed value beats environment variables and user settings.** Once set, developers cannot redirect telemetry to their own collector. That is usually the point, and it is also why a wrong endpoint in a managed block is a fleet-wide outage rather than one person's problem. -* **Managed `telemetry.headers` reach the extension exporter and not the agent host.** They are applied directly rather than through environment variables, which is deliberate: it stops an auth token leaking into spawned tool subprocesses. The consequence is a silent split, where extension telemetry authenticates and agent-host telemetry does not. Expect it and check for it rather than discovering it as missing data. +* **Managed `telemetry.headers` reach the extension exporter and not the agent host.** They are applied directly rather than through environment variables, which is deliberate: it stops an auth token leaking into spawned tool subprocesses. The consequence is a silent split, where extension telemetry authenticates and agent-host telemetry does not. Against a receiver that requires authentication this is data loss, not degraded data: the rejection is a non-retryable HTTP 401 or gRPC `UNAUTHENTICATED`, so the export is dropped rather than queued. The Azure reference covers the relay that closes it, and why the two obvious alternatives are both worse. * **The agent host computes its telemetry configuration at start.** Changing a managed value requires a VS Code reload on every affected machine, not just a policy refresh. * **A developer can find out what applies to them.** **Developer: Policy Diagnostics** from the command palette is the per-device discriminator. It answers "is my setting being overridden" without guessing. diff --git a/.github/skills/experimental/copilot-otel-metrics/references/verification.md b/.github/skills/experimental/copilot-otel-metrics/references/verification.md index 6e12d0ee9..831500657 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/verification.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/verification.md @@ -37,6 +37,21 @@ For the Azure path the same principle applies against Log Analytics: a `count` o Offer `examples/verify.py` when the user wants this run end to end for the local stack. Offer the command, not the execution. +## What the filter reaches is a test result, not a reading of the config + +The recurring failure in this skill's history is a claim about running behavior supported only by a static test that read the configuration and confirmed it said the right thing. The configuration saying so and the Collector doing so are different facts, and the second one is the one a user relies on. + +`tests/test_collector_carriers.py` is where the difference is settled. It starts the pinned Collector with the shipped configuration, sends payloads placing a distinct marker in each of 28 OTLP carriers, and records for each whether it is dropped, replaced, or passed through unchanged. + +Two properties make its output usable as evidence: + +* **A paired control run.** The same payloads run against a derivation with the content processors removed, and every marker must appear. Absence under policy only means something if the instrument would have shown the value had it survived; otherwise "dropped by policy" and "never rendered by the exporter" are indistinguishable. +* **The map is asserted.** A configuration or image change that opens a carrier fails the suite rather than passing quietly. + +When a user asks what the filter protects, answer from that module. Its current result: attributes at every level and map-valued log bodies are governed; span status messages, span event names, trace state, the other log body shapes, severity text, and log event names are scrubbed; and span names, metric metadata, span links, metric exemplars, and the instrumentation scope and schema fields pass through β€” the first pair by choice because dashboards read them, the second pair because no processor in this distribution can reach them, and the last because they are expected to carry only library identity, which is an expectation rather than a control. `SECURITY.md` carries the same table with the gap identifiers. + +A skipped run is not a passing run. On a contributor machine with no container runtime the module skips with a stated reason, and that reason says the carrier map was not verified by that run. Where the result is read as evidence rather than as a contribution gate, the skip is not allowed: strict mode turns a missing runtime into a failure, and it is on by default whenever `CI` is set. `COPILOT_OTEL_STRICT_RUNTIME` overrides that in either direction, so a lane with no usable runtime opts out deliberately rather than by silence. + ## Four things that look like failure and are not **HTTP 200 does not mean stored.** A dropped payload returns `200 {"partialSuccess":{}}`, byte-identical to an accepted one. This is the single most misleading signal in the entire pipeline. Never report success from an export response. @@ -89,7 +104,7 @@ Re-check after every extension update. When a name cannot be verified in the mom Work down this list before assuming something is broken. 1. Was the window reloaded after the settings change? These settings are read at startup. -2. Was the setting written to the file that actually resolves? Application-scoped settings come from the default profile regardless of the active profile. +2. Was the setting written to the file that actually resolves? This depends on the scope the installed build declares. Where a key is application-scoped it comes from the default profile regardless of the active profile; where it declares no scope the active profile's file applies. The global file is correct under either, so check it first. 3. Is a policy or environment variable overriding the setting? **Developer: Policy Diagnostics** answers the policy half. 4. Is the backend actually up and listening on the endpoint the setting names? 5. Has any Copilot activity occurred since the reload? An idle editor emits nothing. diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py b/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py new file mode 100644 index 000000000..9e627736e --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Polyglot fuzz harness for configuration parsing, input policy, and JSONC edits.""" + +from __future__ import annotations + +import json +import pathlib +import re +import sys +from contextlib import suppress + +_TESTS_DIR = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(_TESTS_DIR)) +sys.path.insert(0, str(_TESTS_DIR.parent / "examples")) + +from _input_policy import PolicyError, check_url # noqa: E402 +from settings_upsert import SettingsError, parse_assignment, strip_jsonc # noqa: E402 +from test_local_config import ( # noqa: E402 + MASK, + ConfigError, + load_yaml_text, + simulate_redaction, +) + +try: + import atheris +except ImportError: + atheris = None + FUZZING = False +else: + FUZZING = True + +ALLOW = frozenset({"service.name", "http.url"}) +BLOCKED = (re.compile(r"[0-9a-f]{40}"),) + + +def fuzz_local_config(data: bytes) -> None: + """Exercise parsing, redaction, URL policy, and JSONC handling with arbitrary input. + + Three invariants are asserted, and each is the security property of its + subject: redaction never emits a key outside the allow-list, check_url never + returns for a non-http scheme, and strip_jsonc never changes the length of + the text it is given. + """ + text = data.decode("utf-8", errors="replace") + with suppress(ConfigError): + load_yaml_text(text) + + attributes = {line: text for line in text.splitlines() if line} + kept = simulate_redaction(attributes, ALLOW, BLOCKED) + assert set(kept) <= ALLOW + + try: + parsed = check_url(text) + except PolicyError: + pass + else: + assert parsed.scheme in ("http", "https") + assert parsed.username is None and parsed.password is None + + assert len(strip_jsonc(text)) == len(text) + + with suppress(SettingsError): + parse_assignment(text) + + +class TestLocalConfigFuzzHarness: + """Property tests mirroring the fuzz target.""" + + def test_loader_raises_config_error_or_returns_mapping(self) -> None: + for text in ("", "a: 1", "- list", "a: [\n", "null"): + with suppress(ConfigError): + assert isinstance(load_yaml_text(text), dict) + + def test_redaction_never_emits_a_key_outside_the_allow_list(self) -> None: + for key in ("service.name", "prompt.text", "", "0" * 200, "service.name.extra"): + kept = simulate_redaction({key: "value"}, ALLOW, BLOCKED) + assert set(kept) <= ALLOW + + def test_a_blocked_value_is_masked_rather_than_leaked(self) -> None: + kept = simulate_redaction({"http.url": "b" * 40}, ALLOW, (re.compile(r"b{40}"),)) + assert kept == {"http.url": MASK} + + def test_check_url_never_returns_a_non_http_scheme(self) -> None: + for candidate in ("", "file:///etc/passwd", "http://", "://", "http://[", "data:,"): + try: + parsed = check_url(candidate) + except PolicyError: + continue + assert parsed.scheme in ("http", "https") + + def test_strip_jsonc_preserves_length_and_parses_what_it_can(self) -> None: + for text in ("", "{}", '{"a": 1} // trailing', "/* unterminated", '{"a": "//"}'): + stripped = strip_jsonc(text) + assert len(stripped) == len(text) + with suppress(json.JSONDecodeError): + json.loads(stripped) + + +if __name__ == "__main__" and FUZZING: + atheris.instrument_all() + atheris.Setup(sys.argv, fuzz_local_config) + atheris.Fuzz() diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py new file mode 100644 index 000000000..0738d9812 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py @@ -0,0 +1,172 @@ +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Static checks over the Azure fleet templates. + +These tests read the committed Bicep, Terraform, and Collector files as text and +YAML. Nothing here deploys, plans, or contacts Azure. +""" + +from __future__ import annotations + +import pathlib +import re + +import pytest +import yaml + +AZURE_DIR = pathlib.Path(__file__).resolve().parents[1] / "examples" / "azure" +COLLECTOR_PATH = AZURE_DIR / "otel-collector-config.yaml" +BICEP_PATH = AZURE_DIR / "main.bicep" +MAIN_TF_PATH = AZURE_DIR / "main.tf" +VARIABLES_TF_PATH = AZURE_DIR / "variables.tf" +OUTPUTS_TF_PATH = AZURE_DIR / "outputs.tf" + +TEMPLATE_PATHS = [BICEP_PATH, MAIN_TF_PATH, VARIABLES_TF_PATH, OUTPUTS_TF_PATH, COLLECTOR_PATH] + +# Shapes that indicate a real credential rather than a placeholder or a +# reference to one. +SECRET_PATTERNS = ( + re.compile(r"InstrumentationKey=[0-9a-f]{8}-[0-9a-f]{4}", re.IGNORECASE), + re.compile(r"gh[pousr]_[0-9A-Za-z]{16,}"), + re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), + re.compile(r"\bAKIA[0-9A-Z]{16}\b"), +) + + +@pytest.fixture(scope="module") +def collector() -> dict: + return yaml.safe_load(COLLECTOR_PATH.read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def bicep() -> str: + return BICEP_PATH.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def terraform() -> str: + return "\n".join(path.read_text(encoding="utf-8") for path in (MAIN_TF_PATH, VARIABLES_TF_PATH)) + + +class TestReceiverAuthentication: + """The fleet receiver authenticates senders, in configuration and not in prose.""" + + @pytest.mark.parametrize("protocol", ["http", "grpc"]) + def test_each_protocol_declares_an_authenticator(self, collector: dict, protocol: str) -> None: + config = collector["receivers"]["otlp"]["protocols"][protocol] + assert config["auth"]["authenticator"] == "bearertokenauth" + + def test_the_authenticator_extension_is_active(self, collector: dict) -> None: + assert "bearertokenauth" in collector["extensions"] + assert "bearertokenauth" in collector["service"]["extensions"] + + def test_the_token_is_an_environment_reference(self, collector: dict) -> None: + token = collector["extensions"]["bearertokenauth"]["token"] + assert token.startswith("${env:") + + def test_authentication_is_not_left_commented_out(self) -> None: + text = COLLECTOR_PATH.read_text(encoding="utf-8") + commented = [ + line + for line in text.splitlines() + if line.strip().startswith("#") and "bearertokenauth:" in line + ] + assert commented == [], "authentication is present only as a comment" + + +class TestReceiverTls: + """TLS is configured, and its material comes from the environment.""" + + @pytest.mark.parametrize("protocol", ["http", "grpc"]) + def test_each_protocol_configures_tls(self, collector: dict, protocol: str) -> None: + tls = collector["receivers"]["otlp"]["protocols"][protocol]["tls"] + assert tls["cert_file"].startswith("${env:") + assert tls["key_file"].startswith("${env:") + + +class TestEnvironmentSeparation: + """Separation comes from separate deployments, not from an attribute.""" + + def test_bicep_requires_an_environment_parameter(self, bicep: str) -> None: + assert re.search(r"^param environment string$", bicep, re.MULTILINE) + assert "param environment string = " not in bicep, "environment must have no default" + + def test_terraform_requires_an_environment_variable(self, terraform: str) -> None: + assert 'variable "environment"' in terraform + block = terraform.split('variable "environment"', 1)[1].split("\nvariable ", 1)[0] + assert "default" not in block, "environment must have no default" + + def test_bicep_resource_names_include_the_environment(self, bicep: str) -> None: + for name in ("workspaceName", "appInsightsName", "dashboardName"): + line = next(line for line in bicep.splitlines() if line.startswith(f"var {name}")) + assert "${environment}" in line + + def test_terraform_resource_names_include_the_environment(self, terraform: str) -> None: + assert 'resource_prefix = "${var.name_prefix}-${var.environment}"' in terraform + assert "${local.resource_prefix}-copilot-logs" in terraform + assert "${local.resource_prefix}-copilot-insights" in terraform + + def test_the_collector_environment_attribute_is_an_environment_reference( + self, collector: dict + ) -> None: + attributes = collector["processors"]["resource/fleet"]["attributes"] + by_key = {entry["key"]: entry for entry in attributes} + assert by_key["deployment.environment.name"]["value"].startswith("${env:") + + def test_no_file_calls_a_resource_attribute_an_isolation_control(self) -> None: + for path in TEMPLATE_PATHS: + text = path.read_text(encoding="utf-8").lower() + for claim in ("namespace provides isolation", "isolated by service.namespace"): + assert claim not in text + + +class TestRetentionAndAccess: + """Retention, caps, and reader scope agree across both templates.""" + + def test_retention_defaults_match(self, bicep: str, terraform: str) -> None: + assert "param retentionInDays int = 90" in bicep + assert re.search(r'variable "retention_in_days"[\s\S]*?default\s*=\s*90', terraform) + + def test_daily_cap_defaults_match(self, bicep: str, terraform: str) -> None: + assert "param dailyQuotaGb int = 5" in bicep + assert re.search(r'variable "daily_quota_gb"[\s\S]*?default\s*=\s*5', terraform) + + def test_the_reader_assignment_is_opt_in(self, bicep: str, terraform: str) -> None: + assert "if (!empty(readerPrincipalId))" in bicep + assert 'count = var.reader_principal_id == "" ? 0 : 1' in terraform + + def test_the_reader_assignment_is_scoped_to_the_workspace( + self, bicep: str, terraform: str + ) -> None: + assert "scope: workspace" in bicep + assert "scope = azurerm_log_analytics_workspace.this.id" in terraform + + def test_retention_is_described_as_the_deletion_boundary( + self, bicep: str, terraform: str + ) -> None: + assert "deletion policy" in bicep.lower() + assert "deletion policy" in terraform.lower() + + +class TestStateAndSecrets: + """No template carries a credential, and state sensitivity is stated.""" + + @pytest.mark.parametrize("path", TEMPLATE_PATHS, ids=lambda p: p.name) + def test_no_literal_secret_is_committed(self, path: pathlib.Path) -> None: + text = path.read_text(encoding="utf-8") + for pattern in SECRET_PATTERNS: + assert not pattern.search(text), f"{path.name} matches {pattern.pattern}" + + def test_the_connection_string_output_is_marked_sensitive(self) -> None: + text = OUTPUTS_TF_PATH.read_text(encoding="utf-8") + block = text.split('output "connection_string"', 1)[1] + assert "sensitive = true" in block + + def test_state_sensitivity_is_documented_rather_than_implied(self) -> None: + text = MAIN_TF_PATH.read_text(encoding="utf-8").lower() + assert "state file" in text + assert "does not remove it from state" in text + + def test_bicep_emits_a_command_rather_than_the_connection_string(self, bicep: str) -> None: + assert "output connectionStringCommand string" in bicep + assert "output connectionString string =" not in bicep diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py new file mode 100644 index 000000000..70220260e --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py @@ -0,0 +1,1011 @@ +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Runtime carrier map for the shipped local Collector configuration. + +Every other test module in this suite parses committed files and simulates the +declared policy. That is how this skill came to ship control claims no test +could support: a static check confirmed the YAML said the right thing, and a +runtime claim was written on that basis. + +This module starts the pinned Collector, sends payloads that place a unique +marker in every OTLP carrier the signal model can transport, and records what +the shipped policy does to each one. The recorded map is the citable evidence +behind the minimization claims in `SECURITY.md` and the references, and it is +asserted as a regression boundary, so a configuration or image change that +opens a carrier fails here rather than passing quietly. + +Absence alone proves nothing. A marker can be missing because policy dropped it +or because the exporter never renders that field. Every run is therefore paired +with a negative control that removes the content-minimization processors from +the same configuration. A marker the control does not render is `unobservable`, +never `governed`. +""" + +from __future__ import annotations + +import base64 +import copy +import json +import os +import pathlib +import platform +import shutil +import socket +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass, field + +import pytest +import yaml + +SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1] +COLLECTOR_PATH = SKILL_ROOT / "examples" / "otel-collector-local.yaml" +COMPOSE_PATH = SKILL_ROOT / "examples" / "compose.yaml" + +# The probe runs the same image the stack runs. Read from compose.yaml rather +# than restated here, so a pin bump cannot leave this harness testing an image +# the stack no longer uses. +COLLECTOR_SERVICE = "otel-collector" + +# Every processor whose job is to remove or replace content. The negative +# control removes all of them, because validating the instrument against only +# one of them would let the other hide a marker and make an unobservable +# carrier look governed. +CONTENT_PROCESSORS = ("redaction", "transform/scrub") + +# A skipped run produces no evidence. That is tolerable on a contributor's +# machine and not tolerable anywhere the run's result is cited: `SECURITY.md` +# and the references attribute their minimization claims to this module, so a +# lane that quietly stopped starting the Collector would keep reporting green +# while those claims lost their backing. Strictness is therefore on by default +# wherever `CI` is set, and this variable overrides it in both directions. +STRICT_RUNTIME_ENV = "COPILOT_OTEL_STRICT_RUNTIME" +_FALSE_VALUES = frozenset({"", "0", "false", "no", "off"}) + +MASK = "****" +SCRUB = "[redacted]" + +GOVERNED = "governed" +MASKED = "masked" +PASSED_THROUGH = "passed-through" +UNOBSERVABLE = "unobservable" + +# How a carrier is handled, for the reader. Classification is observed; +# mechanism is declared, and the two are checked against each other only in the +# sense that a carrier with no mechanism must be passed-through. +ALLOW_LIST = "allow-list" +BLOCKED_VALUES = "blocked_values" +CONTENT_SCRUB = "content scrub" +NO_MECHANISM = "none" + +# The probed receiver protocol. OTLP/gRPC is published by the same shipped +# configuration and is not probed here; it is carried as a gap. +PROBED_PROTOCOL = "otlp/http" +UNPROBED_PROTOCOL = "otlp/grpc" + +_BYTES_MARKER_PLAINTEXT = b"CARRIERMARKERLOGBODYBYTES" +# The exporter renders a bytes body as base64, so the observable marker is the +# encoded form rather than the plaintext. +BYTES_MARKER = base64.b64encode(_BYTES_MARKER_PLAINTEXT).decode("ascii") + +# Attributes a shipped detection control reads that the allow-list had been +# dropping. Survival is asserted per key, because an allow-list entry that is +# present in the YAML and still dropped at runtime is the same class of +# unbacked claim this module exists to prevent. +RESTORED_DETECTION_KEYS: dict[str, str] = { + "session.id": "RESTOREDMARKERSESSIONID", + "gen_ai.token.type": "RESTOREDMARKERTOKENTYPE", + "copilot_chat.edit_source": "RESTOREDMARKEREDITSOURCEFLAT", + "copilot_chat.edit.source": "RESTOREDMARKEREDITSOURCEDOTTED", +} + + +@dataclass(frozen=True) +class Carrier: + """One OTLP content carrier, its probe marker, and its observed handling.""" + + name: str + marker: str + expected: str + # The declared mechanism responsible for the expected classification. + mechanism: str = NO_MECHANISM + # Present only for a carrier whose value is replaced rather than removed. + # The literal the exporter must render when replacement occurred. + masked_signature: str | None = None + note: str = "" + + +# The observed carrier map. +# +# Recorded from execution against the pinned Collector with the shipped +# configuration, and validated by the negative control in the same run. This is +# a baseline, not an aspiration: rebaselining a `governed` entry to +# `passed-through` is a policy decision that needs its own evidence. +CARRIERS: tuple[Carrier, ...] = ( + # --- Attributes. Fail-closed everywhere the allow-list reaches. --- + Carrier("resource attribute", "CARRIERMARKERRESOURCEATTR", GOVERNED, ALLOW_LIST), + Carrier("instrumentation scope attribute", "CARRIERMARKERSCOPEATTR", GOVERNED, ALLOW_LIST), + Carrier("span attribute", "CARRIERMARKERSPANATTR", GOVERNED, ALLOW_LIST), + Carrier("span event attribute", "CARRIERMARKERSPANEVENTATTR", GOVERNED, ALLOW_LIST), + Carrier("log record attribute", "CARRIERMARKERLOGATTR", GOVERNED, ALLOW_LIST), + Carrier("metric datapoint attribute", "CARRIERMARKERDATAPOINTATTR", GOVERNED, ALLOW_LIST), + Carrier( + "log body, map", + "CARRIERMARKERLOGBODYMAP", + GOVERNED, + ALLOW_LIST, + note="A map body is treated as an attribute set and reduced to the " + "allow-listed subset. The other body shapes are not, which is why the " + "content scrub covers them.", + ), + # --- Attributes the allow-list cannot reach, and neither can anything + # --- else in this distribution. + Carrier( + "span link attribute", + "CARRIERMARKERLINKATTR", + PASSED_THROUGH, + NO_MECHANISM, + note="The redaction processor does not traverse span links, and OTTL " + "has no spanlink context and refuses to index links, so no processor " + "in this distribution can reach it. Recorded as a gap.", + ), + Carrier( + "span link trace state", + "CARRIERMARKERLINKTRACESTATE", + PASSED_THROUGH, + NO_MECHANISM, + note="Unreachable for the same reason as the link attribute above.", + ), + Carrier( + "metric exemplar filtered attribute", + "CARRIERMARKEREXEMPLARATTR", + PASSED_THROUGH, + NO_MECHANISM, + note="Not traversed by the allow-list. Assigning to datapoint.exemplars " + "parses but has no effect. Recorded as a gap.", + ), + # --- Values replaced by blocked_values. --- + Carrier( + "allow-listed attribute value matching blocked_values", + "CARRIERMARKERMASKSPANATTR", + MASKED, + BLOCKED_VALUES, + masked_signature=f"-> error.type: Str({MASK})", + note="An allowed key keeps its shape; a recognized secret shape in its " + "value is replaced wholesale rather than partially.", + ), + # --- Carriers the content scrub closes. --- + Carrier( + "span status message", + "CARRIERMARKERSPANSTATUSMESSAGE", + MASKED, + CONTENT_SCRUB, + masked_signature=f"Status message : {SCRUB}", + ), + Carrier( + "span event name", + "CARRIERMARKERSPANEVENTNAME", + MASKED, + CONTENT_SCRUB, + masked_signature=f"-> Name: {SCRUB}", + ), + Carrier( + "span trace state", + "CARRIERMARKERSPANTRACESTATE", + GOVERNED, + CONTENT_SCRUB, + note="Cleared rather than replaced, because an empty trace state is " + "valid and a redaction token is not.", + ), + Carrier( + "log body, scalar", + "CARRIERMARKERLOGBODYSCALAR", + MASKED, + CONTENT_SCRUB, + masked_signature=f"Body: Str({SCRUB})", + ), + Carrier( + "log body, array", + "CARRIERMARKERLOGBODYARRAY", + MASKED, + CONTENT_SCRUB, + masked_signature=f"Body: Str({SCRUB})", + ), + Carrier( + "log body, bytes", + BYTES_MARKER, + MASKED, + CONTENT_SCRUB, + masked_signature=f"Body: Str({SCRUB})", + ), + Carrier( + "scalar log body matching blocked_values", + "CARRIERMARKERMASKLOGBODY", + MASKED, + CONTENT_SCRUB, + masked_signature=f"Body: Str({SCRUB})", + note="blocked_values masks this to **** and the content scrub then " + "replaces the whole body, so the scrub token is what reaches the " + "exporter. Both controls apply; the later one is what is observed.", + ), + Carrier( + "log severity text", + "CARRIERMARKERLOGSEVERITYTEXT", + MASKED, + CONTENT_SCRUB, + masked_signature=f"SeverityText: {SCRUB}", + note="Scrubbed because it is a free-form emitter string with no " + "shipped consumer. The structured severity number is unaffected.", + ), + Carrier( + "log record event name", + "CARRIERMARKERLOGEVENTNAME", + MASKED, + CONTENT_SCRUB, + masked_signature=f"EventName: {SCRUB}", + ), + # --- Carriers a shipped consumer requires. Residual exposure, recorded. --- + Carrier( + "span name", + "CARRIERMARKERSPANNAME", + PASSED_THROUGH, + NO_MECHANISM, + note="Deliberately preserved. Three dashboard TraceQL queries match on " + "span name and baseline.py reads Tempo rootTraceName. This is the " + "carrier most likely to hold prompt text, and it is a recorded gap " + "rather than a scrub, because scrubbing it breaks shipped panels.", + ), + Carrier( + "metric name", + "CARRIERMARKERMETRICNAME", + PASSED_THROUGH, + NO_MECHANISM, + note="Preserved: every Prometheus query in the shipped dashboard " + "selects by metric name.", + ), + Carrier("metric description", "CARRIERMARKERMETRICDESCRIPTION", PASSED_THROUGH, NO_MECHANISM), + Carrier("metric unit", "CARRIERMARKERMETRICUNIT", PASSED_THROUGH, NO_MECHANISM), + # --- Library and schema identity. Normally not emitter content, and + # unfiltered either way: the receiver is unauthenticated and no processor + # reaches these fields, so what they carry is an expectation about a + # well-behaved emitter rather than a control. Recorded as G-INF-11. + Carrier( + "instrumentation scope name", + "CARRIERMARKERSCOPENAME", + PASSED_THROUGH, + NO_MECHANISM, + note="Unfiltered and attacker-settable; G-INF-11.", + ), + Carrier( + "instrumentation scope version", "CARRIERMARKERSCOPEVERSION", PASSED_THROUGH, NO_MECHANISM + ), + Carrier("resource schema URL", "CARRIERMARKERRESOURCESCHEMAURL", PASSED_THROUGH, NO_MECHANISM), + Carrier("scope schema URL", "CARRIERMARKERSCOPESCHEMAURL", PASSED_THROUGH, NO_MECHANISM), +) + + +def _attr(key: str, value: str) -> dict: + return {"key": key, "value": {"stringValue": value}} + + +def traces_payload() -> dict: + """One span carrying a distinct marker in every trace-side carrier.""" + return { + "resourceSpans": [ + { + "resource": { + "attributes": [ + _attr("service.name", "copilot-chat"), + _attr("probe.resource.attr", "CARRIERMARKERRESOURCEATTR"), + *( + _attr(key, marker) + for key, marker in RESTORED_DETECTION_KEYS.items() + ), + ] + }, + "schemaUrl": "https://example.invalid/CARRIERMARKERRESOURCESCHEMAURL", + "scopeSpans": [ + { + "scope": { + "name": "CARRIERMARKERSCOPENAME", + "version": "CARRIERMARKERSCOPEVERSION", + "attributes": [ + _attr("probe.scope.attr", "CARRIERMARKERSCOPEATTR") + ], + }, + "schemaUrl": "https://example.invalid/CARRIERMARKERSCOPESCHEMAURL", + "spans": [ + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b174", + "traceState": "probe=CARRIERMARKERSPANTRACESTATE", + "name": "CARRIERMARKERSPANNAME", + "kind": 1, + "startTimeUnixNano": "1700000000000000000", + "endTimeUnixNano": "1700000001000000000", + "attributes": [ + _attr("probe.span.attr", "CARRIERMARKERSPANATTR"), + # An allow-listed key whose value carries a + # recognized secret shape, so masking can be + # observed separately from dropping. + _attr( + "error.type", + "password=CARRIERMARKERMASKSPANATTR", + ), + ], + "status": { + "code": 2, + "message": "CARRIERMARKERSPANSTATUSMESSAGE", + }, + "events": [ + { + "timeUnixNano": "1700000000500000000", + "name": "CARRIERMARKERSPANEVENTNAME", + "attributes": [ + _attr( + "probe.spanevent.attr", + "CARRIERMARKERSPANEVENTATTR", + ) + ], + } + ], + "links": [ + { + "traceId": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "spanId": "bbbbbbbbbbbbbbbb", + "traceState": "probe=CARRIERMARKERLINKTRACESTATE", + "attributes": [ + _attr( + "probe.link.attr", "CARRIERMARKERLINKATTR" + ) + ], + } + ], + } + ], + } + ], + } + ] + } + + +def logs_payload() -> dict: + """Log records covering every body shape and the non-body log fields.""" + return { + "resourceLogs": [ + { + "resource": {"attributes": [_attr("service.name", "copilot-chat")]}, + "scopeLogs": [ + { + "scope": {"name": "probe"}, + "logRecords": [ + { + "timeUnixNano": "1700000000000000000", + "severityNumber": 9, + "severityText": "CARRIERMARKERLOGSEVERITYTEXT", + "eventName": "CARRIERMARKERLOGEVENTNAME", + "body": {"stringValue": "CARRIERMARKERLOGBODYSCALAR"}, + "attributes": [ + _attr("probe.log.attr", "CARRIERMARKERLOGATTR") + ], + }, + { + "timeUnixNano": "1700000000000000000", + "body": { + "kvlistValue": { + "values": [ + _attr( + "probe.body.key", + "CARRIERMARKERLOGBODYMAP", + ) + ] + } + }, + }, + { + "timeUnixNano": "1700000000000000000", + "body": { + "arrayValue": { + "values": [ + {"stringValue": "CARRIERMARKERLOGBODYARRAY"} + ] + } + }, + }, + { + "timeUnixNano": "1700000000000000000", + "body": {"bytesValue": BYTES_MARKER}, + }, + { + "timeUnixNano": "1700000000000000000", + "body": { + "stringValue": "password=CARRIERMARKERMASKLOGBODY" + }, + }, + ], + } + ], + } + ] + } + + +def metrics_payload() -> dict: + """One datapoint carrying markers in metric metadata, attributes, exemplars.""" + return { + "resourceMetrics": [ + { + "resource": {"attributes": [_attr("service.name", "copilot-chat")]}, + "scopeMetrics": [ + { + "scope": {"name": "probe"}, + "metrics": [ + { + "name": "CARRIERMARKERMETRICNAME", + "description": "CARRIERMARKERMETRICDESCRIPTION", + "unit": "CARRIERMARKERMETRICUNIT", + "sum": { + "aggregationTemporality": 2, + "isMonotonic": True, + "dataPoints": [ + { + "startTimeUnixNano": "1700000000000000000", + "timeUnixNano": "1700000001000000000", + "asInt": "1", + "attributes": [ + _attr( + "probe.datapoint.attr", + "CARRIERMARKERDATAPOINTATTR", + ) + ], + "exemplars": [ + { + "timeUnixNano": "1700000001000000000", + "asInt": "1", + "traceId": ( + "5b8efff798038103d269b633813fc60c" + ), + "spanId": "eee19b7ec3c1b174", + "filteredAttributes": [ + _attr( + "probe.exemplar.attr", + "CARRIERMARKEREXEMPLARATTR", + ) + ], + } + ], + } + ], + }, + } + ], + } + ], + } + ] + } + + +def pinned_collector_image() -> str: + """The digest-pinned Collector image the shipped stack runs.""" + compose = yaml.safe_load(COMPOSE_PATH.read_text(encoding="utf-8")) + return compose["services"][COLLECTOR_SERVICE]["image"] + + +def shipped_collector_config() -> dict: + return yaml.safe_load(COLLECTOR_PATH.read_text(encoding="utf-8")) + + +def derive_probe_config(*, remove_content_processors: bool) -> dict: + """Derive a probe configuration from the shipped file. + + The only permitted substitution for an ordinary run is exporter wiring, so + the Collector emits inspectable records instead of shipping them to a store + this harness does not start. Receivers, processors, and processor ordering + are preserved. The negative control is additionally permitted to remove + every content-minimization processor, and makes no other policy change. + """ + config = copy.deepcopy(shipped_collector_config()) + + config["exporters"] = {"debug": {"verbosity": "detailed"}} + # A ten second batch window would dominate the probe's runtime without + # changing what the policy does to a record. + config["processors"]["batch"]["timeout"] = "1s" + + if remove_content_processors: + for name in CONTENT_PROCESSORS: + config["processors"].pop(name, None) + + for pipeline in config["service"]["pipelines"].values(): + pipeline["exporters"] = ["debug"] + if remove_content_processors: + pipeline["processors"] = [ + name for name in pipeline["processors"] if name not in CONTENT_PROCESSORS + ] + + return config + + +class ContainerRuntime: + """A Docker CLI this platform can actually reach.""" + + def __init__(self, argv: list[str], label: str) -> None: + self._argv = argv + self.label = label + + def docker(self, *args: str, timeout: float = 300.0) -> subprocess.CompletedProcess[str]: + return subprocess.run( # noqa: S603 - fixed argv, no shell + [*self._argv, *args], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + def mount_source(self, path: pathlib.Path) -> str: + """Render a host path in the form this Docker CLI can bind-mount.""" + return path.as_posix() + + +class _WslDockerRuntime(ContainerRuntime): + """Windows hosts where Docker is reachable only inside WSL.""" + + def mount_source(self, path: pathlib.Path) -> str: + translated = subprocess.run( # noqa: S603 - fixed argv, no shell + ["wsl.exe", "-e", "wslpath", "-a", path.as_posix()], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if translated.returncode != 0: + raise RuntimeError(f"wslpath failed: {translated.stderr.strip()}") + return translated.stdout.strip() + + +def detect_runtime() -> ContainerRuntime | None: + """Find a reachable Docker daemon without assuming a platform. + + The platform CLI is tried first so the harness executes on the Linux CI + lane. WSL is a Windows-only fallback for hosts where the daemon lives + inside a distribution and no Windows CLI reaches it. + """ + candidates: list[ContainerRuntime] = [] + if shutil.which("docker"): + candidates.append(ContainerRuntime(["docker"], "platform docker CLI")) + if platform.system() == "Windows" and shutil.which("wsl.exe"): + candidates.append(_WslDockerRuntime(["wsl.exe", "-e", "docker"], "docker inside WSL")) + + for candidate in candidates: + try: + probe = candidate.docker("version", "--format", "{{.Server.Version}}", timeout=60) + except (OSError, subprocess.SubprocessError): + continue + if probe.returncode == 0 and probe.stdout.strip(): + return candidate + return None + + +def strict_runtime_required(env: Mapping[str, str] | None = None) -> bool: + """Whether a missing container runtime must fail this module rather than skip it. + + An explicit `STRICT_RUNTIME_ENV` value decides on its own, in either + direction, so a lane without a usable runtime can opt out deliberately and + a contributor can opt in. With no explicit value the standard `CI` variable + decides, because a run whose green result is read as evidence is exactly the + run that must not skip. + """ + source = os.environ if env is None else env + explicit = source.get(STRICT_RUNTIME_ENV) + chosen = explicit if explicit is not None else source.get("CI", "") + return chosen.strip().lower() not in _FALSE_VALUES + + +NO_RUNTIME_REASON = ( + "container runtime is unavailable: no reachable Docker daemon through the " + "platform CLI or, on Windows, through WSL. The carrier map was not " + "verified by this run." +) + + +def resolve_runtime() -> ContainerRuntime: + """Return a reachable runtime, or stop this module the way strictness requires.""" + found = detect_runtime() + if found is not None: + return found + if strict_runtime_required(): + pytest.fail( + f"{NO_RUNTIME_REASON} Strict mode is active, so this is a failure and " + f"not a skip: the runtime-backed claims in SECURITY.md cite this " + f"module, and a silent skip would leave them citing a run that never " + f"happened. Set {STRICT_RUNTIME_ENV}=0 to allow the skip.", + pytrace=False, + ) + pytest.skip(NO_RUNTIME_REASON) + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +@dataclass +class ProbeResult: + """What the Collector emitted for one configuration.""" + + output: str + config: dict + posted: list[str] = field(default_factory=list) + + +def _wait_for_health(port: int, *, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen( # noqa: S310 - fixed loopback URL + f"http://127.0.0.1:{port}/", timeout=5 + ) as response: + if response.status == 200: + return True + except (urllib.error.URLError, TimeoutError, OSError): + time.sleep(0.5) + return False + + +def _post(port: int, signal: str, payload: dict) -> None: + request = urllib.request.Request( # noqa: S310 - fixed loopback URL + f"http://127.0.0.1:{port}/v1/{signal}", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 + if response.status != 200: + raise RuntimeError(f"{signal} export returned HTTP {response.status}") + + +def run_probe(runtime: ContainerRuntime, *, remove_content_processors: bool) -> ProbeResult: + """Start a disposable Collector, export every marker, capture the output.""" + config = derive_probe_config(remove_content_processors=remove_content_processors) + http_port = _free_port() + health_port = _free_port() + variant = "control" if remove_content_processors else "policy" + container = f"copilot-otel-carrier-probe-{variant}-{http_port}" + + with tempfile.TemporaryDirectory(prefix="copilot-otel-probe-") as workdir: + config_path = pathlib.Path(workdir) / "config.yaml" + config_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + + started = runtime.docker( + "run", + "-d", + "--name", + container, + "-p", + f"127.0.0.1:{http_port}:4318", + "-p", + f"127.0.0.1:{health_port}:13133", + "-v", + f"{runtime.mount_source(config_path)}:/etc/otelcol-contrib/config.yaml:ro", + pinned_collector_image(), + "--config=/etc/otelcol-contrib/config.yaml", + ) + if started.returncode != 0: + raise RuntimeError(f"probe container failed to start: {started.stderr.strip()}") + + try: + if not _wait_for_health(health_port, timeout=60): + logs = runtime.docker("logs", container) + raise RuntimeError( + f"probe Collector never became healthy; output:\n{logs.stdout}\n{logs.stderr}" + ) + + posted = [] + for signal, payload in ( + ("traces", traces_payload()), + ("logs", logs_payload()), + ("metrics", metrics_payload()), + ): + _post(http_port, signal, payload) + posted.append(signal) + + output = _await_all_signals(runtime, container, timeout=30) + return ProbeResult(output=output, config=config, posted=posted) + finally: + runtime.docker("rm", "-f", container, timeout=120) + + +def _await_all_signals(runtime: ContainerRuntime, container: str, *, timeout: float) -> str: + """Wait until the exporter has rendered all three signals, then return them.""" + expected = ('"otelcol.signal": "traces"', '"otelcol.signal": "logs"', + '"otelcol.signal": "metrics"') + deadline = time.monotonic() + timeout + output = "" + while time.monotonic() < deadline: + logs = runtime.docker("logs", container, timeout=60) + output = logs.stdout + logs.stderr + if all(marker in output for marker in expected): + return output + time.sleep(1) + missing = [marker for marker in expected if marker not in output] + raise RuntimeError(f"exporter never rendered {missing}; captured output:\n{output}") + + +def classify(carrier: Carrier, policy_output: str, control_output: str) -> str: + """Classify one carrier from the paired runs. + + The control decides observability first. Only a marker the instrument can + render is eligible to be called governed; anything else is `unobservable` + and is carried as a gap rather than counted as a control. + """ + if carrier.marker not in control_output: + return UNOBSERVABLE + if carrier.marker in policy_output: + return PASSED_THROUGH + if carrier.masked_signature and carrier.masked_signature in policy_output: + return MASKED + return GOVERNED + + +@pytest.fixture(scope="module") +def runtime() -> ContainerRuntime: + return resolve_runtime() + + +@pytest.fixture(scope="module") +def policy_output(runtime: ContainerRuntime) -> str: + return run_probe(runtime, remove_content_processors=False).output + + +@pytest.fixture(scope="module") +def control_output(runtime: ContainerRuntime) -> str: + return run_probe(runtime, remove_content_processors=True).output + + +class TestMarkerInventory: + """The markers must be distinguishable before any classification means anything.""" + + def test_every_carrier_has_a_unique_marker(self) -> None: + markers = [carrier.marker for carrier in CARRIERS] + assert len(markers) == len(set(markers)) + + def test_no_marker_is_a_substring_of_another(self) -> None: + overlapping = [ + (one.name, other.name) + for one in CARRIERS + for other in CARRIERS + if one is not other and one.marker in other.marker + ] + assert overlapping == [] + + def test_every_carrier_declares_a_known_classification(self) -> None: + allowed = {GOVERNED, MASKED, PASSED_THROUGH} + unknown = [c.name for c in CARRIERS if c.expected not in allowed] + assert unknown == [], ( + "An unobservable or unclassified carrier is a gap, not a baseline. " + "Record it in SECURITY.md rather than encoding it as an expectation." + ) + + def test_a_masked_expectation_declares_what_masking_looks_like(self) -> None: + undeclared = [c.name for c in CARRIERS if c.expected == MASKED and not c.masked_signature] + assert undeclared == [] + + def test_a_passed_through_carrier_declares_no_mechanism(self) -> None: + """A carrier cannot claim a control and pass through at the same time.""" + contradictory = [ + c.name + for c in CARRIERS + if c.expected == PASSED_THROUGH and c.mechanism != NO_MECHANISM + ] + assert contradictory == [] + + def test_a_handled_carrier_names_the_mechanism_that_handles_it(self) -> None: + unattributed = [ + c.name + for c in CARRIERS + if c.expected in {GOVERNED, MASKED} and c.mechanism == NO_MECHANISM + ] + assert unattributed == [] + + +class TestProbeConfigDerivation: + """The configuration under test must be the shipped one, minus stated changes.""" + + def test_the_probe_preserves_the_shipped_processor_order(self) -> None: + shipped = shipped_collector_config() + derived = derive_probe_config(remove_content_processors=False) + for name, pipeline in derived["service"]["pipelines"].items(): + assert pipeline["processors"] == shipped["service"]["pipelines"][name]["processors"] + + def test_the_probe_preserves_the_shipped_redaction_policy(self) -> None: + shipped = shipped_collector_config() + derived = derive_probe_config(remove_content_processors=False) + assert derived["processors"]["redaction"] == shipped["processors"]["redaction"] + + def test_the_control_removes_only_the_content_processors(self) -> None: + policy = derive_probe_config(remove_content_processors=False) + control = derive_probe_config(remove_content_processors=True) + assert set(policy["processors"]) - set(control["processors"]) == set(CONTENT_PROCESSORS) + for name, pipeline in control["service"]["pipelines"].items(): + expected = [ + step + for step in policy["service"]["pipelines"][name]["processors"] + if step not in CONTENT_PROCESSORS + ] + assert pipeline["processors"] == expected + + def test_the_probe_runs_the_image_the_stack_pins(self) -> None: + assert "@sha256:" in pinned_collector_image() + + +class TestStrictRuntimeMode: + """A run that produces no evidence must not be able to pass quietly. + + This module's output is what `SECURITY.md` cites. If the lane ever loses a + usable container runtime, a skip would keep the suite green while those + claims cite a run that never happened, which is the defect this suite + exists to prevent, one level up. The rule that decides between skipping and + failing is therefore asserted rather than trusted. + """ + + @pytest.mark.parametrize( + ("env", "expected"), + [ + ({}, False), + ({"CI": "true"}, True), + ({"CI": "1"}, True), + ({"CI": "false"}, False), + ({"CI": ""}, False), + ({STRICT_RUNTIME_ENV: "1"}, True), + ({STRICT_RUNTIME_ENV: "true", "CI": "false"}, True), + ({STRICT_RUNTIME_ENV: "0", "CI": "true"}, False), + ({STRICT_RUNTIME_ENV: "off", "CI": "true"}, False), + ], + ) + def test_strictness_follows_the_environment(self, env: dict[str, str], expected: bool) -> None: + assert strict_runtime_required(env) is expected + + def test_a_missing_runtime_fails_when_strictness_is_on( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setitem(globals(), "detect_runtime", lambda: None) + monkeypatch.setenv(STRICT_RUNTIME_ENV, "1") + with pytest.raises(pytest.fail.Exception) as raised: + resolve_runtime() + assert STRICT_RUNTIME_ENV in str(raised.value) + + def test_a_missing_runtime_skips_when_strictness_is_off( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setitem(globals(), "detect_runtime", lambda: None) + monkeypatch.setenv(STRICT_RUNTIME_ENV, "0") + with pytest.raises(pytest.skip.Exception): + resolve_runtime() + + def test_a_reachable_runtime_is_returned_under_strictness( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + stub = ContainerRuntime(["docker"], "stub runtime") + monkeypatch.setitem(globals(), "detect_runtime", lambda: stub) + monkeypatch.setenv(STRICT_RUNTIME_ENV, "1") + assert resolve_runtime() is stub + + +class TestProcessorChainPosition: + """A content processor must sit after memory_limiter and before batch. + + Ahead of `memory_limiter` it would do work the limiter exists to shed; + after `batch` it would run on assembled batches instead of records. + """ + + @pytest.mark.parametrize("processor", CONTENT_PROCESSORS) + def test_the_processor_sits_between_the_limiter_and_the_batcher(self, processor: str) -> None: + pipelines = shipped_collector_config()["service"]["pipelines"] + for name, pipeline in pipelines.items(): + chain = pipeline["processors"] + assert processor in chain, f"{name} does not apply {processor}" + assert chain.index("memory_limiter") < chain.index(processor) < chain.index("batch"), ( + f"{name} applies {processor} outside the limiter-to-batch window" + ) + + +class TestNegativeControl: + """Validate the instrument before trusting anything it reports.""" + + @pytest.mark.parametrize("carrier", CARRIERS, ids=lambda c: c.name) + def test_the_control_without_content_processors_renders_every_marker( + self, carrier: Carrier, control_output: str + ) -> None: + assert carrier.marker in control_output, ( + f"{carrier.name} is not rendered even with the content processors " + f"removed, so its absence under policy proves nothing. Classify it " + f"{UNOBSERVABLE} and record it as a gap rather than as a control." + ) + + +class TestCarrierMap: + """The recorded map is the regression boundary.""" + + @pytest.mark.parametrize("carrier", CARRIERS, ids=lambda c: c.name) + def test_each_carrier_matches_its_recorded_classification( + self, carrier: Carrier, policy_output: str, control_output: str + ) -> None: + observed = classify(carrier, policy_output, control_output) + assert observed == carrier.expected, ( + f"{carrier.name} is now {observed}, recorded as {carrier.expected}. " + "Rebaselining this entry is a policy decision and needs its own " + "evidence; do not edit the map to make this pass." + ) + + +class TestRestoredDetectionAttributes: + """An allow-list entry is a claim until the Collector is observed keeping it. + + Three of these keys were dropped by the allow-list while shipped consumers + read them, and one of them exists in two candidate spellings because the + Prometheus label it produces cannot be reversed into an OTLP key. Asserting + survival per key is what makes a wrong entry fail here rather than in a + silently empty panel. + """ + + @pytest.mark.parametrize("key", sorted(RESTORED_DETECTION_KEYS)) + def test_the_shipped_policy_keeps_the_attribute(self, key: str, policy_output: str) -> None: + assert RESTORED_DETECTION_KEYS[key] in policy_output, ( + f"{key} is in the allow-list but was dropped by the running Collector" + ) + + @pytest.mark.parametrize("key", sorted(RESTORED_DETECTION_KEYS)) + def test_the_attribute_is_rendered_under_its_own_key( + self, key: str, policy_output: str + ) -> None: + assert f"-> {key}: Str({RESTORED_DETECTION_KEYS[key]})" in policy_output + + +class TestShippedConfigurationStartsClean: + """One start of the configuration as shipped, with nothing substituted. + + The carrier harness replaces the exporter block so it can inspect records, + which means it can never observe an exporter-level startup warning. This + runs the file exactly as an operator runs it. + """ + + def test_the_shipped_configuration_logs_no_deprecation_warning( + self, runtime: ContainerRuntime + ) -> None: + container = f"copilot-otel-alias-check-{_free_port()}" + started = runtime.docker( + "run", + "-d", + "--name", + container, + "-v", + f"{runtime.mount_source(COLLECTOR_PATH)}:/etc/otelcol-contrib/config.yaml:ro", + pinned_collector_image(), + "--config=/etc/otelcol-contrib/config.yaml", + ) + if started.returncode != 0: + raise RuntimeError(f"shipped configuration failed to start: {started.stderr.strip()}") + try: + # The exporter is built during startup, so the warning appears in + # the first seconds. Waiting for the LGTM connection to fail is not + # required and would only add flake. + deadline = time.monotonic() + 30 + output = "" + while time.monotonic() < deadline: + logs = runtime.docker("logs", container, timeout=60) + output = logs.stdout + logs.stderr + if "Everything is ready" in output: + break + time.sleep(1) + assert "Everything is ready" in output, f"Collector never finished starting:\n{output}" + assert "deprecated" not in output.lower(), ( + f"the shipped configuration still triggers a deprecation warning:\n{output}" + ) + finally: + runtime.docker("rm", "-f", container, timeout=120) diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py new file mode 100644 index 000000000..64b1618e8 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py @@ -0,0 +1,512 @@ +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Behavioral tests for the shared helper input policy. + +Every rejection case asserts that the refusal happens before any request or +write, which is the property that matters: a guard that rejects after the +side effect is not a guard. +""" + +from __future__ import annotations + +import io +import json +import os +import pathlib +import subprocess +import sys +import urllib.request + +import inspect_metrics +import pytest +import validate_dashboard +import verify +from _input_policy import ( + DEFAULT_ALLOWED_PORTS, + PolicyError, + check_url, + contain_path, + open_url, + origin_of, + require_credentials, +) + +EXAMPLES_DIR = pathlib.Path(__file__).resolve().parents[1] / "examples" + + +class TestScheme: + """Only http and https are addressable.""" + + @pytest.mark.parametrize( + "url", + [ + "file:///etc/passwd", + "ftp://localhost/data", + "data:text/plain,hello", + "gopher://localhost:70/", + "//localhost:3000/api", + ], + ) + def test_a_non_http_scheme_is_refused(self, url: str) -> None: + with pytest.raises(PolicyError): + check_url(url) + + def test_loopback_http_is_allowed(self) -> None: + assert check_url("http://localhost:3000/api/health").hostname == "localhost" + + +class TestAuthority: + """Credentials, missing hosts, and malformed authorities are refused.""" + + @pytest.mark.parametrize( + "url", + [ + "http://user@localhost:3000/", + "http://user:pass@localhost:3000/", + "http://:pass@localhost:3000/", + ], + ) + def test_userinfo_is_refused(self, url: str) -> None: + with pytest.raises(PolicyError, match="credentials"): + check_url(url) + + def test_a_missing_host_is_refused(self) -> None: + with pytest.raises(PolicyError): + check_url("http:///api/health") + + def test_a_malformed_port_is_refused(self) -> None: + with pytest.raises(PolicyError): + check_url("http://localhost:notaport/") + + +class TestPorts: + """Local requests stay on the stack's own ports.""" + + @pytest.mark.parametrize("port", sorted(DEFAULT_ALLOWED_PORTS)) + def test_each_stack_port_is_allowed(self, port: int) -> None: + assert check_url(f"http://127.0.0.1:{port}/").port == port + + @pytest.mark.parametrize("port", [22, 80, 443, 8080, 5432]) + def test_an_unrelated_local_port_is_refused(self, port: int) -> None: + with pytest.raises(PolicyError, match="local port"): + check_url(f"http://127.0.0.1:{port}/") + + +class TestRemoteOptIn: + """A remote target needs an explicit opt-in and TLS.""" + + def test_a_remote_host_is_refused_without_opt_in(self) -> None: + with pytest.raises(PolicyError, match="non-loopback"): + check_url("https://grafana.example.com/") + + def test_a_remote_host_is_allowed_with_opt_in(self) -> None: + assert check_url("https://grafana.example.com/", allow_remote=True).scheme == "https" + + def test_plaintext_remote_is_refused_even_with_opt_in(self) -> None: + with pytest.raises(PolicyError, match="plaintext"): + check_url("http://grafana.example.com/", allow_remote=True) + + +class TestRedirects: + """Policy is re-applied to redirect targets, not only the original URL.""" + + def _handler(self) -> urllib.request.HTTPRedirectHandler: + opener = open_url.__globals__["_PolicyRedirectHandler"] + return opener(allow_remote=False, allowed_ports=DEFAULT_ALLOWED_PORTS) + + def test_a_redirect_off_loopback_is_refused(self) -> None: + request = urllib.request.Request("http://localhost:3000/api") + with pytest.raises(PolicyError, match="non-loopback"): + self._handler().redirect_request( + request, None, 302, "Found", {}, "http://evil.example.com/" + ) + + def test_a_redirect_to_a_file_url_is_refused(self) -> None: + request = urllib.request.Request("http://localhost:3000/api") + with pytest.raises(PolicyError, match="scheme"): + self._handler().redirect_request(request, None, 302, "Found", {}, "file:///etc/passwd") + + def test_a_redirect_to_an_unrelated_local_port_is_refused(self) -> None: + request = urllib.request.Request("http://localhost:3000/api") + with pytest.raises(PolicyError, match="local port"): + self._handler().redirect_request( + request, None, 302, "Found", {}, "http://localhost:22/" + ) + + def test_open_url_refuses_before_opening_anything(self, monkeypatch) -> None: + opened: list[str] = [] + monkeypatch.setattr( + urllib.request.OpenerDirector, + "open", + lambda self, *a, **k: opened.append("opened"), + ) + with pytest.raises(PolicyError): + open_url("http://evil.example.com/") + assert opened == [], "a refused URL still reached the opener" + + +class TestPathContainment: + """A configurable path cannot escape its root.""" + + def test_a_contained_relative_path_resolves(self, tmp_path: pathlib.Path) -> None: + assert contain_path("snapshot.json", tmp_path) == (tmp_path / "snapshot.json").resolve() + + def test_traversal_is_refused(self, tmp_path: pathlib.Path) -> None: + with pytest.raises(PolicyError): + contain_path("../../etc/passwd", tmp_path) + + def test_an_absolute_path_outside_the_root_is_refused(self, tmp_path: pathlib.Path) -> None: + outside = tmp_path.parent / "outside.json" + with pytest.raises(PolicyError): + contain_path(outside, tmp_path) + + def test_a_symlink_pointing_outside_the_root_is_refused(self, tmp_path: pathlib.Path) -> None: + root = tmp_path / "root" + root.mkdir() + target = tmp_path / "outside" + target.mkdir() + link = root / "escape" + try: + link.symlink_to(target, target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("symlink creation is not permitted in this environment") + with pytest.raises(PolicyError): + contain_path(link / "snapshot.json", root) + + def test_the_root_itself_is_allowed(self, tmp_path: pathlib.Path) -> None: + assert contain_path(tmp_path, tmp_path) == tmp_path.resolve() + + +class TestCredentials: + """Missing configuration is reported as configuration, before any request.""" + + def test_both_missing_variables_are_named(self, monkeypatch) -> None: + monkeypatch.delenv("COPILOT_OTEL_GRAFANA_USER", raising=False) + monkeypatch.delenv("COPILOT_OTEL_GRAFANA_PASSWORD", raising=False) + with pytest.raises(PolicyError) as excinfo: + require_credentials("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD") + assert "COPILOT_OTEL_GRAFANA_USER" in str(excinfo.value) + assert "COPILOT_OTEL_GRAFANA_PASSWORD" in str(excinfo.value) + + def test_a_single_missing_variable_is_named(self, monkeypatch) -> None: + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", "admin") + monkeypatch.delenv("COPILOT_OTEL_GRAFANA_PASSWORD", raising=False) + with pytest.raises(PolicyError, match="COPILOT_OTEL_GRAFANA_PASSWORD"): + require_credentials("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD") + + def test_an_empty_value_counts_as_missing(self, monkeypatch) -> None: + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", "admin") + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", "") + with pytest.raises(PolicyError, match="COPILOT_OTEL_GRAFANA_PASSWORD"): + require_credentials("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD") + + def test_supplied_credentials_are_returned_in_order(self, monkeypatch) -> None: + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", "operator") + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", "chosen-by-the-user") + assert require_credentials( + "COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD" + ) == ("operator", "chosen-by-the-user") + + +def run_helper( + name: str, args: list[str], env_overrides: dict[str, str] +) -> subprocess.CompletedProcess[str]: + """Run a bundled helper in its own process with a controlled environment. + + A refusal exits before argparse and before any socket is opened. Running + the helpers as processes is what proves the wiring; importing the policy + module directly would only re-test the module these tests already cover. + """ + env = dict(os.environ) + env.update(env_overrides) + env["COPILOT_OTEL_ALLOW_REMOTE"] = "0" + return subprocess.run( + [sys.executable, str(EXAMPLES_DIR / name), *args], + capture_output=True, + text=True, + timeout=60, + env=env, + cwd=str(EXAMPLES_DIR), + check=False, + ) + + +class TestHelperWiring: + """Each helper actually calls the shared policy, not merely imports it. + + Without these, deleting a `contain_path` call from a helper would leave the + policy tests above green while restoring the exact write this change closed. + """ + + def test_baseline_refuses_a_snapshot_path_outside_the_cache_root( + self, tmp_path: pathlib.Path + ) -> None: + target = tmp_path / "escaped-snapshot.json" + result = run_helper("baseline.py", ["capture"], {"COPILOT_OTEL_BASELINE": str(target)}) + assert result.returncode != 0 + assert "refusing a path outside" in (result.stderr + result.stdout) + assert not target.exists(), "a refused snapshot path was still written" + + def test_baseline_refuses_a_traversal_snapshot_path(self, tmp_path: pathlib.Path) -> None: + result = run_helper( + "baseline.py", ["capture"], {"COPILOT_OTEL_BASELINE": "../../escaped.json"} + ) + assert result.returncode != 0 + assert "refusing a path outside" in (result.stderr + result.stdout) + + def test_validate_dashboard_refuses_a_dashboard_outside_the_examples_directory( + self, tmp_path: pathlib.Path + ) -> None: + outside = tmp_path / "someone-elses-dashboard.json" + outside.write_text("{}", encoding="utf-8") + result = run_helper( + "validate_dashboard.py", + [str(outside)], + { + "COPILOT_OTEL_GRAFANA_USER": "operator", + "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", + }, + ) + assert result.returncode != 0 + assert "refusing a path outside" in (result.stderr + result.stdout) + + def test_validate_dashboard_refuses_a_non_loopback_endpoint_without_opt_in(self) -> None: + result = run_helper( + "validate_dashboard.py", + [], + { + "COPILOT_OTEL_GRAFANA": "https://grafana.example.com", + "COPILOT_OTEL_GRAFANA_USER": "operator", + "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", + }, + ) + assert result.returncode != 0 + assert "non-loopback" in (result.stderr + result.stdout) + + def test_validate_dashboard_refuses_an_unguarded_query_endpoint(self) -> None: + """The Prometheus override was previously unchecked entirely.""" + result = run_helper( + "validate_dashboard.py", + [], + { + "COPILOT_OTEL_PROMETHEUS": "http://evil.example.com:9090", + "COPILOT_OTEL_GRAFANA_USER": "operator", + "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", + }, + ) + assert result.returncode != 0 + assert "non-loopback" in (result.stderr + result.stdout) + + def test_validate_dashboard_reports_missing_credentials_as_configuration(self) -> None: + result = run_helper( + "validate_dashboard.py", + [], + {"COPILOT_OTEL_GRAFANA_USER": "", "COPILOT_OTEL_GRAFANA_PASSWORD": ""}, + ) + assert result.returncode != 0 + combined = result.stderr + result.stdout + assert "COPILOT_OTEL_GRAFANA_USER" in combined + assert "COPILOT_OTEL_GRAFANA_PASSWORD" in combined + + +def _json_response(payload: dict) -> io.BytesIO: + """A minimal stand-in for what open_url returns.""" + return io.BytesIO(json.dumps(payload).encode("utf-8")) + + +# Shaped to satisfy every reader in validate_dashboard at once: a Grafana +# import result, a Prometheus query result, a Prometheus label listing, a Tempo +# search result, and a TraceQL metrics result. +_ANY_STORE_RESPONSE = { + "status": "success", + "url": "/d/copilot-otel/copilot", + "data": {"result": []}, + "traces": [], + "series": [], +} + + +class TestGrafanaCredentialScope: + """The Grafana credential reaches Grafana and nothing else. + + The defect this replaces was a shared request builder that attached the + header unconditionally. Five of the six call sites address Prometheus or + Tempo, neither of which authenticates against Grafana, so every panel + replay handed a third-party store an admin credential for a fourth. + """ + + def test_a_request_built_without_a_credential_carries_no_authorization(self) -> None: + request = validate_dashboard.build_request("http://localhost:9090/api/v1/query") + assert "Authorization" not in request.headers + + def test_a_request_built_with_a_credential_carries_it(self) -> None: + request = validate_dashboard.build_request( + "http://localhost:3000/api/dashboards/db", + b"{}", + "POST", + authorization="Basic supplied-at-the-call-site", + ) + assert request.headers["Authorization"] == "Basic supplied-at-the-call-site" + + def test_only_the_grafana_request_carries_the_credential_end_to_end( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Drive the real tool and inspect every request it actually builds. + + This is the assertion that covers all six call sites rather than a + representative one: the shipped dashboard is walked, every panel query + is issued, and each resulting request object is inspected. + """ + sent: list[urllib.request.Request] = [] + + def fake_open_url(request, **kwargs): # noqa: ANN001, ANN202 + sent.append(request) + return _json_response(_ANY_STORE_RESPONSE) + + monkeypatch.setattr(validate_dashboard, "open_url", fake_open_url) + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", "operator") + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", "chosen-by-the-user") + monkeypatch.delenv("COPILOT_OTEL_ALLOW_REMOTE", raising=False) + + assert validate_dashboard.main([]) == 0 + + authorized = [r for r in sent if "Authorization" in r.headers] + assert len(sent) > 1, "the dashboard walk issued no store queries" + assert len(authorized) == 1, ( + "exactly one request may carry the Grafana credential; " + f"{len(authorized)} did" + ) + assert authorized[0].full_url.startswith("http://localhost:3000/") + for request in sent: + if request is authorized[0]: + continue + assert "Authorization" not in request.headers, ( + f"{request.full_url} received the Grafana credential" + ) + + +class TestRedirectCredentialHandling: + """A redirect may not carry a credential to an origin it was not issued for.""" + + def _handler( + self, *, allow_remote: bool = False + ) -> urllib.request.HTTPRedirectHandler: + handler = open_url.__globals__["_PolicyRedirectHandler"] + return handler(allow_remote=allow_remote, allowed_ports=DEFAULT_ALLOWED_PORTS) + + def _redirect( + self, start: str, target: str, *, allow_remote: bool = False + ) -> urllib.request.Request: + request = urllib.request.Request(start) + request.add_header("Authorization", "Basic issued-for-the-original-origin") + request.add_header("Proxy-Authorization", "Basic proxy") + request.add_header("Cookie", "session=abc") + request.add_header("Accept", "application/json") + return self._handler(allow_remote=allow_remote).redirect_request( + request, None, 302, "Found", {}, target + ) + + @pytest.mark.parametrize("header", ["Authorization", "Proxy-Authorization", "Cookie"]) + def test_a_redirect_to_another_local_port_drops_each_credential_header( + self, header: str + ) -> None: + redirected = self._redirect("http://localhost:3000/api", "http://localhost:9090/api") + assert not any(name.lower() == header.lower() for name in redirected.headers) + + def test_a_same_origin_redirect_keeps_the_credential(self) -> None: + redirected = self._redirect("http://localhost:3000/api", "http://localhost:3000/other") + assert redirected.headers["Authorization"] == "Basic issued-for-the-original-origin" + + def test_an_explicit_default_port_is_the_same_origin(self) -> None: + """`https://h/x` and `https://h:443/y` are one origin, not two.""" + redirected = self._redirect( + "https://grafana.example.com/api", + "https://grafana.example.com:443/other", + allow_remote=True, + ) + assert redirected.headers["Authorization"] == "Basic issued-for-the-original-origin" + + def test_a_different_remote_host_drops_the_credential(self) -> None: + redirected = self._redirect( + "https://grafana.example.com/api", + "https://someone-else.example.com/api", + allow_remote=True, + ) + assert "Authorization" not in redirected.headers + + def test_a_different_remote_port_drops_the_credential(self) -> None: + redirected = self._redirect( + "https://grafana.example.com/api", + "https://grafana.example.com:9090/api", + allow_remote=True, + ) + assert "Authorization" not in redirected.headers + + def test_a_non_credential_header_survives_an_origin_change(self) -> None: + redirected = self._redirect("http://localhost:3000/api", "http://localhost:9090/api") + assert redirected.headers["Accept"] == "application/json" + + def test_the_credential_headers_are_dropped_together(self) -> None: + redirected = self._redirect("http://localhost:3000/api", "http://localhost:9090/api") + remaining = {name.lower() for name in redirected.headers} + assert remaining.isdisjoint({"authorization", "proxy-authorization", "cookie"}) + + +class TestOriginNormalization: + """Origin comparison is the mechanism the redirect rule rests on.""" + + @pytest.mark.parametrize( + ("one", "other"), + [ + ("http://h/a", "http://h:80/b"), + ("https://h/a", "https://h:443/b"), + ("http://H/a", "http://h/b"), + ], + ) + def test_equivalent_urls_share_an_origin(self, one: str, other: str) -> None: + assert origin_of(one) == origin_of(other) + + @pytest.mark.parametrize( + ("one", "other"), + [ + ("http://h/a", "https://h/a"), + ("http://h/a", "http://other/a"), + ("http://h:3000/a", "http://h:9090/a"), + ], + ) + def test_different_urls_have_different_origins(self, one: str, other: str) -> None: + assert origin_of(one) != origin_of(other) + + +class TestPolicyRoutedHelpers: + """Both previously bypassing helpers reach the network only through policy. + + Their endpoints are loopback constants today, so the bypass was not + currently exploitable. It was a maintenance defect: a later endpoint change + in either file would have escaped every control the other helpers inherit. + """ + + @pytest.mark.parametrize("module", [verify, inspect_metrics], ids=lambda m: m.__name__) + def test_the_helper_reaches_http_through_the_policy_boundary( + self, module, monkeypatch: pytest.MonkeyPatch + ) -> None: + opened: list[str] = [] + + def fake_open_url(url, **kwargs): # noqa: ANN001, ANN202 + opened.append(url) + return _json_response({"data": []}) + + monkeypatch.setattr(module, "open_url", fake_open_url) + module.api(module.PROM, "/api/v1/label/__name__/values") + assert opened == ["http://localhost:9090/api/v1/label/__name__/values"] + + @pytest.mark.parametrize( + "name", ["verify.py", "inspect_metrics.py", "validate_dashboard.py", "baseline.py"] + ) + def test_no_bundled_helper_opens_a_url_outside_the_policy_module(self, name: str) -> None: + source = (EXAMPLES_DIR / name).read_text(encoding="utf-8") + assert "urlopen(" not in source, ( + f"{name} opens a URL directly; every helper must go through open_url " + "so scheme, authority, port, redirect, and remote-opt-in rules apply" + ) + diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py new file mode 100644 index 000000000..b82e05b0f --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py @@ -0,0 +1,580 @@ +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Static checks over the local stack configuration shipped with this skill. + +These tests parse the committed Compose and Collector documents and simulate the +declared attribute policy. They never start a container and never assert that a +running Collector behaves as configured; they assert only that the configuration +this skill ships declares the intended policy. +""" + +from __future__ import annotations + +import json +import pathlib +import re +from dataclasses import dataclass +from typing import Any + +import pytest +import yaml + +SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1] +EXAMPLES_DIR = SKILL_ROOT / "examples" +COMPOSE_PATH = EXAMPLES_DIR / "compose.yaml" +COLLECTOR_PATH = EXAMPLES_DIR / "otel-collector-local.yaml" +DASHBOARD_PATH = EXAMPLES_DIR / "dashboards" / "copilot-otel.json" +BASELINE_PATH = EXAMPLES_DIR / "baseline.py" + +MASK = "****" +SCRUB = "[redacted]" + +DIGEST_REFERENCE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$") + +# --- Shipped-consumer derivation ------------------------------------------- +# +# The default disposition for a carrier the Collector does not govern is to +# scrub it. The only thing that earns an exception is a shipped consumer that +# reads it, so the consumer set has to be derived from the shipped artifacts +# rather than curated by hand. A hand-curated set cannot notice a newly +# required attribute, and a scrub decision made against a stale set breaks a +# panel silently. + +# Label names Prometheus reserves or Grafana generates. Excluded by rule, not +# by listing the ones this dashboard happens to use today. +PROMETHEUS_RESERVED_LABELS = frozenset({"le", "quantile", "__name__", "job", "instance"}) + +# A metric reference is an identifier immediately followed by a label matcher +# or a range selector. A function is followed by "(", so this separates the two +# without enumerating PromQL's function set. +PROMQL_METRIC = re.compile(r"\b([a-z_][a-z0-9_]*)\s*[{\[]") +# Grafana resolves label_values(metric, label): metric first, label second. +PROMQL_LABEL_VALUES = re.compile( + r"label_values\(\s*([a-z_][a-z0-9_]*)\s*,\s*([a-z_][a-z0-9_]*)\s*\)" +) +PROMQL_GROUPING = re.compile(r"\b(?:by|without)\s*\(([^)]*)\)") +PROMQL_MATCHER = re.compile(r"([a-zA-Z_][a-zA-Z0-9_]*)\s*(?:=~|!~|=|!=)\s*\"") +# baseline.py reads label values directly rather than through a panel. Its +# reads are consumers too; leaving them out of the inventory is how a detection +# control gets scrubbed without anything noticing. +BASELINE_LABEL_READ = re.compile(r"label_values\(\s*\"([a-z_][a-z0-9_]*)\"\s*\)") + +# TraceQL. The lookbehind keeps `service.name=` and `mode_name !=` from +# reading as span-name matchers; only a bare `name` is the intrinsic. +TRACEQL_SPAN_NAME = re.compile(r"(? frozenset[str]: + """OTLP attribute keys a shipped consumer reads by their real name.""" + return self.span_attributes | self.resource_attributes + + +def _promql_labels(expr: str) -> set[str]: + labels: set[str] = set() + for clause in PROMQL_GROUPING.findall(expr): + labels.update(part.strip() for part in clause.split(",") if part.strip()) + labels.update(PROMQL_MATCHER.findall(expr)) + return labels + + +def shipped_consumers() -> ShippedConsumers: + """Derive the consumer inventory from the dashboard and the baseline helper.""" + dashboard = json.loads(DASHBOARD_PATH.read_text(encoding="utf-8")) + + metric_names: set[str] = set() + labels: set[str] = set() + span_names: list[str] = [] + span_attributes: set[str] = set() + resource_attributes: set[str] = set() + + for variable in dashboard.get("templating", {}).get("list", []): + query = variable.get("query") + if isinstance(query, str): + for metric, label in PROMQL_LABEL_VALUES.findall(query): + metric_names.add(metric) + labels.add(label) + + for panel in dashboard.get("panels", []): + if panel.get("type") == "row": + continue + source = panel.get("datasource", {}).get("type") + for target in panel.get("targets", []): + expr = target.get("expr") or target.get("query") or "" + if source == "prometheus": + metric_names.update(PROMQL_METRIC.findall(expr)) + labels.update(_promql_labels(expr)) + elif source == "tempo": + span_names.extend(TRACEQL_SPAN_NAME.findall(expr)) + for scope, key in TRACEQL_ATTRIBUTE.findall(expr): + target_set = span_attributes if scope == "span" else resource_attributes + target_set.add(key.rstrip(".")) + + # A `$name` token is a Grafana template variable, not a stored label. + labels = {label for label in labels if not label.startswith("$")} + + baseline_source = BASELINE_PATH.read_text(encoding="utf-8") + labels.update(BASELINE_LABEL_READ.findall(baseline_source)) + labels = { + label + for label in labels + if not label.startswith("$") and label not in PROMETHEUS_RESERVED_LABELS + } + trace_fields = { + field for field in ("rootTraceName", "durationMs") if field in baseline_source + } + + return ShippedConsumers( + metric_names=frozenset(metric_names), + prometheus_labels=frozenset(labels), + span_name_matchers=tuple(span_names), + span_attributes=frozenset(span_attributes), + resource_attributes=frozenset(resource_attributes), + trace_fields=frozenset(trace_fields), + ) + + +def prometheus_label_for(attribute_key: str) -> str: + """Render an OTLP attribute key the way Prometheus stores it as a label. + + Comparing in this direction avoids inventing an OTLP spelling from a label. + `copilot_chat_edit_source` has two plausible sources, and reconstructing one + of them would be the guess this whole change exists to stop making. + """ + return re.sub(r"[^a-zA-Z0-9_]", "_", attribute_key) + + +# Attributes observed carrying plaintext content on spans even with content +# capture left at its default. None may appear in the allow-list. +OBSERVED_CONTENT_ATTRIBUTES = frozenset( + { + "copilot_chat.user_request", + "copilot_chat.reasoning_content", + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.system_instructions", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + } +) + + +class ConfigError(ValueError): + """Raised when a configuration document cannot yield the requested value.""" + + +def load_yaml_text(text: str) -> dict[str, Any]: + """Parse YAML text into a mapping, raising ConfigError on anything else.""" + try: + document = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise ConfigError(f"document is not valid YAML: {exc}") from exc + if not isinstance(document, dict): + raise ConfigError("document root is not a mapping") + return document + + +def load_yaml_file(path: pathlib.Path) -> dict[str, Any]: + """Parse a YAML file into a mapping.""" + if not path.is_file(): + raise ConfigError(f"missing configuration file: {path.name}") + return load_yaml_text(path.read_text(encoding="utf-8")) + + +def published_ports(service: dict[str, Any]) -> list[str]: + """Return the declared host port mappings for a Compose service.""" + return [str(entry) for entry in service.get("ports", []) or []] + + +def redaction_policy(collector: dict[str, Any]) -> dict[str, Any]: + """Return the redaction processor configuration from a Collector document.""" + processors = collector.get("processors") or {} + if not isinstance(processors, dict): + raise ConfigError("processors is not a mapping") + for name, config in processors.items(): + if name == "redaction" or str(name).startswith("redaction/"): + return config or {} + raise ConfigError("no redaction processor is declared") + + +def allowed_keys(collector: dict[str, Any]) -> frozenset[str]: + """Return the allow-listed attribute keys declared by the redaction processor.""" + policy = redaction_policy(collector) + return frozenset(str(key) for key in policy.get("allowed_keys", []) or []) + + +def blocked_values(collector: dict[str, Any]) -> tuple[re.Pattern[str], ...]: + """Return the compiled value patterns the redaction processor masks.""" + policy = redaction_policy(collector) + return tuple(re.compile(str(pattern)) for pattern in policy.get("blocked_values", []) or []) + + +def simulate_redaction( + attributes: dict[str, str], + allow: frozenset[str], + blocked: tuple[re.Pattern[str], ...] = (), + allow_all_keys: bool = False, +) -> dict[str, str]: + """Model the Collector redaction processor's fail-closed key and value handling. + + Any key outside the allow-list is dropped rather than kept, so an attribute + this skill has never seen cannot reach storage. Allowed values that still + match a blocked pattern are masked rather than dropped. + """ + result: dict[str, str] = {} + for key, value in attributes.items(): + if not allow_all_keys and key not in allow: + continue + text = str(value) + result[key] = MASK if any(pattern.search(text) for pattern in blocked) else text + return result + + +@pytest.fixture(scope="module") +def compose() -> dict[str, Any]: + """The committed local Compose document.""" + return load_yaml_file(COMPOSE_PATH) + + +@pytest.fixture(scope="module") +def collector() -> dict[str, Any]: + """The committed local Collector document.""" + return load_yaml_file(COLLECTOR_PATH) + + +class TestConfigurationLoading: + """The shipped documents parse and expose the structures later tests read.""" + + def test_compose_declares_services(self, compose: dict[str, Any]) -> None: + assert isinstance(compose.get("services"), dict) + assert compose["services"], "compose declares no services" + + def test_yaml_loader_rejects_a_non_mapping_document(self) -> None: + with pytest.raises(ConfigError): + load_yaml_text("- just\n- a list\n") + + def test_yaml_loader_rejects_malformed_yaml(self) -> None: + with pytest.raises(ConfigError): + load_yaml_text("services: [unclosed\n") + + +class TestRedactionSimulation: + """The policy simulator is fail-closed for keys and masking for values.""" + + def test_unknown_keys_are_dropped(self) -> None: + allow = frozenset({"service.name"}) + result = simulate_redaction({"service.name": "copilot", "prompt.text": "secret"}, allow) + assert result == {"service.name": "copilot"} + + def test_allowed_values_matching_a_blocked_pattern_are_masked(self) -> None: + allow = frozenset({"http.url"}) + blocked = (re.compile(r"[0-9a-f]{40}"),) + result = simulate_redaction({"http.url": "a" * 40}, allow, blocked) + assert result == {"http.url": MASK} + + def test_an_empty_allow_list_drops_everything(self) -> None: + assert simulate_redaction({"any.key": "value"}, frozenset()) == {} + + +class TestTopology: + """The Collector is the only OTLP ingress and LGTM is a private backend.""" + + def test_the_collector_publishes_otlp_on_loopback_only(self, compose: dict[str, Any]) -> None: + ports = published_ports(compose["services"]["otel-collector"]) + assert "127.0.0.1:4317:4317" in ports + assert "127.0.0.1:4318:4318" in ports + assert all(entry.startswith("127.0.0.1:") for entry in ports) + + def test_lgtm_publishes_no_otlp_port_to_the_host(self, compose: dict[str, Any]) -> None: + ports = published_ports(compose["services"]["lgtm"]) + assert not [entry for entry in ports if ":4317:" in entry or ":4318:" in entry] + + def test_every_lgtm_host_mapping_is_loopback(self, compose: dict[str, Any]) -> None: + assert all( + entry.startswith("127.0.0.1:") for entry in published_ports(compose["services"]["lgtm"]) + ) + + def test_both_services_share_one_declared_network(self, compose: dict[str, Any]) -> None: + declared = set(compose.get("networks") or {}) + assert declared, "compose declares no explicit network" + for name in ("otel-collector", "lgtm"): + assert declared.issuperset(compose["services"][name]["networks"]) + + def test_the_collector_exports_to_the_lgtm_service_only( + self, collector: dict[str, Any] + ) -> None: + exporters = collector["exporters"] + endpoints = { + config.get("endpoint") for config in exporters.values() if isinstance(config, dict) + } + assert endpoints == {"lgtm:4317"} + + +class TestRedactionPolicy: + """The declared policy is fail-closed and reaches every pipeline.""" + + def test_the_policy_is_fail_closed(self, collector: dict[str, Any]) -> None: + assert redaction_policy(collector).get("allow_all_keys") is False + + def test_every_pipeline_applies_redaction(self, collector: dict[str, Any]) -> None: + pipelines = collector["service"]["pipelines"] + assert set(pipelines) == {"traces", "metrics", "logs"} + for name, pipeline in pipelines.items(): + assert "redaction" in pipeline["processors"], f"{name} pipeline skips redaction" + + def test_the_allow_list_covers_what_the_dashboards_read( + self, collector: dict[str, Any] + ) -> None: + consumers = shipped_consumers() + allow = allowed_keys(collector) + missing = sorted(consumers.protected_attribute_keys() - allow) + assert missing == [], f"the dashboard reads attributes the allow-list drops: {missing}" + + def test_every_prometheus_label_the_dashboard_reads_has_a_source_key( + self, collector: dict[str, Any] + ) -> None: + """Compare in the OTLP-to-Prometheus direction, never the reverse. + + A label is derived from an attribute key by character substitution, + and the substitution is not reversible. `copilot_chat_edit_source` has + at least two plausible sources, so this asserts that some allow-listed + key normalizes to each derived label rather than reconstructing one. + """ + consumers = shipped_consumers() + covered = {prometheus_label_for(key) for key in allowed_keys(collector)} + uncovered = sorted(consumers.prometheus_labels - covered) + assert uncovered == [], ( + f"the dashboard groups by labels no allow-listed key produces: {uncovered}. " + "Add the source attribute or record the label as a known-empty gap; " + "do not remove it from the derivation." + ) + + def test_the_derivation_finds_the_metric_names_the_dashboard_selects(self) -> None: + """The derivation has to see metric names, not only attributes. + + Without this, a scrub rule could be licensed against metric names on + the grounds that no consumer was named for them. + """ + consumers = shipped_consumers() + assert len(consumers.metric_names) > 15 + assert "gen_ai_client_token_usage_sum" in consumers.metric_names + + def test_the_derivation_finds_the_span_name_matchers(self) -> None: + consumers = shipped_consumers() + assert consumers.span_name_matchers, "no TraceQL span-name matcher was derived" + assert all(matcher.startswith("invoke_agent") for matcher in consumers.span_name_matchers) + + def test_the_derivation_finds_the_baseline_trace_field(self) -> None: + assert "rootTraceName" in shipped_consumers().trace_fields + + def test_observed_content_attributes_are_not_allowed(self, collector: dict[str, Any]) -> None: + assert not (OBSERVED_CONTENT_ATTRIBUTES & allowed_keys(collector)) + + def test_a_novel_attribute_is_dropped_rather_than_stored( + self, collector: dict[str, Any] + ) -> None: + allow = allowed_keys(collector) + emitted = { + "service.name": "copilot-chat", + "gen_ai.request.model": "gpt-5", + # Neither key exists today. A delete-list would pass both through. + "gen_ai.future.prompt_echo": "the user's entire prompt", + "copilot_chat.unreleased_content": "a tool result", + } + kept = simulate_redaction(emitted, allow, blocked_values(collector)) + assert set(kept) == {"service.name", "gen_ai.request.model"} + + def test_known_content_attributes_are_dropped(self, collector: dict[str, Any]) -> None: + emitted = dict.fromkeys(OBSERVED_CONTENT_ATTRIBUTES, "plaintext content") + assert simulate_redaction(emitted, allowed_keys(collector), blocked_values(collector)) == {} + + def test_a_credential_in_an_allowed_value_is_masked(self, collector: dict[str, Any]) -> None: + kept = simulate_redaction( + {"gen_ai.request.model": "ghp_" + "a" * 36}, + allowed_keys(collector), + blocked_values(collector), + ) + assert kept == {"gen_ai.request.model": MASK} + + def test_the_processor_does_not_annotate_telemetry_with_removed_keys( + self, collector: dict[str, Any] + ) -> None: + assert redaction_policy(collector).get("summary") == "silent" + + +class TestGrafanaAccess: + """Grafana requires a credential the operator supplies.""" + + def test_anonymous_access_is_disabled_and_the_login_form_is_restored( + self, compose: dict[str, Any] + ) -> None: + environment = compose["services"]["lgtm"]["environment"] + assert str(environment["GF_AUTH_ANONYMOUS_ENABLED"]).lower() == "false" + assert str(environment["GF_AUTH_DISABLE_LOGIN_FORM"]).lower() == "false" + + def test_admin_credentials_are_required_environment_variables( + self, compose: dict[str, Any] + ) -> None: + environment = compose["services"]["lgtm"]["environment"] + for key, variable in ( + ("GF_SECURITY_ADMIN_USER", "COPILOT_OTEL_GRAFANA_USER"), + ("GF_SECURITY_ADMIN_PASSWORD", "COPILOT_OTEL_GRAFANA_PASSWORD"), + ): + value = str(environment[key]) + assert value.startswith(f"${{{variable}:?"), f"{key} is not a required variable" + + def test_no_password_literal_is_committed(self) -> None: + text = COMPOSE_PATH.read_text(encoding="utf-8") + assert "GF_SECURITY_ADMIN_PASSWORD: admin" not in text + assert "admin:admin" not in text + + def test_the_dashboard_helper_reads_the_same_credential_pair(self) -> None: + text = (EXAMPLES_DIR / "validate_dashboard.py").read_text(encoding="utf-8") + for variable in ("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD"): + assert variable in text, f"{variable} is not read by the helper" + # A default would make the helper usable against a credential the + # operator never chose, which is the failure this pair exists to stop. + assert f'"{variable}",' in text or f'"{variable}"\n' in text + assert f'os.environ.get("{variable}",' not in text, f"{variable} has a default" + assert "require_credentials(" in text, "credentials bypass the shared policy" + + +class TestImagePinning: + """Both images are pinned by immutable digest.""" + + @pytest.mark.parametrize("service", ["otel-collector", "lgtm"]) + def test_the_image_is_digest_pinned(self, compose: dict[str, Any], service: str) -> None: + assert DIGEST_REFERENCE.match(compose["services"][service]["image"]) + + +def scrub_statements(collector: dict[str, Any]) -> list[str]: + """Every OTTL statement the content-scrub processor declares.""" + processor = collector["processors"].get("transform/scrub", {}) + statements: list[str] = [] + for key in ("trace_statements", "log_statements", "metric_statements"): + for entry in processor.get(key) or []: + if isinstance(entry, str): + statements.append(entry) + else: + statements.extend(entry.get("statements", [])) + return statements + + +def statement_target(statement: str) -> str: + """The path an OTTL statement writes to. + + Only the write target matters. A `where` clause may read `span.name` + without endangering it, and treating a read as an overreach would push the + scrub rules into contortions to avoid mentioning what they must not touch. + """ + if "(" not in statement: + return "" + inner = statement.split("(", 1)[1] + depth = 0 + for index, character in enumerate(inner): + if character in "([": + depth += 1 + elif character in ")]": + if depth == 0: + return inner[:index].strip() + depth -= 1 + elif character == "," and depth == 0: + return inner[:index].strip() + return inner.strip() + + +# Intrinsics a shipped consumer reads directly. `span.name` is here because +# three TraceQL queries match on it; the metric intrinsics are here because +# every Prometheus query selects by metric name. +PROTECTED_INTRINSICS = frozenset( + {"span.name", "metric.name", "metric.description", "metric.unit"} +) + +ATTRIBUTE_TARGET = re.compile(r"attributes\[\"([^\"]+)\"\]") + + +def overreaching_statements( + statements: list[str], consumers: ShippedConsumers +) -> list[tuple[str, str]]: + """Statements whose write target is something a shipped consumer reads.""" + protected_keys = consumers.protected_attribute_keys() + findings: list[tuple[str, str]] = [] + for statement in statements: + target = statement_target(statement) + if target in PROTECTED_INTRINSICS: + findings.append((statement, target)) + continue + for key in ATTRIBUTE_TARGET.findall(target): + if key in protected_keys: + findings.append((statement, key)) + return findings + + +class TestScrubStaysInsideTheInventory: + """A scrub rule may not target anything a shipped consumer reads. + + The code review's own suggested fix for the ungoverned-carrier finding was + to normalize span names. That would have broken three dashboard queries and + the baseline helper's `rootTraceName` read. This check is oriented at the + rules that were changed, because asserting that unchanged dashboard files + still contain their strings cannot fail. + """ + + def test_the_scrub_processor_declares_statements(self, collector: dict[str, Any]) -> None: + assert scrub_statements(collector), "the content scrub declares no statements" + + def test_no_scrub_statement_targets_a_shipped_consumer( + self, collector: dict[str, Any] + ) -> None: + findings = overreaching_statements(scrub_statements(collector), shipped_consumers()) + assert findings == [], f"a scrub rule targets telemetry a shipped panel reads: {findings}" + + def test_the_check_catches_a_rule_that_normalizes_span_names(self) -> None: + """The negative case. Without it this check could be vacuous.""" + overreach = ['set(span.name, "[redacted]")'] + assert overreaching_statements(overreach, shipped_consumers()) == [ + ('set(span.name, "[redacted]")', "span.name") + ] + + def test_the_check_catches_a_rule_that_deletes_a_read_attribute(self) -> None: + overreach = ['delete_key(span.attributes["gen_ai.request.model"], "x")'] + findings = overreaching_statements(overreach, shipped_consumers()) + assert findings and findings[0][1] == "gen_ai.request.model" + + def test_a_condition_that_reads_a_protected_field_is_not_an_overreach(self) -> None: + """A `where` clause reads; it does not write.""" + reading = ['set(span.status.message, "[redacted]") where span.name != ""'] + assert overreaching_statements(reading, shipped_consumers()) == [] + + +class TestExporterAlias: + """The exporter uses the alias the pinned Collector asks for.""" + + def test_no_pipeline_references_the_deprecated_alias( + self, collector: dict[str, Any] + ) -> None: + exporters = set(collector["exporters"]) + assert not any(name == "otlp" or name.startswith("otlp/") for name in exporters), ( + "the pinned Collector logs a deprecation warning once per signal for " + "the `otlp` exporter alias; use `otlp_grpc`" + ) + for name, pipeline in collector["service"]["pipelines"].items(): + assert set(pipeline["exporters"]) <= exporters, f"{name} names an undeclared exporter" + + def test_the_internal_hop_is_unchanged_by_the_rename( + self, collector: dict[str, Any] + ) -> None: + exporter = collector["exporters"]["otlp_grpc/lgtm"] + assert exporter["endpoint"] == "lgtm:4317" + assert exporter["tls"]["insecure"] is True diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py new file mode 100644 index 000000000..6a2bbe070 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py @@ -0,0 +1,383 @@ +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Behavioral tests for the settings upsert executable. + +Every test runs against a temporary file. Nothing here reads or writes a real +VS Code settings file. +""" + +from __future__ import annotations + +import json +import pathlib + +import pytest +from settings_upsert import ( + SCHEMA, + SCHEMA_SOURCE, + SettingsError, + apply_changes, + backup_path_for, + check_policy, + main, + parse_assignment, + strip_jsonc, + summarize, + top_level_spans, + upsert, +) + +ENABLED = "github.copilot.chat.otel.enabled" +EXPORTER = "github.copilot.chat.otel.exporterType" +ENDPOINT = "github.copilot.chat.otel.otlpEndpoint" +CAPTURE = "github.copilot.chat.otel.captureContent" +OUTFILE = "github.copilot.chat.otel.outfile" +MAXSIZE = "github.copilot.chat.otel.maxAttributeSizeChars" + +EXISTING = """{ + // A comment the edit must not disturb. + "editor.fontSize": 13, + "github.copilot.chat.otel.enabled": false, + /* block comment */ + "workbench.colorTheme": "Default Dark+" +} +""" + + +@pytest.fixture() +def settings_file(tmp_path: pathlib.Path) -> pathlib.Path: + path = tmp_path / "settings.json" + path.write_text(EXISTING, encoding="utf-8") + return path + + +class TestSchemaProvenance: + """The schema records where it came from and matches the verified build.""" + + def test_provenance_names_a_version_and_artifact(self) -> None: + for field in ("artifact", "extension", "version", "retrieved_from", "verified"): + assert SCHEMA_SOURCE[field] + + def test_the_schema_is_the_verified_seven_keys(self) -> None: + assert len(SCHEMA) == 7 + assert all(key.startswith("github.copilot.chat.otel.") for key in SCHEMA) + + +class TestAssignmentParsing: + """Keys and types are checked before anything else happens.""" + + def test_an_unknown_key_is_refused(self) -> None: + with pytest.raises(SettingsError, match="unknown key"): + parse_assignment("github.copilot.chat.otel.protocol=grpc") + + def test_a_key_without_a_value_is_refused(self) -> None: + with pytest.raises(SettingsError, match="key=value"): + parse_assignment(ENABLED) + + def test_a_boolean_key_refuses_a_non_boolean(self) -> None: + with pytest.raises(SettingsError, match="boolean"): + parse_assignment(f"{ENABLED}=yes") + + def test_an_integer_key_refuses_a_non_integer(self) -> None: + with pytest.raises(SettingsError, match="integer"): + parse_assignment(f"{MAXSIZE}=lots") + + @pytest.mark.parametrize(("raw", "expected"), [("true", True), ("false", False)]) + def test_booleans_are_coerced(self, raw: str, expected: bool) -> None: + assert parse_assignment(f"{ENABLED}={raw}") == (ENABLED, expected) + + def test_integers_are_coerced(self) -> None: + assert parse_assignment(f"{MAXSIZE}=2048") == (MAXSIZE, 2048) + + +class TestPolicy: + """Value combinations the schema cannot express are refused.""" + + def test_enabling_content_capture_is_refused(self) -> None: + with pytest.raises(SettingsError, match="captureContent"): + check_policy({CAPTURE: True}) + + def test_disabling_content_capture_is_allowed(self) -> None: + check_policy({CAPTURE: False}) + + def test_an_unknown_exporter_type_is_refused(self) -> None: + with pytest.raises(SettingsError, match="exporterType"): + check_policy({EXPORTER: "otlp-quic"}) + + @pytest.mark.parametrize( + "endpoint", + [ + "http://evil.example.com:4318", + "file:///tmp/spans", + "http://user:pass@localhost:4318", + "http://localhost:22", + ], + ) + def test_an_unsafe_endpoint_is_refused(self, endpoint: str) -> None: + with pytest.raises(SettingsError, match="otlpEndpoint"): + check_policy({ENDPOINT: endpoint}) + + def test_the_local_collector_endpoint_is_allowed(self) -> None: + check_policy({ENDPOINT: "http://localhost:4318"}) + + def test_outfile_with_an_otlp_exporter_is_refused(self) -> None: + with pytest.raises(SettingsError, match="outfile"): + check_policy({OUTFILE: "/tmp/spans.jsonl", EXPORTER: "otlp-http"}) + + def test_outfile_with_the_file_exporter_is_allowed(self) -> None: + check_policy({OUTFILE: "/tmp/spans.jsonl", EXPORTER: "file"}) + + def test_a_negative_truncation_limit_is_refused(self) -> None: + with pytest.raises(SettingsError, match="negative"): + check_policy({MAXSIZE: -1}) + + +class TestJsoncHandling: + """Comments survive, offsets line up, and only the target value moves.""" + + def test_comments_are_blanked_without_shifting_offsets(self) -> None: + stripped = strip_jsonc(EXISTING) + assert len(stripped) == len(EXISTING) + assert "//" not in stripped + assert "block comment" not in stripped + assert json.loads(stripped)["editor.fontSize"] == 13 + + def test_a_comment_inside_a_string_is_preserved(self) -> None: + text = '{"a": "http://example.com/x"}' + assert json.loads(strip_jsonc(text))["a"] == "http://example.com/x" + + def test_top_level_spans_find_each_key(self) -> None: + spans = top_level_spans(EXISTING) + assert set(spans) >= {"editor.fontSize", ENABLED, "workbench.colorTheme"} + start, end = spans[ENABLED] + assert EXISTING[start:end] == "false" + + def test_upsert_replaces_only_the_target_value(self) -> None: + updated = upsert(EXISTING, {ENABLED: True}) + assert "// A comment the edit must not disturb." in updated + assert "/* block comment */" in updated + assert json.loads(strip_jsonc(updated))[ENABLED] is True + assert json.loads(strip_jsonc(updated))["editor.fontSize"] == 13 + + def test_upsert_appends_a_missing_key(self) -> None: + updated = upsert(EXISTING, {ENDPOINT: "http://localhost:4318"}) + assert json.loads(strip_jsonc(updated))[ENDPOINT] == "http://localhost:4318" + assert "// A comment the edit must not disturb." in updated + + def test_upsert_handles_an_empty_object(self) -> None: + updated = upsert("{}\n", {ENABLED: True}) + assert json.loads(strip_jsonc(updated)) == {ENABLED: True} + + def test_a_nested_key_of_the_same_name_is_not_targeted(self) -> None: + """A language-scoped override must not absorb the edit. + + The span scanner is hand-written, so this is the case most likely to + break silently under a later change: the write would land in a + `"[python]"` block and the user's real setting would stay unchanged. + """ + nested = ( + "{\n" + ' "[python]": { "' + ENABLED + '": false },\n' + ' "' + ENABLED + '": false\n' + "}\n" + ) + assert set(top_level_spans(nested)) == {"[python]", ENABLED} + + parsed = json.loads(strip_jsonc(upsert(nested, {ENABLED: True}))) + assert parsed[ENABLED] is True, "the top-level key was not updated" + assert parsed["[python]"][ENABLED] is False, "the nested override was overwritten" + + +class TestCommandLine: + """Argument handling refuses ambiguous input before touching a file.""" + + def test_a_duplicate_key_is_refused(self, settings_file: pathlib.Path) -> None: + code = main( + [ + "--settings", + str(settings_file), + "--set", + f"{ENABLED}=true", + "--set", + f"{ENABLED}=false", + "--apply", + ] + ) + assert code == 1 + assert settings_file.read_text(encoding="utf-8") == EXISTING + + def test_no_assignment_is_refused(self, settings_file: pathlib.Path) -> None: + assert main(["--settings", str(settings_file)]) == 1 + assert settings_file.read_text(encoding="utf-8") == EXISTING + + +class TestApply: + """Dry runs write nothing; applied edits are backed up and reversible.""" + + def test_a_dry_run_leaves_the_file_untouched(self, settings_file: pathlib.Path) -> None: + apply_changes(settings_file, {ENABLED: True}, apply=False) + assert settings_file.read_text(encoding="utf-8") == EXISTING + + def test_an_applied_edit_writes_and_backs_up(self, settings_file: pathlib.Path) -> None: + apply_changes(settings_file, {ENABLED: True}, apply=True) + assert json.loads(strip_jsonc(settings_file.read_text(encoding="utf-8")))[ENABLED] is True + backups = list(settings_file.parent.glob(f"{settings_file.name}.*.bak")) + assert len(backups) == 1 + assert backups[0].read_text(encoding="utf-8") == EXISTING + + def test_unrelated_settings_survive(self, settings_file: pathlib.Path) -> None: + apply_changes(settings_file, {ENABLED: True}, apply=True) + parsed = json.loads(strip_jsonc(settings_file.read_text(encoding="utf-8"))) + assert parsed["editor.fontSize"] == 13 + assert parsed["workbench.colorTheme"] == "Default Dark+" + + def test_a_refused_edit_writes_nothing(self, settings_file: pathlib.Path) -> None: + with pytest.raises(SettingsError): + apply_changes(settings_file, {CAPTURE: True}, apply=True) + assert settings_file.read_text(encoding="utf-8") == EXISTING + assert list(settings_file.parent.glob(f"{settings_file.name}.*.bak")) == [] + + def test_a_malformed_existing_file_is_refused(self, tmp_path: pathlib.Path) -> None: + broken = tmp_path / "settings.json" + broken.write_text('{"a": [1, 2', encoding="utf-8") + with pytest.raises(SettingsError, match="not valid JSONC"): + apply_changes(broken, {ENABLED: True}, apply=True) + assert broken.read_text(encoding="utf-8") == '{"a": [1, 2' + + def test_a_missing_file_starts_from_an_empty_object(self, tmp_path: pathlib.Path) -> None: + target = tmp_path / "settings.json" + apply_changes(target, {ENABLED: True}, apply=True) + assert json.loads(strip_jsonc(target.read_text(encoding="utf-8"))) == {ENABLED: True} + + +class TestAudit: + """Audit evidence is durable and carries no sensitive value verbatim.""" + + def test_an_audit_record_is_appended(self, settings_file: pathlib.Path) -> None: + audit = settings_file.parent / "audit.jsonl" + apply_changes(settings_file, {ENABLED: True}, apply=True, audit_path=audit) + record = json.loads(audit.read_text(encoding="utf-8").strip()) + assert record["changes"][ENABLED] == {"from": False, "to": True} + assert record["schema_version"] == SCHEMA_SOURCE["version"] + + def test_audit_records_accumulate(self, settings_file: pathlib.Path) -> None: + audit = settings_file.parent / "audit.jsonl" + apply_changes(settings_file, {ENABLED: True}, apply=True, audit_path=audit) + apply_changes(settings_file, {ENABLED: False}, apply=True, audit_path=audit) + assert len(audit.read_text(encoding="utf-8").strip().splitlines()) == 2 + + def test_a_sensitive_value_is_summarized_not_copied(self) -> None: + summarized = summarize(OUTFILE, "/home/someone/private/spans.jsonl") + assert "private" not in str(summarized) + assert "length" in str(summarized) + + def test_an_ordinary_value_is_recorded_directly(self) -> None: + assert summarize(ENABLED, True) is True + + def test_an_endpoint_keeps_its_authority_and_loses_its_path(self) -> None: + """A path or query on an OTLP endpoint is operator-supplied. + + It is where a tenant identifier or a token would sit, and it is not + needed to answer the question the record is kept for. + """ + summarized = summarize(ENDPOINT, "https://ingest.example.com:4318/v1/traces?token=abc123") + assert summarized == "https://ingest.example.com:4318" + assert "token" not in str(summarized) + assert "v1/traces" not in str(summarized) + + def test_a_loopback_endpoint_is_still_recognizable(self) -> None: + assert summarize(ENDPOINT, "http://localhost:4318/v1/traces") == "http://localhost:4318" + + def test_an_unparseable_endpoint_is_reduced_rather_than_copied(self) -> None: + summarized = summarize(ENDPOINT, "not-a-url-at-all/secret-path") + assert "secret-path" not in str(summarized) + + def test_an_absent_value_records_as_absent(self) -> None: + assert summarize(ENDPOINT, None) is None + + +class TestBackupRetention: + """A backup that overwrites the previous backup is not a backup. + + The original defect was a constant `.bak` name: the first run preserved the + operator's file, and the second run replaced that preserved copy with the + already-edited one, destroying the only record of the original. + """ + + def test_a_second_apply_does_not_overwrite_the_first_backup( + self, settings_file: pathlib.Path + ) -> None: + apply_changes(settings_file, {ENABLED: True}, apply=True) + after_first = settings_file.read_text(encoding="utf-8") + apply_changes(settings_file, {ENABLED: False}, apply=True) + + backups = sorted(settings_file.parent.glob(f"{settings_file.name}.*.bak")) + assert len(backups) == 2 + contents = {path.read_text(encoding="utf-8") for path in backups} + assert EXISTING in contents, "the operator's original file was not retained" + assert after_first in contents + + def test_a_same_second_collision_gets_its_own_name(self, tmp_path: pathlib.Path) -> None: + settings = tmp_path / "settings.json" + settings.write_text("{}\n", encoding="utf-8") + first = backup_path_for(settings) + first.write_text("taken", encoding="utf-8") + second = backup_path_for(settings) + assert second != first + assert not second.exists() + + def test_the_backup_sits_beside_the_settings_file(self, tmp_path: pathlib.Path) -> None: + settings = tmp_path / "settings.json" + assert backup_path_for(settings).parent == tmp_path + assert backup_path_for(settings).name.startswith("settings.json.") + assert backup_path_for(settings).name.endswith(".bak") + + +class TestTrailingCommas: + """A trailing comma is accepted; a comma inside a string is untouchable. + + The suggested fix for this finding was a regex that removed any comma + followed by a closing brace. That rewrites string literals too, and the + untouched-key comparison guarding this edit could not have caught it, + because both sides of that comparison are produced by the same pass. + """ + + def test_a_trailing_comma_in_an_object_parses(self) -> None: + assert json.loads(strip_jsonc('{"a": 1,}')) == {"a": 1} + + def test_a_trailing_comma_in_an_array_parses(self) -> None: + assert json.loads(strip_jsonc('{"a": [1, 2,],}')) == {"a": [1, 2]} + + def test_a_trailing_comma_followed_by_a_comment_parses(self) -> None: + assert json.loads(strip_jsonc('{"a": 1, // note\n}')) == {"a": 1} + + def test_a_comma_inside_a_string_literal_survives_byte_for_byte(self) -> None: + source = '{"note": "literal, }", "a": 1,}' + stripped = strip_jsonc(source) + assert len(stripped) == len(source), "offsets were not preserved" + assert '"literal, }"' in stripped, "a comma inside a string was rewritten" + assert json.loads(stripped) == {"note": "literal, }", "a": 1} + + def test_the_scan_preserves_every_offset(self) -> None: + source = '{\n "a": 1, // trailing\n}\n' + assert len(strip_jsonc(source)) == len(source) + + def test_an_edit_survives_a_document_with_a_trailing_comma( + self, tmp_path: pathlib.Path + ) -> None: + settings = tmp_path / "settings.json" + settings.write_text('{\n "editor.fontSize": 13,\n}\n', encoding="utf-8") + apply_changes(settings, {ENABLED: True}, apply=True) + parsed = json.loads(strip_jsonc(settings.read_text(encoding="utf-8"))) + assert parsed[ENABLED] is True + assert parsed["editor.fontSize"] == 13 + + def test_a_string_containing_a_brace_is_not_read_as_structure( + self, tmp_path: pathlib.Path + ) -> None: + settings = tmp_path / "settings.json" + settings.write_text('{\n "note": "literal, }",\n}\n', encoding="utf-8") + apply_changes(settings, {ENABLED: True}, apply=True) + text = settings.read_text(encoding="utf-8") + assert '"literal, }"' in text + assert json.loads(strip_jsonc(text))["note"] == "literal, }" diff --git a/.github/skills/experimental/copilot-otel-metrics/uv.lock b/.github/skills/experimental/copilot-otel-metrics/uv.lock new file mode 100644 index 000000000..892a68d94 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/uv.lock @@ -0,0 +1,179 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "atheris" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/88/fd6ad595dafa9c7ce56dbfcaff0c7244988dac3af86c771166c6516ccf6b/atheris-3.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ec5e11f21a4c197fe91f7aea2b2de88e623c73a21fc07b105ac6329a1588457b", size = 36875908, upload-time = "2026-06-17T00:04:01.104Z" }, + { url = "https://files.pythonhosted.org/packages/4e/18/e19718c384fd7d801d0da7485407daef9af6194b6d8c8818175bec5efec6/atheris-3.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f8a9f51ce8369026e8eb7b7174835e8c4c85a1a6db5d9add36c15100779d2a39", size = 36800563, upload-time = "2026-06-17T00:04:04.559Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ff/ae7a5bfe99033e510bea4ed09934e636d93777317a48147369bc0dc2b71f/atheris-3.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:315a0b5c819852b1ffe1ca72efc389c7724881f2c33e4aacb8c6bcec49bd5011", size = 36772569, upload-time = "2026-06-17T00:04:07.702Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "copilot-otel-metrics-skill" +version = "0.0.0" +source = { virtual = "." } + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pyyaml" }, + { name = "ruff" }, +] +fuzz = [ + { name = "atheris" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.0" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "ruff", specifier = ">=0.15" }, +] +fuzz = [{ name = "atheris", specifier = ">=3.0" }] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" }, + { url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" }, + { url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" }, + { url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" }, + { url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" }, + { url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" }, + { url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" }, + { url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" }, + { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, + { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, +] From bb678cd67b151eec4b8ef991e7966d017cd3a5f5 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 13:49:12 -0700 Subject: [PATCH 02/28] refactor(hooks)!: retire the telemetry hook and its owned runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest registered one command against eleven lifecycle events, so installing the package enrolled the developer in collection across every prompt, tool call, and subagent dispatch. Hooks are fail-closed, so a telemetry defect could deny every tool call in a session. BREAKING CHANGE: automatic hook-driven telemetry collection is removed. Capture is now opt-in through an editor setting handled by the copilot-otel-metrics skill. Local JSONL to static HTML reporting is replaced by Grafana dashboards and query helpers. Resolves #2496 Supersedes #2562 Refs #2717 🧹 - Generated by Copilot --- .github/hooks/shared/telemetry.json | 94 - .../telemetry/Invoke-TelemetryClean.ps1 | 129 -- .../telemetry/Invoke-TelemetryCollector.ps1 | 141 -- .../telemetry/Invoke-TelemetryReport.ps1 | 255 --- .../hooks/shared/telemetry/_telemetry_core.py | 1566 ----------------- .../hooks/shared/telemetry/clean-telemetry.sh | 107 -- .../telemetry/generate-telemetry-report.sh | 230 --- .github/hooks/shared/telemetry/pyproject.toml | 31 - .github/hooks/shared/telemetry/report.html | 887 ---------- .../shared/telemetry/telemetry-collector.sh | 95 - .../telemetry/tests/corpus/0_jsonl_event | 1 - .../telemetry/tests/corpus/1_event_alias | 1 - .../telemetry/tests/corpus/2_build_entry | 1 - .../shared/telemetry/tests/corpus/README.md | 34 - .../shared/telemetry/tests/fuzz_harness.py | 137 -- .../telemetry/tests/test_telemetry_core.py | 1412 --------------- .github/hooks/shared/telemetry/uv.lock | 297 ---- 17 files changed, 5418 deletions(-) delete mode 100644 .github/hooks/shared/telemetry.json delete mode 100644 .github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 delete mode 100644 .github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 delete mode 100644 .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 delete mode 100644 .github/hooks/shared/telemetry/_telemetry_core.py delete mode 100755 .github/hooks/shared/telemetry/clean-telemetry.sh delete mode 100755 .github/hooks/shared/telemetry/generate-telemetry-report.sh delete mode 100644 .github/hooks/shared/telemetry/pyproject.toml delete mode 100644 .github/hooks/shared/telemetry/report.html delete mode 100755 .github/hooks/shared/telemetry/telemetry-collector.sh delete mode 100644 .github/hooks/shared/telemetry/tests/corpus/0_jsonl_event delete mode 100644 .github/hooks/shared/telemetry/tests/corpus/1_event_alias delete mode 100644 .github/hooks/shared/telemetry/tests/corpus/2_build_entry delete mode 100644 .github/hooks/shared/telemetry/tests/corpus/README.md delete mode 100644 .github/hooks/shared/telemetry/tests/fuzz_harness.py delete mode 100644 .github/hooks/shared/telemetry/tests/test_telemetry_core.py delete mode 100644 .github/hooks/shared/telemetry/uv.lock diff --git a/.github/hooks/shared/telemetry.json b/.github/hooks/shared/telemetry.json deleted file mode 100644 index cad55ab3f..000000000 --- a/.github/hooks/shared/telemetry.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "version": 1, - "description": "Records Copilot session lifecycle events to local telemetry for reporting.", - "hooks": { - "sessionStart": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "userPromptSubmitted": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "userPromptSubmit": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "preToolUse": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "postToolUse": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "subagentStart": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "subagentStop": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "sessionEnd": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "stop": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "agentStop": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ], - "preCompact": [ - { - "type": "command", - "bash": "${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh", - "powershell": "& (Join-Path ([string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? '.github' : $env:CLAUDE_PLUGIN_ROOT) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')", - "timeoutSec": 10 - } - ] - } -} diff --git a/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 deleted file mode 100644 index 6fc8285f4..000000000 --- a/.github/hooks/shared/telemetry/Invoke-TelemetryClean.ps1 +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env pwsh -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -#Requires -Version 7.4 - -<# -.SYNOPSIS - Removes telemetry artifacts written by the Copilot telemetry hooks. -.DESCRIPTION - Delegates to the shared Python engine's clean mode. By default cleans this - project's telemetry store; -AllDirs extends the cleanup to every registered - project plus the user-level HVE home directory. This thin wrapper keeps the - cleanup logic in a single implementation (Python) shared with the bash entry - point clean-telemetry.sh. -.PARAMETER AllDirs - Also clean every per-project telemetry directory recorded in the user-level - registry, plus the generated launchers, report, and registry in the HVE home - directory. -.PARAMETER Path - Telemetry directory. Scopes the cleanup to this directory alone, overriding - -AllDirs. Default: /.copilot-tracking/telemetry -.PARAMETER DryRun - List what would be removed without deleting anything. -.PARAMETER Force - Skip the confirmation prompt (required for non-interactive use). -.NOTES - Runs via: pwsh Invoke-TelemetryClean.ps1 -#> -[CmdletBinding(SupportsShouldProcess, ConfirmImpact = 'High')] -param( - [switch]$AllDirs, - [string]$Path, - [switch]$DryRun, - [switch]$Force -) - -$ErrorActionPreference = 'Stop' - -#region Resolve repo root -# Match the collector and anchor on the caller's workspace: the script itself -# may live outside the repo it cleans, so its location says nothing. -# Only needed to build the default telemetry path; an explicit -Path makes the -# whole lookup unnecessary, including the git subprocess. -if (-not $Path) { - $RepoRoot = $env:HVE_REPO_ROOT - if (-not $RepoRoot -and (Get-Command git -ErrorAction SilentlyContinue)) { - try { $RepoRoot = & git rev-parse --show-toplevel 2>$null } catch { $RepoRoot = $null } - } - if (-not $RepoRoot) { - # ProviderPath, not .Path: the latter is PowerShell-qualified ('Work:\', - # 'HKLM:\SOFTWARE'). A non-filesystem provider survives this as a path with - # no drive qualifier, which the guard below rejects. - $RepoRoot = $PWD.ProviderPath - } -} -#endregion Resolve repo root - -# Require Python3 for the shared telemetry engine -$Python = Get-Command python3 -ErrorAction SilentlyContinue -if (-not $Python) { - $Python = Get-Command python -ErrorAction SilentlyContinue -} -if (-not $Python) { - Write-Error "'python3' is required but not installed" - exit 1 -} - -# Resolve the shared telemetry engine from the skill directory -$CorePy = Join-Path $PSScriptRoot '_telemetry_core.py' -if (-not (Test-Path $CorePy)) { - Write-Error "Telemetry engine not found: $CorePy" - exit 1 -} - -$TelemetryDir = if ($Path) { $Path } else { Join-Path $RepoRoot '.copilot-tracking' 'telemetry' } - -# Hand Python a native absolute path. -Path may be relative or on a PSDrive, and -# PowerShell does not sync [Environment]::CurrentDirectory with Set-Location, so -# the child process would resolve a relative path against a different directory -# than the one shown in the confirmation prompt below. Guarding the resolved path -# covers both branches: a non-filesystem location leaves a drive-less path -# ('HKEY_LOCAL_MACHINE\SOFTWARE\...') that Python would silently take as relative -# to its own working directory and delete from there. -$Provider = $null -$TelemetryDir = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath( - $TelemetryDir, [ref]$Provider, [ref]$null) -if ($Provider.Name -ne 'FileSystem') { - Write-Error "Telemetry path resolved to '$TelemetryDir' on the $($Provider.Name) provider from location '$($PWD.Path)'; re-run from a filesystem directory, pass an absolute -Path, or set HVE_REPO_ROOT." - exit 1 -} - -# An explicit -Path narrows the scope. The generated launcher always passes -# -AllDirs, so announce the override rather than silently widening or dropping -# a destructive scope flag. -$UseAllDirs = [bool]$AllDirs -and -not $Path -if ($AllDirs -and $Path) { - Write-Warning "Ignoring -AllDirs because -Path was given; scope is '$TelemetryDir'." -} - -# Prompt before destructive deletion. Skipped on -DryRun (non-destructive) and -# bypassed with -Force (required for non-interactive use). -if (-not $DryRun -and -not $Force) { - $scope = if ($UseAllDirs) { - 'ALL registered telemetry stores plus the user-level HVE home directory' - } else { - $TelemetryDir - } - if (-not $PSCmdlet.ShouldProcess($scope, 'Permanently remove telemetry artifacts')) { - Write-Host 'Aborted.' - exit 0 - } -} - -# Build the clean-mode argument list mirroring the bash wrapper's flags. -$CliArgs = @('clean') -if ($UseAllDirs) { $CliArgs += '--all-dirs' } -if ($DryRun) { $CliArgs += '--dry-run' } - -# Scope the variable to this call, as the bash wrapper's prefix assignment does: -# leaking it would silently redirect the collector and report scripts for the -# rest of the session. -$PreviousTelemetryDir = $env:HVE_TELEMETRY_DIR -$env:HVE_TELEMETRY_DIR = $TelemetryDir -try { - & $Python.Source $CorePy @CliArgs -} -finally { - $env:HVE_TELEMETRY_DIR = $PreviousTelemetryDir -} diff --git a/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 deleted file mode 100644 index 54c7a32e0..000000000 --- a/.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1 +++ /dev/null @@ -1,141 +0,0 @@ -ο»Ώ#!/usr/bin/env pwsh -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT - -<# -.SYNOPSIS - Copilot hook handler that delegates telemetry collection to the shared Python engine. -.DESCRIPTION - Reads JSON from stdin for each hook lifecycle event, checks the opt-in gate, - and delegates all processing to _telemetry_core.py. This thin wrapper keeps - the collection logic in a single implementation (Python) shared with the bash - hook entry point. -.NOTES - Runs via: Copilot agent hook (stdin JSON, stdout JSON) -#> -[CmdletBinding()] -param() - -$ErrorActionPreference = 'Stop' - -# The hook contract is that this script always answers with a continue signal: -# a silent hook stalls the agent turn. The answer is emitted from finally so -# every path below, including an early return, still produces it. -try { - #region Resolve repo root - $RepoRoot = $env:HVE_REPO_ROOT - if (-not $RepoRoot -and (Get-Command git -ErrorAction SilentlyContinue)) { - try { $RepoRoot = & git rev-parse --show-toplevel 2>$null } catch { $RepoRoot = $null } - } - if (-not $RepoRoot) { - $RepoRoot = '.' - } - #endregion Resolve repo root - - #region Opt-in gate - # Test-Path takes -LiteralPath throughout: these paths come from the - # environment and from the repository, so a bracket in a directory name must - # not be read as a wildcard and quietly report the path as missing. - $Enabled = $env:HVE_TELEMETRY -eq '1' - if (-not $Enabled) { - $MarkerPath = Join-Path $RepoRoot '.hve-telemetry' - $Enabled = Test-Path -LiteralPath $MarkerPath - } - if (-not $Enabled) { - return - } - #endregion Opt-in gate - - # Require Python3 for JSON processing - $Python = Get-Command python3 -ErrorAction SilentlyContinue - if (-not $Python) { - $Python = Get-Command python -ErrorAction SilentlyContinue - } - if (-not $Python) { - Write-Warning 'HVE telemetry enabled but no python3 or python found; events will not be recorded' - return - } - - # Resolve the shared telemetry engine from the skill directory - $CorePy = Join-Path $PSScriptRoot '_telemetry_core.py' - - if (-not (Test-Path -LiteralPath $CorePy)) { - Write-Warning "Telemetry engine not found at $CorePy; events will not be recorded" - return - } - - # Nested 2-arg Join-Path: the 3-arg form needs PS 6+, and this hook can be - # launched by a host that picks Windows PowerShell 5.1. - $TelemetryDir = if ($env:HVE_TELEMETRY_DIR) { $env:HVE_TELEMETRY_DIR } else { Join-Path (Join-Path $RepoRoot '.copilot-tracking') 'telemetry' } - $StackDir = Join-Path $TelemetryDir '.stacks' - try { - if (-not (Test-Path -LiteralPath $TelemetryDir)) { - New-Item -ItemType Directory -Path $TelemetryDir -Force | Out-Null - } - if (-not (Test-Path -LiteralPath $StackDir)) { - New-Item -ItemType Directory -Path $StackDir -Force | Out-Null - } - } - catch { - # An unwritable store is a telemetry failure, not a turn failure. The - # engine below skips its own writes for the same reason. - Write-Verbose "Telemetry directory unavailable: $_" - } - - # Delegate all JSON processing to the shared Python telemetry engine. - # The hook contract is raw process stdin, not a PowerShell object pipeline: - # [CmdletBinding()] makes this an advanced script, which rejects pipeline input - # it cannot bind to a parameter, leaving $input empty. - # Decode explicitly as UTF-8: [Console]::In would use the OEM console codepage - # under Windows PowerShell 5.1 and corrupt non-ASCII payload bytes. - $RawInput = '' - if ([Console]::IsInputRedirected) { - $Reader = [IO.StreamReader]::new([Console]::OpenStandardInput(), [Text.UTF8Encoding]::new($false)) - try { $RawInput = $Reader.ReadToEnd() } - finally { $Reader.Dispose() } - } - - # Dump raw input for diagnostics (first 5 events only). This records hook - # payloads verbatim, including the full prompt text and tool inputs such as - # file contents and shell command strings, which can contain secrets. The - # processed sessions-*.jsonl stream already provides the diagnostic signal, - # so the verbatim dump is a separate explicit opt-in (off by default) - # layered on top of the telemetry gate. See docs/customization/local-telemetry.md. - if ($env:HVE_TELEMETRY_RAW -eq '1') { - try { - $RawLog = Join-Path $TelemetryDir 'raw-input.jsonl' - $RawCount = 0 - if (Test-Path -LiteralPath $RawLog) { - # Array subexpression: Get-Content returns a bare string for a - # single-line file, whose .Count is 1 only by PS 3+ unified property. - $RawCount = @(Get-Content -LiteralPath $RawLog).Count - } - if ($RawCount -lt 5) { - # Not Add-Content -Encoding UTF8: that emits a BOM under Windows - # PowerShell 5.1, which makes the first JSONL line unparseable. - # Trim the inbound line ending and write LF so one event is one JSONL - # record, as on bash; CRLF here would leave a blank line between records. - [IO.File]::AppendAllText($RawLog, $RawInput.TrimEnd("`r", "`n") + "`n", [Text.UTF8Encoding]::new($false)) - } - } - catch { - Write-Verbose "Raw telemetry capture failed: $_" - } - } - - $env:HVE_REPO_ROOT = $RepoRoot - $env:HVE_TELEMETRY_DIR = $TelemetryDir - # Windows PowerShell 5.1 defaults $OutputEncoding to ASCII, which flattens - # every non-ASCII character to '?' when piping into a native process. - $OutputEncoding = [Text.UTF8Encoding]::new($false) - # Engine stderr is left unredirected so it reaches the host log, matching the - # bash wrapper. Whether a non-zero exit throws depends on the host version, - # and either way the finally below still returns continue. - $RawInput | & $Python.Source $CorePy collect -} -catch { - Write-Verbose "Telemetry collection error: $_" -} -finally { - '{"continue":true}' -} diff --git a/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 b/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 deleted file mode 100644 index 8774e11f8..000000000 --- a/.github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env pwsh -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -#Requires -Version 7.4 - -<# -.SYNOPSIS - Generates a self-contained telemetry report from Copilot hook JSONL files. -.DESCRIPTION - Native PowerShell counterpart to generate-telemetry-report.sh. Discovers the - session files for a target date, embeds their contents into a copy of - report.html, and writes a self-contained report.generated.html that renders - automatically without drag & drop. - - Orchestration (file discovery, JSON embedding, template injection) is native - PowerShell so Windows hosts need no bash. Token/model enrichment is delegated - to the shared Python engine (_telemetry_core.py) via its aggregate-debug, - aggregate-session, and list-dirs modes - the same modes the bash entry point - uses - so the enrichment logic stays single-sourced across platforms. Python - is optional: when absent, the report is produced without enrichment. -.PARAMETER Date - Target date (yyyy-MM-dd). Default: today (UTC). Use 'all' to include every - sessions-*.jsonl file. -.PARAMETER AllDirs - Scan every per-project telemetry directory recorded in the user-level - registry (~/.hve/telemetry-dirs) for a combined cross-project report. - Ignored when -Path is given. -.PARAMETER Path - Telemetry directory. Scopes the report to this directory alone, overriding - -AllDirs. Default: /.copilot-tracking/telemetry -.PARAMETER DebugLog - Optional debug log JSONL (e.g. main.jsonl) for token data. When omitted, VS - Code debug logs are auto-discovered and the precise model version plus token - data are joined in by session id. -.PARAMETER Output - Output path. Default: /report.generated.html -.PARAMETER Open - Open the generated report in the default browser. -.NOTES - Runs via: pwsh Invoke-TelemetryReport.ps1 -#> -[CmdletBinding()] -param( - [Alias('d')] - [string]$Date, - [Alias('a')] - [switch]$AllDirs, - [Alias('p')] - [string]$Path, - [Alias('l')] - [string]$DebugLog, - [Alias('o')] - [string]$Output, - [switch]$Open -) - -$ErrorActionPreference = 'Stop' - -$TemplatePath = Join-Path $PSScriptRoot 'report.html' -$CorePy = Join-Path $PSScriptRoot '_telemetry_core.py' - -#region Resolve repo root -# Match the collector and anchor on the caller's workspace: the script itself -# may live outside the repo it reports on, so its location says nothing. -# Only needed to build the default telemetry path; an explicit -Path makes the -# whole lookup unnecessary, including the git subprocess. -if (-not $Path) { - $RepoRoot = $env:HVE_REPO_ROOT - if (-not $RepoRoot -and (Get-Command git -ErrorAction SilentlyContinue)) { - try { $RepoRoot = & git rev-parse --show-toplevel 2>$null } catch { $RepoRoot = $null } - } - if (-not $RepoRoot) { - # ProviderPath, not .Path: the latter is PowerShell-qualified ('Work:\', - # 'HKLM:\SOFTWARE') and is not a path Python can open. - $RepoRoot = $PWD.ProviderPath - } -} -#endregion Resolve repo root - -# Python enables best-effort enrichment only; the report works without it. -$Python = Get-Command python3 -ErrorAction SilentlyContinue -if (-not $Python) { - $Python = Get-Command python -ErrorAction SilentlyContinue -} - -# Emit the user-level registry of per-project telemetry directories (one path -# per line). Used by -AllDirs for cross-project reports. Empty without Python. -function Get-RegistryDir { - if (-not $Python) { return @() } - # Windows PowerShell 5.1 decodes native stdout with the OEM codepage, which - # corrupts registry paths holding non-ASCII characters and silently drops - # those stores from cross-project reports. - $PrevEncoding = [Console]::OutputEncoding - try { - [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 - $lines = & $Python.Source $CorePy list-dirs 2>$null - if ($LASTEXITCODE -eq 0 -and $lines) { - return @($lines | Where-Object { $_ -and $_.Trim() }) - } - } catch { - Write-Verbose "Registry lookup failed; continuing without cross-project dirs: $_" - } finally { - [Console]::OutputEncoding = $PrevEncoding - } - return @() -} - -# Best-effort enrichment: extract llm_request events (precise model, token, and -# duration data) from VS Code debug logs, scoped to the supplied hook files. -# Returns $true when matching events were written to $OutFile. -function Invoke-AggregateDebug { - param([string]$OutFile, [string[]]$HookFile) - if (-not $Python) { return $false } - & $Python.Source $CorePy aggregate-debug $OutFile @HookFile 2>$null - return ($LASTEXITCODE -eq 0) -} - -# Enrichment from CLI session state: produces llm_request-compatible events so -# CLI sessions without VS Code debug logs still show model/token/duration data. -function Invoke-AggregateSession { - param([string]$OutFile, [string[]]$HookFile) - if (-not $Python) { return $false } - & $Python.Source $CorePy aggregate-session $OutFile @HookFile 2>$null - return ($LASTEXITCODE -eq 0) -} - -if (-not (Test-Path -LiteralPath $TemplatePath)) { - Write-Error "Template not found: $TemplatePath" - exit 1 -} - -$TargetDate = if ($Date) { $Date } else { (Get-Date).ToUniversalTime().ToString('yyyy-MM-dd') } -$TelemetryPath = if ($Path) { $Path } else { Join-Path $RepoRoot '.copilot-tracking' 'telemetry' } -# PowerShell does not sync [Environment]::CurrentDirectory with Set-Location, so -# a relative or PSDrive path would resolve elsewhere in the Python child process. -$TelemetryPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($TelemetryPath) -$OutputPath = if ($Output) { $Output } else { Join-Path $TelemetryPath 'report.generated.html' } - -# Determine which telemetry directories to scan. With -AllDirs, prepend every -# directory recorded in the user-level registry (cross-project view). An -# explicit -Path wins so the generated cross-project launcher, which always -# passes -AllDirs, can still be scoped to one store. -if ($AllDirs -and $Path) { - Write-Warning "Ignoring -AllDirs because -Path was given; scanning '$TelemetryPath' only." -} -$SearchDirs = [System.Collections.Generic.List[string]]::new() -if ($AllDirs -and -not $Path) { - foreach ($d in (Get-RegistryDir)) { $SearchDirs.Add($d) } -} -$SearchDirs.Add($TelemetryPath) - -# Collect session files for the target date across the chosen directories, -# de-duplicating directories that appear more than once. The trailing '*' also -# picks up any shard a collector wrote when it could not lock the shared log; -# report.html orders every record it loads by ts. -$Pattern = if ($TargetDate -eq 'all') { 'sessions-*.jsonl' } else { "sessions-$TargetDate*.jsonl" } -$Files = [System.Collections.Generic.List[string]]::new() -$SeenDirs = [System.Collections.Generic.HashSet[string]]::new() - -# The temp directory is created inside the try whose finally removes it, so a -# failure between the two can never leave it behind. -$TmpDir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid().ToString()) -try { - New-Item -ItemType Directory -Path $TmpDir -Force | Out-Null - foreach ($dir in $SearchDirs) { - if (-not $dir) { continue } - # Registry entries and the default path can spell one directory differently - # (separator style on Windows), so key de-duplication on the canonical form. - $DirKey = $dir - try { $DirKey = [System.IO.Path]::GetFullPath($dir) } catch { Write-Verbose "Unnormalizable telemetry dir '$dir': $_" } - if (-not $SeenDirs.Add($DirKey)) { continue } - if (-not (Test-Path -LiteralPath $dir -PathType Container)) { continue } - Get-ChildItem -LiteralPath $dir -Filter $Pattern -File -ErrorAction SilentlyContinue | - Sort-Object Name | - ForEach-Object { $Files.Add($_.FullName) } - } - - if ($DebugLog) { - if (-not (Test-Path -LiteralPath $DebugLog)) { - Write-Error "Debug log not found: $DebugLog" - exit 1 - } - $Files.Add((Resolve-Path -LiteralPath $DebugLog).Path) - } elseif ($Files.Count -gt 0) { - # Auto-enrich with precise model + token data from VS Code debug logs and - # CLI session state, scoped to the sessions already collected. - $DebugAgg = Join-Path $TmpDir 'debug-llm-requests.jsonl' - if (Invoke-AggregateDebug -OutFile $DebugAgg -HookFile $Files.ToArray()) { - $Files.Add($DebugAgg) - } - - $CliAgg = Join-Path $TmpDir 'cli-session-state.jsonl' - if (Invoke-AggregateSession -OutFile $CliAgg -HookFile $Files.ToArray()) { - $Files.Add($CliAgg) - } - } - - if ($Files.Count -eq 0) { - Write-Warning "No telemetry files found in '$TelemetryPath' for date '$TargetDate'." - exit 0 - } - - # Build a JSON array of {name, content} objects. Session content quoting - # markup can steer the HTML tokenizer out of the host ' - $Lines = (Get-Content -LiteralPath $TemplatePath -Raw) -split "`n" - $Builder = [System.Text.StringBuilder]::new() - $Injected = $false - for ($i = 0; $i -lt $Lines.Count; $i++) { - if (-not $Injected -and $Lines[$i].Contains('id="embeddedData"')) { - [void]$Builder.Append($TagOpen).Append($Data).Append($TagClose) - $Injected = $true - } else { - [void]$Builder.Append($Lines[$i]) - } - if ($i -lt $Lines.Count - 1) { [void]$Builder.Append("`n") } - } - - $OutDir = Split-Path -Parent $OutputPath - if ($OutDir -and -not (Test-Path -LiteralPath $OutDir)) { - New-Item -ItemType Directory -Path $OutDir -Force | Out-Null - } - [System.IO.File]::WriteAllText($OutputPath, $Builder.ToString()) - - Write-Host "Wrote self-contained report: $OutputPath" - $Names = ($Files | ForEach-Object { Split-Path -Leaf $_ }) -join ', ' - Write-Host ("Embedded {0} file(s): {1}" -f $Files.Count, $Names) - - if ($Open) { - if ($IsWindows) { - Start-Process $OutputPath - } elseif ($IsMacOS -and (Get-Command open -ErrorAction SilentlyContinue)) { - & open $OutputPath - } elseif (Get-Command xdg-open -ErrorAction SilentlyContinue) { - & xdg-open $OutputPath - } - } -} finally { - if ($TmpDir -and (Test-Path -LiteralPath $TmpDir)) { - Remove-Item -LiteralPath $TmpDir -Recurse -Force -ErrorAction SilentlyContinue - } -} diff --git a/.github/hooks/shared/telemetry/_telemetry_core.py b/.github/hooks/shared/telemetry/_telemetry_core.py deleted file mode 100644 index dfc99c849..000000000 --- a/.github/hooks/shared/telemetry/_telemetry_core.py +++ /dev/null @@ -1,1566 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -"""Canonical telemetry engine shared by the Copilot hook collectors. - -This module is the single source of truth for telemetry collection. The bash -collector invokes the ``collect`` mode to record one hook event (and enrich -the session at ``Stop``, ``SessionEnd``, and ``PreCompact``), while the report -generators invoke the -``aggregate-debug``, ``aggregate-session``, and ``list-dirs`` modes to join -model/token data and discover per-project telemetry stores for reports. -Clean scripts invoke the ``clean`` mode to remove telemetry artifacts -from one or every registered store. - -The PowerShell collector ``Invoke-TelemetryCollector.ps1`` is a thin wrapper -that delegates to this same engine via ``collect``, so the collection logic -stays single-sourced across platforms. -""" - -from __future__ import annotations - -import datetime -import glob -import hashlib -import json -import os -import secrets -import shlex -import shutil -import sys -import time -from collections.abc import Iterable, Iterator -from pathlib import Path, PureWindowsPath -from typing import Any - -if os.name == "nt": - import msvcrt - - -def _detect_client() -> str: - """Infer the Copilot surface that invoked this hook.""" - if os.environ.get("GITHUB_COPILOT_API_TOKEN"): - return "cloud-agent" - if os.environ.get("VSCODE_PID") or os.environ.get("VSCODE_IPC_HOOK_CLI"): - return "vscode" - return "cli" - - -EVENT_ALIASES = { - "sessionStart": "SessionStart", - "userPromptSubmitted": "UserPromptSubmit", - "preToolUse": "PreToolUse", - "postToolUse": "PostToolUse", - "subagentStart": "SubagentStart", - "subagentStop": "SubagentStop", - "agentStop": "Stop", - "sessionEnd": "SessionEnd", - "preCompact": "PreCompact", -} - -# Documented sessionEnd ``reason`` values (GitHub Copilot hooks reference). -# Shape inference matches only these so an unrelated ``reason`` key on some -# other or future event is not mistaken for a session end. -SESSION_END_REASONS = frozenset({"complete", "error", "abort", "timeout", "user_exit"}) - -# Canonical field -> the payload keys that carry it, in priority order. Three -# documented payload formats reach this collector: VS Code (snake_case), the -# Copilot CLI camelCase format, and the CLI "VS Code compatible" format it sends -# when events are configured in PascalCase. They disagree on both casing and -# naming (tool output is ``tool_response`` in VS Code, ``tool_result`` in the -# CLI compatible format, ``toolResult`` in CLI camelCase). This table is the -# only place surface-specific key names appear, so shape inference and entry -# building cannot drift apart when a surface adds or renames a key. Add keys -# only when a surface documents them. -PAYLOAD_ALIASES: dict[str, tuple[str, ...]] = { - "event_name": ("hook_event_name", "hookEventName", "event"), - "session_id": ("session_id", "sessionId"), - "cwd": ("cwd",), - "timestamp": ("timestamp",), - "tool_name": ("tool_name", "toolName"), - "tool_input": ("tool_input", "toolArgs"), - "tool_use_id": ("tool_use_id", "toolUseId"), - "tool_result": ("tool_response", "tool_result", "toolResult"), - "tool_result_text": ("text_result_for_llm", "textResultForLlm"), - # VS Code names the subagent only as ``agent_type``; the CLI sends a name. - "agent_name": ("agent_name", "agentName", "agent_type", "agentType"), - # Unique per subagent invocation on VS Code; absent on surfaces that send - # only a name, which then has to serve as the identity. - "agent_id": ("agent_id", "agentId"), - "agent_display_name": ("agent_display_name", "agentDisplayName"), - "prompt": ("prompt",), - "source": ("source",), - "trigger": ("trigger",), - "stop_reason": ("stop_reason", "stopReason"), - "reason": ("reason",), -} - - -def _payload_get(data: dict, field: str, default: Any = "") -> Any: - """Return the first non-empty alias value for a canonical payload field.""" - for key in PAYLOAD_ALIASES[field]: - value = data.get(key) - if value: - return value - return default - - -def _payload_has(data: dict, field: str) -> bool: - """Return True when a payload carries any alias of a canonical field.""" - return any(key in data for key in PAYLOAD_ALIASES[field]) - - -def iter_jsonl(path: str | os.PathLike[str]) -> Iterator[dict]: - """Yield each well-formed JSON object from a JSONL file, skipping junk.""" - try: - with open(path, encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except ValueError: - continue - if isinstance(obj, dict): - yield obj - except OSError: - # File cannot be opened or read (e.g., does not exist or permission denied) - return - - -def _fallback_stem() -> str: - """Return a filename stem no other live collector process will produce. - - The UTC time orders fallback files beside their day log, the pid separates - concurrent writers, and the random suffix keeps a recycled pid from - reopening a departed process's file. The suffix carries the uniqueness on - Windows, whose clock resolution is coarse enough to collapse the timestamp - for writers started in the same tick. - """ - stamp = datetime.datetime.now(datetime.UTC).strftime("%H%M%S%f") - return f"{stamp}-{os.getpid()}-{secrets.token_hex(8)}" - - -# O_BINARY keeps Windows from expanding "\n" and desynchronizing the byte count. -_APPEND_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_BINARY", 0) - -# Telemetry records carry prompt previews and working-directory paths, and the -# registry carries every store location, so nothing here is readable by group or -# other. umask can only clear further bits, never restore them. -_OWNER_ONLY_FILE = 0o600 -_OWNER_ONLY_EXEC = 0o700 - - -def _write_all(fd: int, payload: bytes) -> None: - """Write every byte; a signal or a full disk can cut one write short.""" - view = memoryview(payload) - while view: - written = os.write(fd, view) - if not written: - # Retrying a write that consumed nothing spins forever inside a - # synchronous hook, so surface it to the caller's fallback instead. - raise OSError("write made no progress") - view = view[written:] - - -# Copilot runs hooks concurrently: parallel tool calls and subagents each spawn -# their own collector process, and they share one log per day. POSIX resolves -# O_APPEND under the inode lock, so nothing further is needed there. The Windows -# CRT emulates O_APPEND as seek-to-end plus write in user space, so racing -# collectors resolve the same offset and the second overwrites the first; a -# byte-range lock closes that window. A writer that cannot take the lock falls -# back to a private shard, which readers merge alongside the log. -if os.name == "nt": - # A record is a few hundred bytes, so a collision clears in microseconds. - # LK_LOCK would sleep a full second per retry for up to ten seconds, which - # is charged to the agent turn; poll instead, at the timescale of the write. - _LOCK_ATTEMPTS = 2000 - _LOCK_BACKOFF_SECONDS = 0.001 - - def _lock_append(fd: int) -> bool: - """Take the byte-0 mutex and seek to end. False if the lock is refused. - - Byte 0 is never read as data; it is only somewhere to anchor the lock. - Windows drops the lock when the handle closes, including on a crash, so - a dead collector cannot wedge the log. - """ - os.lseek(fd, 0, os.SEEK_SET) - for _ in range(_LOCK_ATTEMPTS): - try: - msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) - except OSError: - time.sleep(_LOCK_BACKOFF_SECONDS) - continue - os.lseek(fd, 0, os.SEEK_END) - return True - return False - - def _unlock_append(fd: int) -> None: - os.lseek(fd, 0, os.SEEK_SET) - try: - msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) - except OSError: - # The lock is released by the close in _append_shared's finally, and - # by the OS if this process dies, so a failed explicit unlock leaves - # nothing to repair and must not mask the record that was written. - pass - -else: - - def _lock_append(fd: int) -> bool: - """POSIX appends atomically under O_APPEND; there is no lock to take.""" - return True - - def _unlock_append(fd: int) -> None: - return - - -def append_line(target: Path, line: str) -> None: - """Append one newline-terminated line to the shared log for its day. - - Best-effort: a store that cannot be written is a telemetry failure, not a - turn failure, so every path returns rather than raising into the hook. - - Args: - target: The shared log path. Its parent is created when missing, and a - sibling shard is used when the log cannot be locked or written. - line: The record text, already newline-terminated by the caller. - """ - payload = line.encode("utf-8") - try: - target.parent.mkdir(parents=True, exist_ok=True) - if _append_shared(target, payload): - return - except OSError: - # A refused lock returns False; an unwritable or torn log raises. Both - # reach the shard below, which is the only way the record still lands. - pass - # A filesystem that refuses locks would let writers interleave, so this - # process takes a file of its own. Readers match the shard alongside the - # log, so the record still lands. - try: - _append_private( - target.with_name(f"{target.stem}.{_fallback_stem()}{target.suffix}"), payload - ) - except OSError: - # Nothing in this store is writable; drop the record. - return - - -def _append_shared(target: Path, payload: bytes) -> bool: - """Append under the platform's append guarantee. False if unavailable.""" - fd = os.open(target, _APPEND_FLAGS, _OWNER_ONLY_FILE) - try: - if not _lock_append(fd): - return False - try: - _write_all(fd, payload) - finally: - _unlock_append(fd) - finally: - os.close(fd) - return True - - -def _append_private(target: Path, payload: bytes) -> None: - """Append to a file only this process writes, so no lock is required.""" - fd = os.open(target, _APPEND_FLAGS, _OWNER_ONLY_FILE) - try: - _write_all(fd, payload) - finally: - os.close(fd) - - -def append_jsonl(target: Path, entry: dict) -> None: - """Append one JSON object as a JSONL record. - - Args: - target: The shared log path, as for :func:`append_line`. - entry: The record, serialized on one line. - """ - append_line(target, json.dumps(entry) + "\n") - - -def _write_text_atomic(path: Path, text: str, mode: int = _OWNER_ONLY_FILE) -> None: - """Replace ``path`` in one step so a reader never observes a partial file. - - Plain truncate-then-write leaves a window in which the file is empty or - half written, and every collector racing on ``first_write`` rewrites these - files at once. The temp name carries the pid so concurrent writers do not - collide on the staging file itself. - - Args: - path: The destination, replaced by an atomic rename. - text: The full file contents, written as UTF-8. - mode: Permission bits the staging file is created with, so the content - is never briefly readable more widely than the destination. - """ - tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") - try: - fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode) - try: - _write_all(fd, text.encode("utf-8")) - finally: - os.close(fd) - os.replace(tmp, path) - except OSError: - tmp.unlink(missing_ok=True) - raise - - -def _is_safe_sid(sid: str) -> bool: - """Return True when a session id is safe to embed in a filesystem path. - - Rejects empty ids and any value containing a path separator or ``..`` - traversal sequence so callers never build paths outside their store. A bare - drive or UNC specifier such as ``C:`` holds no separator yet still re-anchors - the path when joined, discarding the store prefix, so reject anchors too. - """ - if not sid or ".." in sid: - return False - if os.sep in sid or "/" in sid or "\\" in sid: - return False - return not PureWindowsPath(sid).anchor - - -def _is_contained(child: Path, parent: Path) -> bool: - """Return True when ``child`` resolves inside ``parent``. - - Checked immediately before a destructive operation so a path that escaped - its store cannot be removed, whatever produced it. - """ - try: - return child.resolve().is_relative_to(parent.resolve()) - except OSError: - return False - - -def collect_sids(hook_files: Iterable[str]) -> set[str]: - """Collect every safe session id referenced across the given hook files.""" - sids: set[str] = set() - for hook_file in hook_files: - for obj in iter_jsonl(hook_file): - sid = obj.get("sid") - if sid and _is_safe_sid(sid): - sids.add(sid) - return sids - - -def copilot_home() -> Path: - """Return the Copilot home directory, honoring ``COPILOT_HOME``.""" - override = os.environ.get("COPILOT_HOME") - return Path(override) if override else Path.home() / ".copilot" - - -def hve_home() -> Path: - """Return the HVE user-level directory, honoring ``HVE_HOME``.""" - override = os.environ.get("HVE_HOME") - return Path(override) if override else Path.home() / ".hve" - - -def telemetry_registry() -> Path: - """Return the user-level registry of per-project telemetry dirs. - - A directory holding one marker per store rather than a shared list file: - every writer targets a distinct path and rewriting it is a no-op, so - collectors racing on the first write of the day need no coordination. - """ - return hve_home() / "telemetry-dirs" - - -def _registry_marker(registry: Path, resolved: str) -> Path: - """Return the marker path for one store, named by a digest of its path.""" - return registry / f"{hashlib.sha256(resolved.encode('utf-8')).hexdigest()[:32]}.path" - - -# Registry file used before the marker directory: one absolute path per line. -_LEGACY_REGISTRY_NAME = "telemetry-dirs.txt" - - -def _migrate_legacy_registry(registry: Path) -> None: - """Import the pre-directory registry file once, then retire it. - - Without this an upgraded install silently drops every store it had already - registered. Active projects re-register on their next event, but a retired - one never does, and that is exactly the set ``clean --all-dirs`` exists to - reclaim. Renaming rather than deleting keeps the original recoverable. - """ - legacy = registry.parent / _LEGACY_REGISTRY_NAME - try: - if not legacy.is_file(): - return - lines = legacy.read_text(encoding="utf-8").splitlines() - except OSError: - return - for line in lines: - entry = line.strip() - if entry: - register_telemetry_dir(Path(entry), registry) - try: - legacy.rename(legacy.with_name(_LEGACY_REGISTRY_NAME + ".migrated")) - except OSError: - # Left in place; re-importing is idempotent, so the next call retries. - return - - -def read_registry_dirs(registry: Path | None = None) -> list[str]: - """Return registered telemetry directories, sorted and de-duplicated.""" - registry = registry if registry is not None else telemetry_registry() - _migrate_legacy_registry(registry) - try: - markers = sorted(registry.glob("*.path")) - except OSError: - # Registry dir is missing or unreadable; treat as no registered dirs. - return [] - dirs: set[str] = set() - for marker in markers: - try: - entry = marker.read_text(encoding="utf-8").strip() - except OSError: - continue - if entry: - dirs.add(entry) - return sorted(dirs) - - -def register_telemetry_dir(tel_dir: Path, registry: Path | None = None) -> None: - """Record an absolute telemetry directory in the user-level registry. - - Lets the report tooling discover every per-project telemetry store without - relying on environment propagation. Resolves to an absolute path and writes - the marker whole, so repeated and concurrent registrations converge on the - same content instead of appending duplicates. - """ - registry = registry if registry is not None else telemetry_registry() - try: - resolved = str(tel_dir.resolve()) - except OSError: - resolved = os.path.abspath(str(tel_dir)) - marker = _registry_marker(registry, resolved) - if marker.exists(): - return - try: - registry.mkdir(parents=True, exist_ok=True) - _write_text_atomic(marker, resolved + "\n") - except OSError: - # Cannot create the registry entry; skip recording this dir. - return - - -_BASH_LAUNCHER = """#!/usr/bin/env bash -# Generated by HVE telemetry. Regenerated each session; edits will be lost. -# Cross-project telemetry report launcher. Lives in the HVE home directory -# alongside the registry of per-project telemetry stores, so it does not need -# the (version-pinned) extension install path. Run from this directory: -# ./generate-report.sh # today, every project -# ./generate-report.sh --date all # every captured day, every project -REPORT_SCRIPT=__REPORT_SCRIPT__ -if [[ ! -f "$REPORT_SCRIPT" ]]; then - echo "Telemetry report script not found: $REPORT_SCRIPT" >&2 - echo "Start a new Copilot session to regenerate this launcher." >&2 - exit 1 -fi -exec bash "$REPORT_SCRIPT" --all-dirs --output __OUT__ "$@" -""" - -_PWSH_LAUNCHER = """# Generated by HVE telemetry. Regenerated each session; edits will be lost. -# Cross-project telemetry report launcher. Lives in the HVE home directory -# alongside the registry of per-project telemetry stores. Runs natively through -# PowerShell. Run from this directory: -# ./generate-report.ps1 # today, every project -# ./generate-report.ps1 -Date all # every captured day, every project -$ReportScript = '__REPORT_SCRIPT__' -if (-not (Test-Path $ReportScript)) { - Write-Error "Telemetry report script not found: $ReportScript" - Write-Error 'Start a new Copilot session to regenerate this launcher.' - exit 1 -} -& $ReportScript -AllDirs -Output '__OUT__' @args -""" - - -_BASH_CLEAN_LAUNCHER = """#!/usr/bin/env bash -# Generated by HVE telemetry. Regenerated each session; edits will be lost. -# Cross-project telemetry cleanup launcher. Removes telemetry artifacts from -# every registered per-project store and this HVE home directory. Run from -# this directory: -# ./clean-telemetry.sh # remove all telemetry artifacts -# ./clean-telemetry.sh --dry-run # list what would be removed -CLEAN_SCRIPT=__CLEAN_SCRIPT__ -if [[ ! -f "$CLEAN_SCRIPT" ]]; then - echo "Telemetry clean script not found: $CLEAN_SCRIPT" >&2 - echo "Start a new Copilot session to regenerate this launcher." >&2 - exit 1 -fi -exec bash "$CLEAN_SCRIPT" --all-dirs "$@" -""" - -_PWSH_CLEAN_LAUNCHER = """\ -# Generated by HVE telemetry. Regenerated each session; edits will be lost. -# Cross-project telemetry cleanup launcher. Removes telemetry artifacts from -# every registered per-project store and this HVE home directory. Runs natively -# through PowerShell. Run from this directory: -# ./clean-telemetry.ps1 # remove all telemetry artifacts -# ./clean-telemetry.ps1 -DryRun # list what would be removed -$CleanScript = '__CLEAN_PS1__' -if (-not (Test-Path $CleanScript)) { - Write-Error "Telemetry clean script not found: $CleanScript" - Write-Error 'Start a new Copilot session to regenerate this launcher.' - exit 1 -} -& $CleanScript -AllDirs @args -""" - - -def _is_windows() -> bool: - """Return True when running on Windows. - - Factored out so launcher generation can pick the native interpreter and so - tests can exercise both platform branches. - """ - return os.name == "nt" - - -def write_report_launchers(script_dir: Path | None = None) -> None: - """Emit cross-project report and cleanup launchers into the HVE home dir. - - Extension users lack the repository's launcher scripts and cannot easily - locate the version-pinned extension install path. These launchers live next - to the registry in the HVE home directory, are refreshed each session, and - delegate to the canonical report generator and cleanup wrappers in - cross-project mode so running them spans every project. - - Only the launchers for the host platform are written: PowerShell (``.ps1``) - on Windows, POSIX shell (``.sh``) elsewhere. Both the report and cleanup - launchers are fully native per platform (bash wrappers on POSIX, PowerShell - wrappers on Windows), so no cross-interpreter dependency is required. - """ - if script_dir is None: - script_dir = Path(__file__).resolve().parent - report_script = str(script_dir / "generate-telemetry-report.sh") - report_ps1 = str(script_dir / "Invoke-TelemetryReport.ps1") - clean_script = str(script_dir / "clean-telemetry.sh") - clean_ps1 = str(script_dir / "Invoke-TelemetryClean.ps1") - home = hve_home() - out_path = str(home / "report.generated.html") - try: - home.mkdir(parents=True, exist_ok=True) - if _is_windows(): - # Quote for PowerShell single-quoted strings (double embedded '). - pwsh_text = _PWSH_LAUNCHER.replace( - "__REPORT_SCRIPT__", report_ps1.replace("'", "''") - ).replace("__OUT__", out_path.replace("'", "''")) - pwsh_clean_text = _PWSH_CLEAN_LAUNCHER.replace( - "__CLEAN_PS1__", clean_ps1.replace("'", "''") - ) - _write_text_atomic(home / "generate-report.ps1", pwsh_text) - _write_text_atomic(home / "clean-telemetry.ps1", pwsh_clean_text) - else: - # Shell-quote so unusual install paths (spaces, quotes, ``$``) - # cannot break or inject into the generated launchers. - bash_text = _BASH_LAUNCHER.replace( - "__REPORT_SCRIPT__", shlex.quote(report_script) - ).replace("__OUT__", shlex.quote(out_path)) - bash_clean_text = _BASH_CLEAN_LAUNCHER.replace( - "__CLEAN_SCRIPT__", shlex.quote(clean_script) - ) - _write_text_atomic(home / "generate-report.sh", bash_text, mode=_OWNER_ONLY_EXEC) - _write_text_atomic(home / "clean-telemetry.sh", bash_clean_text, mode=_OWNER_ONLY_EXEC) - except OSError: - # Cannot write launchers (e.g., permission denied); skip generation. - return - - -def find_process_log(state_dir: Path, home: Path) -> str | None: - """Locate the CLI process log via the session lock file PID.""" - pid = None - try: - for lock_file in os.listdir(state_dir): - if lock_file.startswith("inuse.") and lock_file.endswith(".lock"): - pid = lock_file.split(".")[1] - break - except OSError: - # State dir cannot be listed (e.g., does not exist); no log to find. - return None - if not pid: - return None - candidates = glob.glob(str(home / "logs" / f"process-*-{pid}.log")) - return candidates[0] if candidates else None - - -def _log_references_interactions(log_path: str, interaction_ids: set[str]) -> bool: - """Return True when a process log references any of the interaction ids. - - Uses a cheap line substring scan so logs that cannot belong to the session - are rejected without a full JSON parse. - """ - try: - with open(log_path, encoding="utf-8", errors="replace") as handle: - for line in handle: - if "interaction_id" in line and any(iid in line for iid in interaction_ids): - return True - except OSError: - # Log cannot be read; treat as not referencing this session. - return False - return False - - -def find_process_logs_for_session( - state_dir: Path, home: Path, interaction_ids: set[str] -) -> list[str]: - """Return the process logs that hold usage for a session. - - Prefers the log named after the live session lock PID. When that lock is - gone (the session has ended), falls back to scanning every process log for - one whose entries reference this session's interaction ids, so per-request - input token data survives past session end rather than degrading to the - compaction-only state fallback. - """ - locked = find_process_log(state_dir, home) - if locked: - return [locked] - if not interaction_ids: - return [] - matches: list[str] = [] - for path in sorted(glob.glob(str(home / "logs" / "process-*.log"))): - if _log_references_interactions(path, interaction_ids): - matches.append(path) - return matches - - -def parse_process_log(log_path: str, interaction_ids: set[str]) -> list[dict]: - """Parse assistant_usage blocks from a process log, filtered by id.""" - results: list[dict] = [] - # Process logs use brace-delimited JSON blocks (one top-level '{' … '}' per - # entry) rather than newline-delimited JSON, so we accumulate lines between - # matching braces and only parse blocks containing assistant_usage data. - in_block = False - block_lines: list[str] = [] - block_has_usage = False - try: - with open(log_path, encoding="utf-8") as handle: - for line in handle: - stripped = line.rstrip() - if stripped == "{": - in_block = True - block_lines = [stripped] - block_has_usage = False - elif in_block: - block_lines.append(stripped) - if '"assistant_usage"' in stripped: - block_has_usage = True - if stripped == "}": - if block_has_usage: - try: - obj = json.loads("\n".join(block_lines)) - except ValueError: - obj = None - if obj and obj.get("kind") == "assistant_usage": - props = obj.get("properties", {}) - if props.get("interaction_id", "") in interaction_ids: - results.append(obj) - in_block = False - block_lines = [] - except OSError: - # Log cannot be read; return whatever was parsed so far. - return results - return results - - -def scan_session_state(state_file: str | os.PathLike[str]) -> dict: - """Read events.jsonl once for session metadata and interaction ids.""" - interaction_ids: set[str] = set() - models: dict[str, int] = {} - subagent_map: dict[str, str] = {} - messages = 0 - turns = 0 - reasoning_effort = "" - first_ts = "" - last_ts = "" - for evt in iter_jsonl(state_file): - data = evt.get("data", {}) - if not isinstance(data, dict): - continue - ts = evt.get("timestamp", "") - if ts: - if not first_ts or ts < first_ts: - first_ts = ts - if not last_ts or ts > last_ts: - last_ts = ts - etype = evt.get("type", "") - if etype == "assistant.message": - messages += 1 - model = data.get("model", "") - if model: - models[model] = models.get(model, 0) + 1 - iid = data.get("interactionId", "") - if iid: - interaction_ids.add(iid) - elif etype == "assistant.turn_start": - turns += 1 - iid = data.get("interactionId", "") - if iid: - interaction_ids.add(iid) - elif etype == "session.model_change": - reasoning_effort = data.get("reasoningEffort", "") - elif etype == "subagent.started": - tcid = data.get("toolCallId", "") - aname = data.get("agentName", "") or data.get("agentDisplayName", "") - if tcid and aname: - subagent_map[tcid] = aname - return { - "interaction_ids": interaction_ids, - "models": models, - "subagent_map": subagent_map, - "messages": messages, - "turns": turns, - "reasoning_effort": reasoning_effort, - "first_ts": first_ts, - "last_ts": last_ts, - } - - -def _totals_from_process_log(entries: list[dict]) -> dict: - """Accumulate token totals and per-model usage from process-log entries.""" - total_input = total_output = cache_read = cache_write = total_nano_aiu = 0 - total_input_uncached = 0 - model_usage: dict[str, dict] = {} - for entry in entries: - props = entry.get("properties", {}) - metrics = entry.get("metrics", {}) - model = props.get("model", "unknown") - total_input += metrics.get("input_tokens", 0) - total_input_uncached += metrics.get("input_tokens_uncached", 0) - total_output += metrics.get("output_tokens", 0) - cache_read += metrics.get("cache_read_tokens", 0) - cache_write += metrics.get("cache_write_tokens", 0) - total_nano_aiu += metrics.get("total_nano_aiu", 0) - bucket = model_usage.setdefault( - model, - { - "output_tokens": 0, - "messages": 0, - "input_tokens": 0, - "input_tokens_uncached": 0, - }, - ) - bucket["output_tokens"] += metrics.get("output_tokens", 0) - bucket["input_tokens"] += metrics.get("input_tokens", 0) - bucket["input_tokens_uncached"] += metrics.get("input_tokens_uncached", 0) - bucket["messages"] += 1 - return { - "output_tokens": total_output, - "input_tokens": total_input, - "input_tokens_uncached": total_input_uncached, - "cache_read_tokens": cache_read, - "cache_write_tokens": cache_write, - "total_nano_aiu": total_nano_aiu, - "model_usage": model_usage, - } - - -def _per_agent_usage_from_process_log( - entries: list[dict], subagent_map: dict[str, str] -) -> dict[str, dict]: - """Partition process-log entries by agent_id and compute per-agent totals. - - Returns a dict keyed by agent display name with token usage per subagent. - Only includes agents that have at least one request attributed to them. - The root agent (entries without agent_id or with initiator != "sub-agent") - is keyed as "root". - """ - agent_entries: dict[str, list[dict]] = {} - for entry in entries: - props = entry.get("properties", {}) - if props.get("initiator") == "sub-agent": - agent_id = props.get("agent_id", "") - label = subagent_map.get(agent_id, agent_id or "sub-agent") - else: - label = "root" - agent_entries.setdefault(label, []).append(entry) - - result: dict[str, dict] = {} - for label, agent_list in agent_entries.items(): - totals = _totals_from_process_log(agent_list) - result[label] = { - "output_tokens": totals["output_tokens"], - "input_tokens": totals["input_tokens"], - "input_tokens_uncached": totals.get("input_tokens_uncached", 0), - "cache_read_tokens": totals["cache_read_tokens"], - "cache_write_tokens": totals["cache_write_tokens"], - "total_nano_aiu": totals["total_nano_aiu"], - "requests": sum(1 for _ in agent_list), - } - return result - - -def _new_usage_bucket() -> dict: - """Return a fresh per-model usage bucket with the unified key shape.""" - return { - "output_tokens": 0, - "messages": 0, - "input_tokens": 0, - "input_tokens_uncached": 0, - } - - -def _totals_from_state_fallback(state_file: str | os.PathLike[str]) -> dict: - """Approximate token totals from events.jsonl when no process log exists. - - Sums per-model ``session.shutdown`` ``modelMetrics`` across every shutdown - in the file. A resumed session writes one shutdown per run segment with - counters that reset on each resume, so the segments must be summed to - recover whole-session totals. ``usage.inputTokens`` already includes cache - reads and writes, so fresh (uncached) input is recovered by subtracting - them. - - Output is reconciled against the ``assistant.message`` per-message sum, - which stays complete even when a run segment ends without ``modelMetrics`` - (an aborted or minimal shutdown) or emits subagent output under no tracked - model. The larger of the two sources wins so output is never undercounted. - - When no shutdown exists yet (a live session that has not ended a segment), - only per-message output is known; input, cache, and AIU are reported as - ``None`` so the report can distinguish "unknown" from a true zero. - """ - total_input = shutdown_output = cache_read = cache_write = total_nano_aiu = 0 - total_input_uncached = 0 - model_usage: dict[str, dict] = {} - msg_output_total = 0 - msg_output_by_model: dict[str, int] = {} - had_shutdown = False - for evt in iter_jsonl(state_file): - data = evt.get("data", {}) - if not isinstance(data, dict): - continue - etype = evt.get("type", "") - if etype == "assistant.message": - output_tokens = data.get("outputTokens", 0) - if output_tokens: - msg_output_total += output_tokens - model = data.get("model", "") - if model: - msg_output_by_model[model] = msg_output_by_model.get(model, 0) + output_tokens - continue - if etype != "session.shutdown": - continue - metrics = data.get("modelMetrics", {}) - if not isinstance(metrics, dict): - continue - had_shutdown = True - for model, m in metrics.items(): - if not isinstance(m, dict): - continue - usage = m.get("usage", {}) - in_tok = usage.get("inputTokens", 0) - out_tok = usage.get("outputTokens", 0) - cr = usage.get("cacheReadTokens", 0) - cw = usage.get("cacheWriteTokens", 0) - uncached = max(in_tok - cr - cw, 0) - requests = m.get("requests", {}).get("count", 0) - total_input += in_tok - shutdown_output += out_tok - cache_read += cr - cache_write += cw - total_input_uncached += uncached - total_nano_aiu += m.get("totalNanoAiu", 0) - bucket = model_usage.setdefault(model, _new_usage_bucket()) - bucket["output_tokens"] += out_tok - bucket["input_tokens"] += in_tok - bucket["input_tokens_uncached"] += uncached - bucket["messages"] += requests - if had_shutdown: - # Reconcile output per model and in total against the message sum, - # which is complete even when a segment lacked modelMetrics. - for model, out_tok in msg_output_by_model.items(): - bucket = model_usage.setdefault(model, _new_usage_bucket()) - if out_tok > bucket["output_tokens"]: - bucket["output_tokens"] = out_tok - return { - "output_tokens": max(shutdown_output, msg_output_total), - "input_tokens": total_input, - "input_tokens_uncached": total_input_uncached, - "cache_read_tokens": cache_read, - "cache_write_tokens": cache_write, - "total_nano_aiu": total_nano_aiu, - "model_usage": model_usage, - } - # Live session with no completed segment: only per-message output is known. - for model, out_tok in msg_output_by_model.items(): - model_usage.setdefault(model, _new_usage_bucket())["output_tokens"] += out_tok - return { - "output_tokens": msg_output_total, - "input_tokens": None, - "input_tokens_uncached": None, - "cache_read_tokens": None, - "cache_write_tokens": None, - "total_nano_aiu": None, - "model_usage": model_usage, - } - - -def build_session_summary( - sid: str, - state_dir: Path, - state_file: str | os.PathLike[str], - home: Path, - ts_override: str | None = None, - client: str = "", -) -> dict: - """Build a SessionSummary event for a session. - - Prefers precise per-request metrics from the CLI process log and falls - back to summed ``session.shutdown`` metrics in events.jsonl when the - process log is unavailable. The inner readers each swallow their own - ``OSError`` and yield empty data, so a summary is produced even when the - underlying files are unreadable. - """ - meta = scan_session_state(state_file) - interaction_ids = meta["interaction_ids"] - process_logs = find_process_logs_for_session(state_dir, home, interaction_ids) - totals = None - agent_usage: dict[str, dict] | None = None - token_source = "state_fallback" - if process_logs and interaction_ids: - entries: list[dict] = [] - for log in process_logs: - entries.extend(parse_process_log(log, interaction_ids)) - if entries: - totals = _totals_from_process_log(entries) - token_source = "process_log" - # Compute per-subagent token attribution when subagents were used. - if meta["subagent_map"]: - agent_usage = _per_agent_usage_from_process_log(entries, meta["subagent_map"]) - if totals is None: - totals = _totals_from_state_fallback(state_file) - - summary = { - "ts": ts_override if ts_override is not None else meta["last_ts"], - "sid": sid, - "event": "SessionSummary", - "first_ts": meta["first_ts"], - "last_ts": meta["last_ts"], - "models": meta["models"], - "model_usage": totals["model_usage"], - "output_tokens": totals["output_tokens"], - "input_tokens": totals["input_tokens"], - "cache_read_tokens": totals["cache_read_tokens"], - "cache_write_tokens": totals["cache_write_tokens"], - "total_nano_aiu": totals["total_nano_aiu"], - "token_source": token_source, - "turns": meta["turns"], - "messages": meta["messages"], - } - # Fresh (uncached) input is available from the process log and from summed - # session.shutdown metrics; omit the key only when the fallback found no - # shutdown (None) so the report can distinguish "unknown" from a true zero. - uncached = totals.get("input_tokens_uncached") - if uncached is not None: - summary["input_tokens_uncached"] = uncached - if meta["reasoning_effort"]: - summary["reasoning_effort"] = meta["reasoning_effort"] - if meta["subagent_map"]: - summary["subagent_map"] = meta["subagent_map"] - if agent_usage: - summary["agent_usage"] = agent_usage - if client: - summary["client"] = client - return summary - - -def _infer_event_from_shape(data: dict) -> str: - """Infer a canonical event name from payload shape. - - The Copilot CLI does not send an event-name field in the hook payload - (no ``hook_event_name``/``hookEventName``/``event``), unlike VS Code. It - also fires one shared command for every manifest event, so the event must - be inferred from which fields are present. Fields are resolved through - ``PAYLOAD_ALIASES``, so inference is surface-agnostic. Checks are ordered - so that events sharing fields are disambiguated by their distinguishing - field: - - * tool result present -> PostToolUse (also carries a tool name) - * tool name present -> PreToolUse - * prompt present -> UserPromptSubmit - * agent name present -> SubagentStop when a stop reason is set (a - Subagent stop), otherwise SubagentStart - * trigger present -> PreCompact - * stop reason present without an agent name -> Stop (an agent turn end) - * source present -> SessionStart - * ``reason`` holding a documented session-end value (``complete``, - ``error``, ``abort``, ``timeout``, ``user_exit``) -> SessionEnd (the - session terminates; unlike an agent Stop, no stop reason) - """ - if _payload_has(data, "tool_result"): - return "PostToolUse" - if _payload_has(data, "tool_name"): - return "PreToolUse" - if _payload_has(data, "prompt"): - return "UserPromptSubmit" - if _payload_has(data, "agent_name"): - return "SubagentStop" if _payload_has(data, "stop_reason") else "SubagentStart" - if _payload_has(data, "trigger"): - return "PreCompact" - if _payload_has(data, "stop_reason"): - return "Stop" - if _payload_has(data, "source"): - return "SessionStart" - if _payload_get(data, "reason") in SESSION_END_REASONS: - return "SessionEnd" - return "unknown" - - -def _normalize_event(data: dict) -> str: - """Resolve the canonical PascalCase event name from a hook payload.""" - event = "unknown" - for key in PAYLOAD_ALIASES["event_name"]: - value = data.get(key) - if value and value != "unknown": - event = value - break - if event == "unknown": - # The Copilot CLI omits the event name entirely; recover it from the - # payload shape so CLI sessions record telemetry like VS Code sessions. - event = _infer_event_from_shape(data) - return EVENT_ALIASES.get(event, event) - - -def _normalize_timestamp(raw_ts: object) -> str: - """Coerce a hook timestamp (epoch ms or string) to an ISO-8601 string.""" - if isinstance(raw_ts, (int, float)): - return datetime.datetime.fromtimestamp(raw_ts / 1000, tz=datetime.UTC).isoformat() - if isinstance(raw_ts, str) and raw_ts: - return raw_ts - return datetime.datetime.now(datetime.UTC).isoformat() - - -def _token_estimate(path: str) -> int: - """Estimate token count as ceil(file_size / 4). - - Uses the common ~4 chars-per-token heuristic for LLM token budgets. - """ - try: - # Ceiling division without importing math. - return int(-(-os.path.getsize(path) // 4)) - except OSError: - # File size unavailable (e.g., missing file); estimate zero tokens. - return 0 - - -class _AgentStack: - """Per-session record of active agents, kept as an append-only op log. - - Tracks which subagents have started but not stopped so telemetry entries can - attribute tool calls to the correct agent context. Subagents start and stop - in their own collector processes, so the state is appended and replayed on - read rather than read-modify-written, which would lose all but one of a - simultaneous batch. Appends go through :func:`append_line`, so the replay - also picks up any shard a writer left behind when it could not take the lock. - - Records are keyed by the surface's unique invocation id rather than the - agent's name: concurrent subagents of the same type share a name, so - name-keyed removal evicts an arbitrary one. - """ - - def __init__(self, stack_dir: Path, sid: str) -> None: - self.stack_dir = stack_dir - self.session_dir = stack_dir / sid if _is_safe_sid(sid) else None - self.stack_file = self.session_dir / "ops.log" if self.session_dir else None - - def _replay(self) -> list[tuple[str, str]]: - """Rebuild the (key, name) pairs of started-but-not-stopped agents.""" - active: list[tuple[str, str]] = [] - if not self.session_dir: - return active - ops: list[dict] = [] - for shard in self.session_dir.glob("*.log"): - ops.extend(iter_jsonl(shard)) - # Shards interleave, so recover a single order from the recorded time. - # A push must settle before a pop stamped in the same tick, or the pop - # matches nothing and its agent stays active forever. - ops.sort(key=lambda op: (str(op.get("ts", "")), 0 if op.get("op") == "push" else 1)) - for op in ops: - name = op.get("agent", "") - key = op.get("id") or name - if op.get("op") == "push": - active.append((key, name)) - continue - for i in reversed(range(len(active))): - if active[i][0] == key: - del active[i] - break - # An unmatched pop means its push was lost; keeping the other - # records reports the caller as unknown instead of evicting a - # live agent and misattributing every tool call after it. - return active - - def active(self) -> list[str]: - """Return the names of every started-but-not-stopped agent.""" - return [name for _, name in self._replay()] - - def push(self, agent_id: str, name: str = "") -> None: - """Record that an agent started. - - Args: - agent_id: The surface's unique invocation id, used as the record - key. Falls back to ``name`` when the payload omits it, which - makes concurrent same-named agents indistinguishable. - name: The agent's display name, reported by :meth:`active`. - """ - if not self.stack_file: - return - append_jsonl( - self.stack_file, - { - "ts": datetime.datetime.now(datetime.UTC).isoformat(), - "op": "push", - "id": agent_id or name, - "agent": name, - }, - ) - - def pop(self, agent_id: str, name: str = "") -> None: - """Record that an agent stopped. The log is discarded by :meth:`clear`. - - Args: - agent_id: The invocation id passed to :meth:`push`; ``name`` is the - fallback key, matching push so the pair cancels. - name: The agent's display name, used only as that fallback. - """ - if not self.session_dir or not self.session_dir.exists(): - return - append_jsonl( - self.stack_file, - { - "ts": datetime.datetime.now(datetime.UTC).isoformat(), - "op": "pop", - "id": agent_id or name, - }, - ) - - def clear(self) -> None: - """Discard this session's op log directory.""" - if not self.session_dir or not self.session_dir.is_dir(): - return - # Re-check containment at the point of deletion: _is_safe_sid gates the - # id, but a recursive remove should not rest on that alone. - if not _is_contained(self.session_dir, self.stack_dir): - return - shutil.rmtree(self.session_dir, ignore_errors=True) - - -def _attribute_agent(entry: dict, active: list[str]) -> None: - """Credit a tool call to the agent that made it, when that is knowable. - - A tool payload names no caller and a turn's tool calls run in parallel, so - an active subagent is not evidence that it issued this one. Only an empty - active set is conclusive; otherwise every candidate is recorded and the - choice is left to offline analysis. - - Args: - entry: The telemetry record, mutated in place. Gains ``agents`` (a - candidate list) when subagents are active, or ``agent`` (a single - name) when none are. The two shapes are mutually exclusive. - active: Names of every started-but-not-stopped agent. - """ - if active: - entry["agents"] = ["root", *active] - else: - entry["agent"] = "root" - - -def build_entry(data: dict, event: str, stack: _AgentStack) -> dict | None: - """Build the JSONL telemetry entry for a single hook event. - - Returns ``None`` for unrecognized events, which the caller drops. - """ - sid = _payload_get(data, "session_id") - cwd = _payload_get(data, "cwd", os.getcwd()) - ts = _normalize_timestamp(_payload_get(data, "timestamp")) - tool_name = _payload_get(data, "tool_name") - tool_input = _payload_get(data, "tool_input", {}) - tool_result = _payload_get(data, "tool_result") - - # The Copilot CLI serializes tool arguments as a JSON string rather than an - # object; decode it so key extraction and file-path detection below work. - if isinstance(tool_input, str): - try: - tool_input = json.loads(tool_input) - except ValueError: - tool_input = {} - - if event == "unknown": - return None - - entry: dict = {"ts": ts, "sid": sid, "event": event, "cwd": cwd} - - # Pairs a Pre with its Post, which is the only handle offline analysis has - # on when a call was in flight. - tool_use_id = _payload_get(data, "tool_use_id") - if tool_use_id and event in ("PreToolUse", "PostToolUse"): - entry["tool_use_id"] = tool_use_id - - if event == "SessionStart": - entry["source"] = _payload_get(data, "source") - entry["client"] = _detect_client() - elif event == "UserPromptSubmit": - entry["prompt"] = _payload_get(data, "prompt")[:200] - elif event == "PreToolUse": - entry["tool"] = tool_name - entry["tool_input_keys"] = list(tool_input.keys()) if isinstance(tool_input, dict) else [] - _attribute_agent(entry, stack.active()) - # Detect instructions and skills by file path convention to track - # which artifacts the agent loaded during the session. Normalize - # Windows backslash separators so splitting works cross-platform. - fpath = tool_input.get("filePath") if isinstance(tool_input, dict) else None - if isinstance(fpath, str): - norm = fpath.replace("\\", "/") - if norm.endswith(".instructions.md"): - entry["instruction"] = norm.split("/")[-1] - entry["tokens"] = _token_estimate(fpath) - elif norm.endswith("SKILL.md"): - # SKILL.md sits at the skill root in every layout, collection or flat. - parts = norm.rstrip("/").split("/") - if len(parts) >= 2: - entry["skill"] = parts[-2] - entry["tokens"] = _token_estimate(fpath) - if isinstance(tool_input, dict) and tool_name in ("runSubagent", "task"): - entry["subagent"] = ( - tool_input.get("agentName") - or tool_input.get("agent_type") - or tool_input.get("description", "") - ) - elif event == "PostToolUse": - entry["tool"] = tool_name - if isinstance(tool_result, dict): - text = _payload_get(tool_result, "tool_result_text") - entry["tool_response_len"] = len(text if isinstance(text, str) else str(text)) - elif isinstance(tool_result, str): - entry["tool_response_len"] = len(tool_result) - else: - entry["tool_response_len"] = len(json.dumps(tool_result)) - _attribute_agent(entry, stack.active()) - elif event == "SubagentStart": - agent_name = _payload_get(data, "agent_name") - agent_id = _payload_get(data, "agent_id") - entry["agent_name"] = agent_name - entry["agent_display_name"] = _payload_get(data, "agent_display_name") - if agent_id: - entry["agent_id"] = agent_id - stack.push(agent_id, agent_name) - elif event == "SubagentStop": - agent_name = _payload_get(data, "agent_name") - agent_id = _payload_get(data, "agent_id") - entry["agent_name"] = agent_name - if agent_id: - entry["agent_id"] = agent_id - stack.pop(agent_id, agent_name) - elif event == "PreCompact": - entry["trigger"] = _payload_get(data, "trigger") - elif event == "Stop": - # Fall back to ``reason`` for surfaces that name the event Stop but - # supply a session-end style payload without a stop reason. - entry["stop_reason"] = _payload_get(data, "stop_reason") or _payload_get(data, "reason") - stack.clear() - elif event == "SessionEnd": - entry["reason"] = _payload_get(data, "reason") - stack.clear() - - return entry - - -def _read_stdin_text() -> str: - """Read the hook payload as UTF-8 regardless of the host locale. - - Windows decodes ``sys.stdin`` with the ANSI codepage and a POSIX/C locale - decodes it as ASCII, either of which mangles or rejects UTF-8 payload - bytes. ``UnicodeDecodeError`` subclasses ``ValueError``, so a rejected - payload would otherwise be dropped as if it were malformed JSON. - """ - buffer = getattr(sys.stdin, "buffer", None) - if buffer is None: - return sys.stdin.read() - return buffer.read().decode("utf-8", errors="replace") - - -def _mode_collect() -> int: - """Process a single hook event from stdin; returns a process exit code.""" - try: - data = json.loads(_read_stdin_text()) - except ValueError: - return 0 - if not isinstance(data, dict): - return 0 - - sid = _payload_get(data, "session_id") - # Reject session IDs containing path separators or traversal sequences - # to prevent writes outside the telemetry directory. - if sid and not _is_safe_sid(sid): - return 0 - - event = _normalize_event(data) - tel_dir = Path(os.environ.get("HVE_TELEMETRY_DIR", ".copilot-tracking/telemetry")) - date_str = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d") - log_file = tel_dir / f"sessions-{date_str}.jsonl" - stack_dir = tel_dir / ".stacks" - stack = _AgentStack(stack_dir, sid) - - entry = build_entry(data, event, stack) - if entry is None: - return 0 - - # Register before the first write of the day so a store whose SessionStart - # was never delivered (telemetry enabled mid-session, or a surface that - # omits the event) still reaches cross-project reports. Bounded to one - # registry read per store per day plus each session start. - first_write = not log_file.exists() - try: - tel_dir.mkdir(parents=True, exist_ok=True) - except OSError: - # An unwritable store is a telemetry failure, not a turn failure. The - # writes below are best-effort and skip themselves for the same reason. - return 0 - if event == "SessionStart" or first_write: - register_telemetry_dir(tel_dir) - write_report_launchers() - append_jsonl(log_file, entry) - - # Enrich the log with a SessionSummary of token totals and model usage. - # Emitted at SessionEnd (the authoritative final snapshot), at Stop (each - # agent turn end, capturing periodically before process logs rotate), and - # at PreCompact: process logs rotate aggressively, so capturing a snapshot - # before compaction preserves per-request input data that would otherwise - # degrade to the compaction-only state fallback once the log is gone. - # Multiple summaries per session are expected; the report replaces by - # provenance rank and freshness rather than accumulating, so snapshots - # never double-count. - if event in ("Stop", "SessionEnd", "PreCompact") and sid: - home = copilot_home() - state_dir = home / "session-state" / sid - state_file = state_dir / "events.jsonl" - if state_file.is_file(): - summary = build_session_summary( - sid, - state_dir, - state_file, - home, - ts_override=entry["ts"], - client=_detect_client(), - ) - if summary is not None: - append_jsonl(log_file, summary) - return 0 - - -def _workspace_storage_dirs() -> list[Path]: - """Return candidate VS Code workspaceStorage roots for this host. - - Covers remote/server installs (``~/.vscode-server*/data``) and local - installs, whose user-data dir is platform specific. - """ - home = Path.home() - roots = [home / d / "data" for d in (".vscode-server-insiders", ".vscode-server", ".vscode")] - - if _is_windows(): - appdata = os.environ.get("APPDATA") - local_base = Path(appdata) if appdata else home / "AppData/Roaming" - elif sys.platform == "darwin": - local_base = home / "Library/Application Support" - else: - xdg = os.environ.get("XDG_CONFIG_HOME") - local_base = Path(xdg) if xdg else home / ".config" - - roots += [local_base / d for d in ("Code - Insiders", "Code", "VSCodium")] - return [r / "User/workspaceStorage" for r in roots] - - -def _mode_aggregate_debug(out: str, hook_files: list[str]) -> int: - """Emit llm_request events from VS Code debug logs for collected sids.""" - sids = collect_sids(hook_files) - if not sids: - return 1 - - patterns = [str(base / "**/debug-logs/**/*.jsonl") for base in _workspace_storage_dirs()] - count = 0 - with open(out, "w", encoding="utf-8") as writer: - for pattern in patterns: - for path in glob.glob(pattern, recursive=True): - for obj in iter_jsonl(path): - if obj.get("type") == "llm_request" and obj.get("sid") in sids: - writer.write(json.dumps(obj) + "\n") - count += 1 - return 0 if count else 1 - - -def _mode_aggregate_session(out: str, hook_files: list[str]) -> int: - """Emit SessionSummary events from CLI session state for collected sids.""" - sids = collect_sids(hook_files) - if not sids: - return 1 - - home = copilot_home() - state_base = home / "session-state" - count = 0 - with open(out, "w", encoding="utf-8") as writer: - for sid in sids: - state_dir = state_base / sid - state_file = state_dir / "events.jsonl" - if not state_file.is_file(): - continue - summary = build_session_summary(sid, state_dir, state_file, home, client="cli") - if summary is not None: - writer.write(json.dumps(summary) + "\n") - count += 1 - return 0 if count else 1 - - -# Telemetry artifacts written into a per-project store. Cleanup targets only -# these known names so a directory a user pointed ``HVE_TELEMETRY_DIR`` at is -# never removed wholesale. -_TELEMETRY_FILE_ARTIFACTS = ("raw-input.jsonl", "report.generated.html") -# One session log per UTC day, plus any fallback shard a collector wrote when it -# could not lock that log (sessions-.--.jsonl). -_TELEMETRY_GLOB_ARTIFACTS = ("sessions-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*.jsonl",) -_TELEMETRY_DIR_ARTIFACTS = (".stacks",) - -# Artifacts written into the HVE home directory (registry plus generated -# cross-project launchers and report). ``telemetry-dirs.txt`` and its lock are -# the pre-directory registry, listed so an upgraded install is tidied up too. -_HVE_HOME_ARTIFACTS = ( - "telemetry-dirs", - "telemetry-dirs.txt", - "telemetry-dirs.txt.lock", - "telemetry-dirs.txt.migrated", - "report.generated.html", - "generate-report.sh", - "generate-report.ps1", - "clean-telemetry.sh", - "clean-telemetry.ps1", -) - - -def _remove_path(path: Path, dry_run: bool, removed: list[str]) -> None: - """Remove a file or directory, recording the deleted path. - - Missing paths and removal errors are ignored so cleanup is best-effort and - never aborts partway. - """ - if not path.exists() and not path.is_symlink(): - return - if dry_run: - removed.append(str(path)) - return - try: - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink() - except OSError: - # Removal failed (e.g., permission denied); leave the path in place. - return - removed.append(str(path)) - - -def clean_telemetry_dir(tel_dir: Path, dry_run: bool, removed: list[str]) -> None: - """Remove known telemetry artifacts from a single per-project store.""" - if not tel_dir.is_dir(): - return - for name in _TELEMETRY_FILE_ARTIFACTS: - _remove_path(tel_dir / name, dry_run, removed) - for pattern in _TELEMETRY_GLOB_ARTIFACTS: - for match in sorted(tel_dir.glob(pattern)): - _remove_path(match, dry_run, removed) - for name in _TELEMETRY_DIR_ARTIFACTS: - _remove_path(tel_dir / name, dry_run, removed) - - -def _mode_clean(all_dirs: bool, dry_run: bool) -> int: - """Remove telemetry artifacts from the current store. - - With ``all_dirs`` the scope expands to every registered store plus the - generated launchers, report, and registry in the HVE home directory. - """ - removed: list[str] = [] - targets: list[Path] = [] - if all_dirs: - targets.extend(Path(d) for d in read_registry_dirs()) - targets.append(Path(os.environ.get("HVE_TELEMETRY_DIR", ".copilot-tracking/telemetry"))) - - seen: set[str] = set() - for tel_dir in targets: - try: - key = str(tel_dir.resolve()) - except OSError: - key = str(tel_dir) - if key in seen: - continue - seen.add(key) - clean_telemetry_dir(tel_dir, dry_run, removed) - - if all_dirs: - home = hve_home() - for name in _HVE_HOME_ARTIFACTS: - _remove_path(home / name, dry_run, removed) - - verb = "Would remove" if dry_run else "Removed" - if removed: - for item in removed: - sys.stdout.write(f"{verb}: {item}\n") - sys.stdout.write(f"{verb} {len(removed)} item(s).\n") - else: - sys.stdout.write("No telemetry artifacts found to remove.\n") - return 0 - - -def _mode_list_dirs() -> int: - """Print registered telemetry dirs that still exist; prune dead entries. - - Pruning unlinks the marker for a store that has been deleted, keeping the - cross-project report scan fast as repositories come and go. - """ - registry = telemetry_registry() - for directory in read_registry_dirs(registry): - if Path(directory).is_dir(): - sys.stdout.write(directory + "\n") - continue - try: - _registry_marker(registry, directory).unlink(missing_ok=True) - except OSError: - # Cannot prune this marker; leave it rather than fail the listing. - continue - return 0 - - -def main(argv: list[str]) -> int: - """Dispatch a CLI mode. See module docstring for the contract.""" - if not argv: - sys.stderr.write( - "usage: _telemetry_core.py " - " ...\n" - ) - return 2 - mode = argv[0] - if mode == "collect": - return _mode_collect() - if mode == "aggregate-debug": - if len(argv) < 2: - return 2 - return _mode_aggregate_debug(argv[1], argv[2:]) - if mode == "aggregate-session": - if len(argv) < 2: - return 2 - return _mode_aggregate_session(argv[1], argv[2:]) - if mode == "list-dirs": - return _mode_list_dirs() - if mode == "clean": - rest = argv[1:] - all_dirs = "--all-dirs" in rest or "-a" in rest - dry_run = "--dry-run" in rest or "-n" in rest - return _mode_clean(all_dirs=all_dirs, dry_run=dry_run) - sys.stderr.write(f"unknown mode: {mode}\n") - return 2 - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/.github/hooks/shared/telemetry/clean-telemetry.sh b/.github/hooks/shared/telemetry/clean-telemetry.sh deleted file mode 100755 index 7744deb4d..000000000 --- a/.github/hooks/shared/telemetry/clean-telemetry.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -# -# clean-telemetry.sh -# Removes telemetry artifacts written by the Copilot telemetry hooks. By -# default it cleans this project's telemetry store; --all-dirs extends the -# cleanup to every registered project plus the user-level HVE home directory. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly SCRIPT_DIR -readonly CORE_PY="${SCRIPT_DIR}/_telemetry_core.py" - -# Match the collector and anchor on the caller's workspace: the script itself -# may live outside the repo it cleans, so its location says nothing. -REPO_ROOT="${HVE_REPO_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || true)}" -[[ -n "${REPO_ROOT}" ]] || REPO_ROOT="${PWD}" -readonly REPO_ROOT - -usage() { - cat <<'EOF' -Usage: clean-telemetry.sh [options] - -Removes telemetry artifacts (sessions-*.jsonl, raw-input.jsonl, .stacks/, -report.generated.html) from a telemetry store. Unrelated files are preserved. - -Options: - -a, --all-dirs Also clean every per-project telemetry directory recorded - in the user-level registry, plus the generated launchers, - report, and registry in the HVE home directory. Ignored - when --path is given. - -p, --path DIR Telemetry directory. Scopes the cleanup to this directory - alone, overriding --all-dirs. - Default: /.copilot-tracking/telemetry - -n, --dry-run List what would be removed without deleting anything. - -y, --yes Skip the confirmation prompt (required for non-interactive - use). - -h, --help Show this help. - -EOF -} - -err() { - printf "ERROR: %s\n" "$1" >&2 - exit 1 -} - -# Prompt before destructive deletion. Aborts when no interactive terminal is -# available so cleanup never proceeds unattended without an explicit --yes. -confirm_cleanup() { - local scope_desc="$1" - printf "About to permanently remove telemetry artifacts from: %s\n" "${scope_desc}" - [[ -t 0 ]] || err "Confirmation required but no interactive terminal is available; re-run with --yes to proceed" - local reply - read -r -p "Continue? [y/N] " reply - case "${reply}" in - [yY]|[yY][eE][sS]) ;; - *) printf "Aborted.\n"; exit 0 ;; - esac -} - -main() { - local telemetry_path="${REPO_ROOT}/.copilot-tracking/telemetry" - local all_dirs=0 - local dry_run=0 - local assume_yes=0 - local path_explicit=0 - - while [[ $# -gt 0 ]]; do - case "$1" in - -a|--all-dirs) all_dirs=1; shift ;; - -p|--path) telemetry_path="$2"; path_explicit=1; shift 2 ;; - -n|--dry-run) dry_run=1; shift ;; - -y|--yes) assume_yes=1; shift ;; - -h|--help) usage; exit 0 ;; - *) err "Unknown option: $1" ;; - esac - done - - command -v python3 &>/dev/null || err "'python3' is required but not installed" - [[ -f "${CORE_PY}" ]] || err "Telemetry engine not found: ${CORE_PY}" - - # An explicit --path narrows the scope. The generated launcher always passes - # --all-dirs, so announce the override rather than silently widening or - # dropping a destructive scope flag. - if (( all_dirs && path_explicit )); then - all_dirs=0 - printf "Ignoring --all-dirs because --path was given; scope is '%s'.\n" \ - "${telemetry_path}" >&2 - fi - - if (( ! dry_run && ! assume_yes )); then - local scope_desc="${telemetry_path}" - (( all_dirs )) && scope_desc="ALL registered telemetry stores plus the user-level HVE home directory" - confirm_cleanup "${scope_desc}" - fi - - local -a args=("clean") - (( all_dirs )) && args+=("--all-dirs") - (( dry_run )) && args+=("--dry-run") - - HVE_TELEMETRY_DIR="${telemetry_path}" python3 "${CORE_PY}" "${args[@]}" -} - -main "$@" diff --git a/.github/hooks/shared/telemetry/generate-telemetry-report.sh b/.github/hooks/shared/telemetry/generate-telemetry-report.sh deleted file mode 100755 index 79d280f9b..000000000 --- a/.github/hooks/shared/telemetry/generate-telemetry-report.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -# -# generate-telemetry-report.sh -# Generates a self-contained telemetry report from hook JSONL files by -# embedding their contents into a copy of report.html. The generated report -# renders automatically without drag & drop. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly SCRIPT_DIR -readonly TEMPLATE_PATH="${SCRIPT_DIR}/report.html" - -# Match the collector and anchor on the caller's workspace: the script itself -# may live outside the repo it reports on, so its location says nothing. -REPO_ROOT="${HVE_REPO_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || true)}" -[[ -n "${REPO_ROOT}" ]] || REPO_ROOT="${PWD}" -readonly REPO_ROOT - -usage() { - cat <<'EOF' -Usage: generate-telemetry-report.sh [options] - -Options: - -d, --date DATE Target date (yyyy-MM-dd). Default: today (UTC). - Use 'all' to include every sessions-*.jsonl file. - -a, --all-dirs Scan every per-project telemetry directory recorded in - the user-level registry (~/.hve/telemetry-dirs) for - a combined cross-project report. Ignored when --path is - given. - -p, --path DIR Telemetry directory. Scopes the report to this directory - alone, overriding --all-dirs. - Default: /.copilot-tracking/telemetry - -l, --debug-log FILE Optional debug log JSONL (e.g. main.jsonl) for tokens. - When omitted, VS Code debug logs are auto-discovered and - the precise model version (e.g. claude-opus-4.6) plus - token data are joined in by session id. - -o, --output FILE Output path. Default: /report.generated.html - --open Open the generated report in the default browser. - -h, --help Show this help. - -EOF -} - -err() { - printf "ERROR: %s\n" "$1" >&2 - exit 1 -} - -# Best-effort enrichment: extract llm_request events (precise model, token, and -# duration data) from VS Code debug logs, filtered to the session ids already -# collected from hook files. Writes matching events to $1. Returns non-zero when -# python3 is unavailable or no matching events are found (e.g. CLI-only users). -aggregate_debug_requests() { - local out="$1"; shift - command -v python3 &>/dev/null || return 1 - python3 "${SCRIPT_DIR}/_telemetry_core.py" aggregate-debug "$out" "$@" -} - -# Enrichment from CLI session state: reads ~/.copilot/session-state/{sid}/events.jsonl -# and produces llm_request-compatible events for the report. This provides model, -# token, and duration data for CLI sessions that have no VS Code debug logs. -aggregate_session_state() { - local out="$1"; shift - command -v python3 &>/dev/null || return 1 - python3 "${SCRIPT_DIR}/_telemetry_core.py" aggregate-session "$out" "$@" -} - -# Emit the user-level registry of per-project telemetry directories (one path -# per line), pruning stale entries. Used by --all-dirs for cross-project reports. -registry_dirs() { - command -v python3 &>/dev/null || return 0 - python3 "${SCRIPT_DIR}/_telemetry_core.py" list-dirs 2>/dev/null || true -} - -main() { - local target_date - target_date="$(date -u +%Y-%m-%d)" - local telemetry_path="${REPO_ROOT}/.copilot-tracking/telemetry" - local debug_log="" - local output_path="" - local open_report=0 - local all_dirs=0 - local path_explicit=0 - - # Temp files/dirs cleaned up on return (single trap to avoid overrides). - # EXIT is listed too: several paths below exit outright, and RETURN alone - # would leave the enrichment temp directory behind in /tmp on each of them. - local -a tmp_files=() - # shellcheck disable=SC2154 # 't' is the loop variable, assigned within the trap body. - trap 'for t in "${tmp_files[@]:-}"; do [[ -n "$t" ]] && rm -rf "$t"; done' RETURN EXIT - - while [[ $# -gt 0 ]]; do - case "$1" in - -d|--date) target_date="$2"; shift 2 ;; - -a|--all-dirs) all_dirs=1; shift ;; - -p|--path) telemetry_path="$2"; path_explicit=1; shift 2 ;; - -l|--debug-log) debug_log="$2"; shift 2 ;; - -o|--output) output_path="$2"; shift 2 ;; - --open) open_report=1; shift ;; - -h|--help) usage; exit 0 ;; - *) err "Unknown option: $1" ;; - esac - done - - command -v jq &>/dev/null || err "'jq' is required but not installed" - [[ -f "${TEMPLATE_PATH}" ]] || err "Template not found: ${TEMPLATE_PATH}" - - # Default the output alongside the telemetry data it summarizes. - [[ -n "${output_path}" ]] || output_path="${telemetry_path}/report.generated.html" - - # Determine which telemetry directories to scan. With --all-dirs, prepend - # every directory recorded in the user-level registry (cross-project view). - # An explicit --path wins so the generated cross-project launcher, which - # always passes --all-dirs, can still be scoped to one store. - if (( all_dirs && path_explicit )); then - printf "Ignoring --all-dirs because --path was given; scanning '%s' only.\n" \ - "${telemetry_path}" >&2 - fi - declare -a search_dirs=() - if (( all_dirs && ! path_explicit )); then - while IFS= read -r d; do - [[ -n "$d" ]] && search_dirs+=("$d") - done < <(registry_dirs) - fi - search_dirs+=("${telemetry_path}") - - # Collect session files for the target date across the chosen directories, - # de-duplicating directories that appear more than once. The trailing '*' - # also picks up any shard a collector wrote when it could not lock the shared - # log; report.html orders every record it loads by ts. - local prefix="sessions-${target_date}" - [[ "${target_date}" == "all" ]] && prefix="sessions-" - declare -a files=() - local seen_dirs="" - local dir f - for dir in "${search_dirs[@]}"; do - [[ -n "$dir" && "$seen_dirs" != *"|${dir}|"* ]] || continue - seen_dirs+="|${dir}|" - [[ -d "$dir" ]] || continue - # A glob rather than find | sort -z: pathname expansion is already sorted - # and needs no GNU-only flags, so this runs on macOS's BSD userland. Only - # the '*' is unquoted, so a date holding a space cannot split into two - # patterns; an unmatched glob expands to itself and the -f test discards it. - for f in "$dir"/"$prefix"*.jsonl; do - [[ -f "$f" ]] && files+=("$f") - done - done - - if [[ -n "${debug_log}" ]]; then - [[ -f "${debug_log}" ]] || err "Debug log not found: ${debug_log}" - files+=("${debug_log}") - elif [[ ${#files[@]} -gt 0 ]]; then - # Auto-enrich with precise model + token data from VS Code debug logs, - # scoped to the sessions already collected. Silently skipped when none match. - local agg_dir agg_file - agg_dir="$(mktemp -d)" - tmp_files+=("${agg_dir}") - agg_file="${agg_dir}/debug-llm-requests.jsonl" - if aggregate_debug_requests "${agg_file}" "${files[@]}"; then - files+=("${agg_file}") - fi - - # Also enrich from CLI session state (provides model/token data for CLI users). - local cli_agg_file="${agg_dir}/cli-session-state.jsonl" - if aggregate_session_state "${cli_agg_file}" "${files[@]}"; then - files+=("${cli_agg_file}") - fi - fi - - if [[ ${#files[@]} -eq 0 ]]; then - printf "No telemetry files found in '%s' for date '%s'.\n" \ - "${telemetry_path}" "${target_date}" >&2 - exit 0 - fi - - # Build a compact JSON array of {name, content} objects with jq, then escape - # every '<' as \u003c. Session content routinely quotes markup, and inside a - # but also an - # unbalanced - - - - - -Local Generated Report - - - - -

Local Generated Report

-

Compare agents, models, instructions, and skills across local sessions

- -
-

Drag & drop session JSONL files here or click to browse

-

sessions-*.jsonl · main.jsonl (debug logs for token data)

- -
-
- - - - -
-
- -

Session Matrix

-
- - - -
- -

Loaded Instructions & Skills

-

Instruction and skill files a session opened via a tool call (estimated ~4 chars/token). Host-attached files are not counted here — see per-session Token Usage above for actual cost.

-
- -

Tool Heatmap

-

Tool call counts per session (PreToolUse events)

-
- -

Tool Latency

-

Avg tool execution time from PreToolUse→PostToolUse pairs (ms)

-
-
- - - - diff --git a/.github/hooks/shared/telemetry/telemetry-collector.sh b/.github/hooks/shared/telemetry/telemetry-collector.sh deleted file mode 100755 index 52995d8d3..000000000 --- a/.github/hooks/shared/telemetry/telemetry-collector.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -# -# telemetry-collector.sh -# Copilot hook handler that appends structured JSONL telemetry events. -# Uses Python3 for JSON processing. Fast no-op when telemetry is disabled. - -set -euo pipefail - -main() { - local input - input=$(cat) - - # Resolve repository root for reliable path anchoring across all surfaces - # (CLI, VS Code, cloud agent). The explicit override wins over git discovery - # so a host can anchor the store outside the current checkout. - local repo_root="${HVE_REPO_ROOT:-}" - if [[ -z "$repo_root" ]] && command -v git &>/dev/null; then - repo_root="$(git rev-parse --show-toplevel 2>/dev/null || true)" - fi - [[ -z "$repo_root" ]] && repo_root="." - - # Opt-in gate: exit immediately if telemetry is not enabled - if [[ "${HVE_TELEMETRY:-}" != "1" ]]; then - if [[ ! -f "$repo_root/.hve-telemetry" ]]; then - echo '{"continue":true}' - return 0 - fi - fi - - # Require Python for JSON processing. The engine needs 3.11 (datetime.UTC), - # and on some systems `python` is still a 2.x or an older 3.x, so probe the - # version rather than trusting the name. - local python_bin="" candidate - for candidate in python3 python; do - if command -v "$candidate" &>/dev/null && - "$candidate" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)' &>/dev/null; then - python_bin="$candidate" - break - fi - done - if [[ -z "$python_bin" ]]; then - echo "WARNING: HVE telemetry enabled but no Python 3.11+ found, events will not be recorded" >&2 - echo '{"continue":true}' - return 0 - fi - - # Resolve the shared telemetry engine from the skill directory. - local script_dir core_py - script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - core_py="${script_dir}/_telemetry_core.py" - - if [[ ! -f "$core_py" ]]; then - echo "WARNING: Telemetry engine not found at $core_py, events will not be recorded" >&2 - echo '{"continue":true}' - return 0 - fi - - local telemetry_dir="${HVE_TELEMETRY_DIR:-$repo_root/.copilot-tracking/telemetry}" - # An unwritable store is a telemetry failure, not a turn failure: under set -e - # an unguarded mkdir would exit before the continue signal and stall the turn. - if ! mkdir -p "$telemetry_dir" "$telemetry_dir/.stacks" 2>/dev/null; then - echo '{"continue":true}' - return 0 - fi - - # Dump raw input for diagnostics (first 5 events only). This records hook - # payloads verbatim, including the full prompt text and tool inputs such as - # file contents and shell command strings, which can contain secrets. The - # processed sessions-*.jsonl stream already provides the diagnostic signal, - # so the verbatim dump is a separate explicit opt-in (off by default) layered - # on top of the telemetry gate. See docs/customization/local-telemetry.md. - if [[ "${HVE_TELEMETRY_RAW:-}" == "1" ]]; then - local raw_log="$telemetry_dir/raw-input.jsonl" - local raw_count=0 - if [[ -f "$raw_log" ]]; then - raw_count=$(wc -l < "$raw_log") - fi - if (( raw_count < 5 )); then - echo "$input" >> "$raw_log" - fi - fi - - # Delegate all JSON processing to the shared telemetry engine. The engine - # records the event and, at Stop, SessionEnd, and PreCompact, enriches the - # session with token/cost data. - export HVE_REPO_ROOT="$repo_root" - export HVE_TELEMETRY_DIR="$telemetry_dir" - echo "$input" | "$python_bin" "$core_py" collect || true - - echo '{"continue":true}' -} - -main "$@" diff --git a/.github/hooks/shared/telemetry/tests/corpus/0_jsonl_event b/.github/hooks/shared/telemetry/tests/corpus/0_jsonl_event deleted file mode 100644 index d131b1928..000000000 --- a/.github/hooks/shared/telemetry/tests/corpus/0_jsonl_event +++ /dev/null @@ -1 +0,0 @@ -0{"event": "SessionStart", "session_id": "abc", "timestamp": "t"} diff --git a/.github/hooks/shared/telemetry/tests/corpus/1_event_alias b/.github/hooks/shared/telemetry/tests/corpus/1_event_alias deleted file mode 100644 index ee8d65fbd..000000000 --- a/.github/hooks/shared/telemetry/tests/corpus/1_event_alias +++ /dev/null @@ -1 +0,0 @@ -1sessionStart \ No newline at end of file diff --git a/.github/hooks/shared/telemetry/tests/corpus/2_build_entry b/.github/hooks/shared/telemetry/tests/corpus/2_build_entry deleted file mode 100644 index 0eecce576..000000000 --- a/.github/hooks/shared/telemetry/tests/corpus/2_build_entry +++ /dev/null @@ -1 +0,0 @@ -2PreToolUse \ No newline at end of file diff --git a/.github/hooks/shared/telemetry/tests/corpus/README.md b/.github/hooks/shared/telemetry/tests/corpus/README.md deleted file mode 100644 index 37b581549..000000000 --- a/.github/hooks/shared/telemetry/tests/corpus/README.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -title: Fuzz Corpus Seeds -description: Seed inputs for coverage-guided fuzzing with the Atheris fuzz harness -author: Microsoft -ms.date: 2026-06-18 -ms.topic: reference -keywords: - - fuzz - - corpus - - atheris - - telemetry -estimated_reading_time: 2 ---- - - -# Fuzz Corpus Seeds - -Seed inputs for the telemetry Atheris fuzz harness. Each file is raw bytes consumed -by `fuzz_dispatch`, which routes `data[0] % 3` to one of three targets. - -## Naming Convention - -`{target_index}_{description}` where `target_index` matches the `FUZZ_TARGETS` -array position: - -| Index | Target | -|-------|------------------------| -| 0 | `fuzz_iter_jsonl` | -| 1 | `fuzz_normalize_event` | -| 2 | `fuzz_build_entry` | - -The first byte selects the target; the remaining bytes are the input payload. - -*πŸ€– Crafted with precision by ✨Copilot following brilliant human instruction, then carefully refined by our team of discerning human reviewers.* diff --git a/.github/hooks/shared/telemetry/tests/fuzz_harness.py b/.github/hooks/shared/telemetry/tests/fuzz_harness.py deleted file mode 100644 index 07e0f1f7a..000000000 --- a/.github/hooks/shared/telemetry/tests/fuzz_harness.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -"""Polyglot fuzz harness for telemetry core logic. - -Runs as a pytest test when Atheris is not installed. -Runs as an Atheris coverage-guided fuzz target when executed directly. -""" - -from __future__ import annotations - -import sys -from contextlib import suppress -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -import _telemetry_core as core # noqa: E402 - -try: - import atheris -except ImportError: - atheris = None - FUZZING = False -else: - FUZZING = True - - -def fuzz_iter_jsonl(data: bytes) -> None: - """Fuzz JSONL parsing with arbitrary bytes.""" - provider = atheris.FuzzedDataProvider(data) - import tempfile - from pathlib import Path - - content = provider.ConsumeUnicodeNoSurrogates(500) - with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f: - f.write(content) - f.flush() - list(core.iter_jsonl(f.name)) - Path(f.name).unlink(missing_ok=True) - - -def fuzz_normalize_event(data: bytes) -> None: - """Fuzz event normalization with arbitrary payloads.""" - provider = atheris.FuzzedDataProvider(data) - payload = {} - for key in ("hook_event_name", "hookEventName", "event"): - if provider.ConsumeBool(): - payload[key] = provider.ConsumeUnicodeNoSurrogates(30) - core._normalize_event(payload) - - -def fuzz_build_entry(data: bytes) -> None: - """Fuzz entry building with arbitrary payloads.""" - provider = atheris.FuzzedDataProvider(data) - events = [ - "SessionStart", - "UserPromptSubmit", - "PreToolUse", - "PostToolUse", - "SubagentStart", - "SubagentStop", - "PreCompact", - "Stop", - "unknown", - ] - event = events[provider.ConsumeIntInRange(0, len(events) - 1)] - payload = { - "session_id": provider.ConsumeUnicodeNoSurrogates(20), - "timestamp": provider.ConsumeUnicodeNoSurrogates(30), - "tool_name": provider.ConsumeUnicodeNoSurrogates(15), - "prompt": provider.ConsumeUnicodeNoSurrogates(50), - } - import shutil - import tempfile - - stack_dir = Path(tempfile.mkdtemp()) - stack = core._AgentStack(stack_dir, payload["session_id"]) - with suppress(Exception): - core.build_entry(payload, event, stack) - # rmtree, not unlink: a session's op log lives in a subdirectory. - shutil.rmtree(stack_dir, ignore_errors=True) - - -FUZZ_TARGETS = [ - fuzz_iter_jsonl, - fuzz_normalize_event, - fuzz_build_entry, -] - - -def fuzz_dispatch(data: bytes) -> None: - """Route input to one fuzz target.""" - if len(data) < 2: - return - target_index = data[0] % len(FUZZ_TARGETS) - FUZZ_TARGETS[target_index](data[1:]) - - -class TestTelemetryFuzzHarness: - """Property tests mirroring fuzz-target behavior.""" - - def test_given_aliased_events_when_normalize_event_then_resolves(self) -> None: - assert core._normalize_event({"hook_event_name": "sessionStart"}) == "SessionStart" - assert core._normalize_event({"hook_event_name": "agentStop"}) == "Stop" - - def test_given_unknown_event_when_normalize_event_then_passes_through(self) -> None: - assert core._normalize_event({"hook_event_name": "CustomEvent"}) == "CustomEvent" - - def test_given_fallback_keys_when_normalize_event_then_resolves(self) -> None: - assert core._normalize_event({"hookEventName": "preToolUse"}) == "PreToolUse" - assert core._normalize_event({"event": "postToolUse"}) == "PostToolUse" - - def test_given_unknown_event_when_build_entry_then_returns_none(self, tmp_path) -> None: - stack = core._AgentStack(tmp_path / ".stacks", "sid") - assert core.build_entry({}, "unknown", stack) is None - - def test_given_session_start_when_build_entry_then_populates_fields(self, tmp_path) -> None: - stack = core._AgentStack(tmp_path / ".stacks", "sid") - entry = core.build_entry( - {"hook_event_name": "SessionStart", "source": "cli", "timestamp": "t"}, - "SessionStart", - stack, - ) - assert entry["event"] == "SessionStart" - assert entry["source"] == "cli" - - def test_given_non_dict_lines_when_iter_jsonl_then_skips_them(self, tmp_path) -> None: - f = tmp_path / "test.jsonl" - f.write_text('[1, 2]\n"string"\n{"valid": true}\n') - rows = list(core.iter_jsonl(f)) - assert rows == [{"valid": True}] - - -if __name__ == "__main__" and FUZZING: - atheris.instrument_all() - atheris.Setup(sys.argv, fuzz_dispatch) - atheris.Fuzz() diff --git a/.github/hooks/shared/telemetry/tests/test_telemetry_core.py b/.github/hooks/shared/telemetry/tests/test_telemetry_core.py deleted file mode 100644 index 8fb89b8d0..000000000 --- a/.github/hooks/shared/telemetry/tests/test_telemetry_core.py +++ /dev/null @@ -1,1412 +0,0 @@ -# Copyright (c) 2026 Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT -"""Tests for the canonical telemetry engine (_telemetry_core).""" - -from __future__ import annotations - -import io -import json -import os -import re -import subprocess -import sys - -import pytest - -import _telemetry_core as core - - -def _write_jsonl(path, rows): - """Write a list of dicts as newline-delimited JSON.""" - path.write_text("".join(json.dumps(r) + "\n" for r in rows)) - - -@pytest.fixture(autouse=True) -def _isolate_hve_home(tmp_path, monkeypatch): - """Keep registry and launcher writes out of the real user home.""" - monkeypatch.setenv("HVE_HOME", str(tmp_path / "hve-home")) - - -def test_given_blank_and_malformed_lines_when_iter_jsonl_then_skips_them(tmp_path): - f = tmp_path / "events.jsonl" - f.write_text('{"a": 1}\n\n \nnot-json\n{"b": 2}\n') - rows = list(core.iter_jsonl(f)) - assert rows == [{"a": 1}, {"b": 2}] - - -def test_given_missing_file_when_iter_jsonl_then_returns_empty(tmp_path): - assert list(core.iter_jsonl(tmp_path / "nope.jsonl")) == [] - - -def test_given_overlapping_sids_when_collect_sids_then_dedups(tmp_path): - a = tmp_path / "a.jsonl" - b = tmp_path / "b.jsonl" - _write_jsonl(a, [{"sid": "s1"}, {"sid": "s2"}, {"event": "x"}]) - _write_jsonl(b, [{"sid": "s2"}, {"sid": "s3"}]) - assert core.collect_sids([str(a), str(b)]) == {"s1", "s2", "s3"} - - -def test_given_lock_pid_when_find_process_log_then_resolves_path(tmp_path): - home = tmp_path / "home" - state_dir = home / "session-state" / "sid1" - state_dir.mkdir(parents=True) - (state_dir / "inuse.4242.lock").write_text("") - logs = home / "logs" - logs.mkdir() - target = logs / "process-abc-4242.log" - target.write_text("{}\n") - assert core.find_process_log(state_dir, home) == str(target) - - -def test_given_no_lock_when_find_process_log_then_returns_none(tmp_path): - home = tmp_path / "home" - state_dir = home / "session-state" / "sid1" - state_dir.mkdir(parents=True) - assert core.find_process_log(state_dir, home) is None - - -def test_given_mixed_blocks_when_parse_process_log_then_filters_by_interaction_and_kind(tmp_path): - log = tmp_path / "process.log" - log.write_text( - "{\n" - ' "kind": "assistant_usage",\n' - ' "properties": {"interaction_id": "i1", "model": "m"},\n' - ' "metrics": {"output_tokens": 5}\n' - "}\n" - "{\n" - ' "kind": "assistant_usage",\n' - ' "properties": {"interaction_id": "other"}\n' - "}\n" - "{\n" - ' "kind": "something_else"\n' - "}\n" - ) - entries = core.parse_process_log(str(log), {"i1"}) - assert len(entries) == 1 - assert entries[0]["properties"]["interaction_id"] == "i1" - - -def test_given_session_events_when_scan_session_state_then_collects_metadata(tmp_path): - state = tmp_path / "events.jsonl" - _write_jsonl( - state, - [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:01Z", - "data": {"model": "gpt", "interactionId": "i1"}, - }, - { - "type": "assistant.turn_start", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"interactionId": "i2"}, - }, - { - "type": "session.model_change", - "timestamp": "2026-01-01T00:00:02Z", - "data": {"reasoningEffort": "high"}, - }, - { - "type": "subagent.started", - "timestamp": "2026-01-01T00:00:03Z", - "data": {"toolCallId": "t1", "agentName": "Researcher"}, - }, - ], - ) - meta = core.scan_session_state(state) - assert meta["messages"] == 1 - assert meta["turns"] == 1 - assert meta["models"] == {"gpt": 1} - assert meta["interaction_ids"] == {"i1", "i2"} - assert meta["reasoning_effort"] == "high" - assert meta["subagent_map"] == {"t1": "Researcher"} - assert meta["first_ts"] == "2026-01-01T00:00:00Z" - assert meta["last_ts"] == "2026-01-01T00:00:03Z" - - -def _make_session(tmp_path, sid, state_rows, process_rows=None, pid=None, write_lock=True): - """Create a minimal session directory tree with optional process log. - - Set ``write_lock=False`` to simulate a session that has ended (its lock - file removed) while its process log still exists, exercising the - interaction-id fallback path. - """ - home = tmp_path / "home" - state_dir = home / "session-state" / sid - state_dir.mkdir(parents=True) - _write_jsonl(state_dir / "events.jsonl", state_rows) - if pid is not None and write_lock: - (state_dir / f"inuse.{pid}.lock").write_text("") - if process_rows is not None and pid is not None: - logs = home / "logs" - logs.mkdir(exist_ok=True) - # Build process-log blocks in the brace-delimited format the parser - # expects (top-level '{' on its own line, not JSONL). - blocks = [] - for r in process_rows: - inner = json.dumps(r, indent=2) - blocks.append(inner + "\n") - text = "".join(blocks) - (logs / f"process-x-{pid}.log").write_text(text) - return home, state_dir, state_dir / "events.jsonl" - - -def test_given_process_log_when_build_session_summary_then_uses_process_log(tmp_path): - state_rows = [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"model": "m", "interactionId": "i1"}, - } - ] - process_rows = [ - { - "kind": "assistant_usage", - "properties": {"interaction_id": "i1", "model": "m"}, - "metrics": { - "input_tokens": 10, - "input_tokens_uncached": 7, - "output_tokens": 20, - "cache_read_tokens": 1, - "cache_write_tokens": 2, - "total_nano_aiu": 99, - }, - } - ] - home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows, process_rows, pid=777) - summary = core.build_session_summary("sid1", state_dir, state_file, home) - assert summary["input_tokens"] == 10 - assert summary["input_tokens_uncached"] == 7 - assert summary["output_tokens"] == 20 - assert summary["cache_write_tokens"] == 2 - assert summary["total_nano_aiu"] == 99 - assert summary["model_usage"]["m"]["input_tokens"] == 10 - assert summary["model_usage"]["m"]["input_tokens_uncached"] == 7 - assert summary["token_source"] == "process_log" - - -def test_given_ended_session_when_build_summary_then_matches_log_by_iid(tmp_path): - state_rows = [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"model": "m", "interactionId": "i1"}, - } - ] - process_rows = [ - { - "kind": "assistant_usage", - "properties": {"interaction_id": "i1", "model": "m"}, - "metrics": { - "input_tokens": 30, - "input_tokens_uncached": 5, - "output_tokens": 12, - "cache_read_tokens": 25, - "cache_write_tokens": 0, - "total_nano_aiu": 42, - }, - } - ] - # pid names the process log file, but write_lock=False removes the lock so - # the PID-based lookup fails and the interaction-id scan must recover it. - home, state_dir, state_file = _make_session( - tmp_path, "sid1", state_rows, process_rows, pid=888, write_lock=False - ) - summary = core.build_session_summary("sid1", state_dir, state_file, home) - assert summary["token_source"] == "process_log" - assert summary["input_tokens"] == 30 - assert summary["input_tokens_uncached"] == 5 - - -def test_given_no_process_log_when_build_session_summary_then_falls_back_to_state(tmp_path): - state_rows = [ - { - "type": "session.shutdown", - "timestamp": "2026-01-01T00:00:01Z", - "data": { - "modelMetrics": { - "m": { - "requests": {"count": 2}, - "usage": { - "inputTokens": 12, - "outputTokens": 7, - "cacheReadTokens": 4, - "cacheWriteTokens": 5, - }, - "totalNanoAiu": 50, - } - } - }, - }, - ] - home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) - summary = core.build_session_summary("sid1", state_dir, state_file, home) - assert summary["output_tokens"] == 7 - assert summary["input_tokens"] == 12 - assert summary["cache_read_tokens"] == 4 - # Unified schema always reports cache_write_tokens. - assert summary["cache_write_tokens"] == 5 - assert summary["token_source"] == "state_fallback" - assert summary["total_nano_aiu"] == 50 - # inputTokens includes cache, so fresh input is recovered by subtraction. - assert summary["input_tokens_uncached"] == 3 - - -def test_given_resumed_session_when_build_session_summary_then_sums_shutdowns(tmp_path): - def _shutdown(ts, in_tok, out_tok, cr, cw, nano): - return { - "type": "session.shutdown", - "timestamp": ts, - "data": { - "modelMetrics": { - "m": { - "requests": {"count": 1}, - "usage": { - "inputTokens": in_tok, - "outputTokens": out_tok, - "cacheReadTokens": cr, - "cacheWriteTokens": cw, - }, - "totalNanoAiu": nano, - } - } - }, - } - - state_rows = [ - _shutdown("2026-01-01T00:00:01Z", 10, 3, 4, 2, 20), - _shutdown("2026-01-01T00:00:02Z", 30, 5, 8, 6, 40), - ] - home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) - summary = core.build_session_summary("sid1", state_dir, state_file, home) - assert summary["token_source"] == "state_fallback" - assert summary["input_tokens"] == 40 - assert summary["output_tokens"] == 8 - assert summary["cache_read_tokens"] == 12 - assert summary["cache_write_tokens"] == 8 - assert summary["total_nano_aiu"] == 60 - # Fresh input summed per segment: (10-4-2) + (30-8-6) = 4 + 16 = 20. - assert summary["input_tokens_uncached"] == 20 - - -def test_given_no_shutdown_when_build_session_summary_then_input_unknown(tmp_path): - state_rows = [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"model": "m", "outputTokens": 7, "interactionId": "i1"}, - }, - ] - home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) - summary = core.build_session_summary("sid1", state_dir, state_file, home) - assert summary["output_tokens"] == 7 - # No shutdown segment exists, so input is unknown (None), not a true zero. - assert summary["input_tokens"] is None - assert summary["cache_read_tokens"] is None - assert summary["total_nano_aiu"] is None - assert summary["token_source"] == "state_fallback" - # Fresh input is unknown, so the key is omitted. - assert "input_tokens_uncached" not in summary - - -def test_given_shutdown_missing_metrics_when_summary_then_output_from_messages(tmp_path): - # One segment ends with modelMetrics; a later segment aborts without them. - # assistant.message output stays complete, so the message sum (3+9=12) - # must win over the lone shutdown's output (3). - state_rows = [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"model": "m", "outputTokens": 3, "interactionId": "i1"}, - }, - { - "type": "session.shutdown", - "timestamp": "2026-01-01T00:00:01Z", - "data": { - "modelMetrics": { - "m": { - "requests": {"count": 1}, - "usage": { - "inputTokens": 10, - "outputTokens": 3, - "cacheReadTokens": 4, - "cacheWriteTokens": 2, - }, - "totalNanoAiu": 20, - } - } - }, - }, - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:02Z", - "data": {"model": "m", "outputTokens": 9, "interactionId": "i2"}, - }, - # Aborted resume segment: shutdown present but no modelMetrics. - { - "type": "session.shutdown", - "timestamp": "2026-01-01T00:00:03Z", - "data": {}, - }, - ] - home, state_dir, state_file = _make_session(tmp_path, "sid1", state_rows) - summary = core.build_session_summary("sid1", state_dir, state_file, home) - assert summary["token_source"] == "state_fallback" - # Output reconciled to the complete message sum, not the undercounting shutdown. - assert summary["output_tokens"] == 12 - assert summary["model_usage"]["m"]["output_tokens"] == 12 - # Input/cache/AIU still come from the one segment that reported metrics. - assert summary["input_tokens"] == 10 - assert summary["cache_read_tokens"] == 4 - assert summary["input_tokens_uncached"] == 4 - - -def _session_records(tel_dir): - """Read every record the store holds, in file order.""" - records = [] - for path in _session_files(tel_dir): - text = path.read_text(encoding="utf-8") - records.extend(json.loads(line) for line in text.splitlines() if line.strip()) - return records - - -def _session_files(tel_dir): - """Return every session log and shard the store holds, in name order.""" - return sorted(tel_dir.glob("sessions-*.jsonl")) - - -def test_given_one_active_agent_when_replayed_then_reports_full_name(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - stack.push("sub-1", "Researcher") - assert stack.active() == ["Researcher"] - - -def test_given_overlapping_subagents_when_one_stops_then_removes_it_by_id(tmp_path): - """Subagents can overlap, so a stop must not simply drop the newest entry.""" - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - stack.push("sub-1", "Alpha") - stack.push("sub-2", "Beta") - stack.pop("sub-1", "Alpha") - assert stack.active() == ["Beta"] - stack.pop("sub-2", "Beta") - assert stack.active() == [] - - -def test_given_same_named_subagents_when_one_stops_then_keeps_the_other(tmp_path): - """Concurrent subagents share a type name; only the invocation id separates them.""" - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - stack.push("sub-1", "Explore") - stack.push("sub-2", "Explore") - stack.pop("sub-1", "Explore") - assert stack.active() == ["Explore"] - - -def test_given_unmatched_pop_when_replaying_then_keeps_live_agents(tmp_path): - """A lost push must not let the stop evict an unrelated running agent.""" - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - stack.push("sub-1", "Alpha") - stack.pop("sub-9", "Ghost") - assert stack.active() == ["Alpha"] - - -def test_given_same_timestamp_push_and_pop_when_replaying_then_cancels(tmp_path): - """A short subagent stamps both ops in one tick on a coarse clock. - - Sorting the pop first would leave it unmatched and the agent active forever. - """ - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - same_ts = "2026-01-01T00:00:00+00:00" - for op in ("pop", "push"): - core.append_jsonl(stack.stack_file, {"ts": same_ts, "op": op, "id": "sub-1", "agent": "A"}) - assert stack.active() == [] - - -def test_given_name_only_surface_when_pushed_then_name_serves_as_identity(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - stack.push("", "Alpha") - stack.pop("", "Alpha") - assert stack.active() == [] - - -def test_given_active_agents_when_cleared_then_returns_to_root(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - stack.push("sub-1", "Alpha") - stack.push("sub-2", "Beta") - stack.clear() - assert stack.active() == [] - - -def test_given_subagent_pushed_when_build_entry_pretooluse_then_lists_candidates(tmp_path): - """Tool calls run in parallel, so an active subagent is only a candidate.""" - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - core.build_entry( - {"hook_event_name": "SubagentStart", "agent_name": "Coder"}, "SubagentStart", stack - ) - entry = core.build_entry( - {"hook_event_name": "PreToolUse", "tool_name": "read"}, "PreToolUse", stack - ) - assert "agent" not in entry - assert entry["agents"] == ["root", "Coder"] - - -def test_given_no_subagent_when_build_entry_pretooluse_then_credits_root(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - entry = core.build_entry( - {"hook_event_name": "PreToolUse", "tool_name": "read"}, "PreToolUse", stack - ) - assert entry["agent"] == "root" - assert "agents" not in entry - - -def test_given_overlapping_subagents_when_build_entry_then_declines_to_guess(tmp_path): - """No tool payload names its caller, so overlap must not credit one agent.""" - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - stack.push("sub-1", "Alpha") - stack.push("sub-2", "Beta") - entry = core.build_entry( - {"hook_event_name": "PostToolUse", "tool_name": "read", "tool_response": ""}, - "PostToolUse", - stack, - ) - assert "agent" not in entry - assert entry["agents"] == ["root", "Alpha", "Beta"] - - -def test_given_tool_use_id_when_build_entry_then_records_it(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - entry = core.build_entry( - {"hook_event_name": "PreToolUse", "tool_name": "read", "tool_use_id": "call-7"}, - "PreToolUse", - stack, - ) - assert entry["tool_use_id"] == "call-7" - - -def test_given_no_tool_use_id_when_build_entry_then_omits_it(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - entry = core.build_entry( - {"hook_event_name": "PreToolUse", "tool_name": "read"}, "PreToolUse", stack - ) - assert "tool_use_id" not in entry - - -def test_given_vscode_subagent_lifecycle_when_build_entry_then_stops_by_id(tmp_path): - """VS Code identifies a subagent by agent_id; agent_type is only its label.""" - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - for aid in ("sub-1", "sub-2"): - core.build_entry( - {"session_id": "sid1", "agent_id": aid, "agent_type": "Explore"}, - "SubagentStart", - stack, - ) - core.build_entry( - {"session_id": "sid1", "agent_id": "sub-1", "agent_type": "Explore"}, - "SubagentStop", - stack, - ) - assert stack.active() == ["Explore"] - - -def test_given_unknown_event_when_build_entry_then_returns_none(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - assert core.build_entry({"hook_event_name": "unknown"}, "unknown", stack) is None - - -def test_given_skill_path_when_build_entry_then_detects_skill(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - skill_file = tmp_path / "SKILL.md" - skill_file.write_text("x" * 40) - data = { - "hook_event_name": "PreToolUse", - "tool_name": "read", - "tool_input": {"filePath": "/repo/.github/skills/coll/my-skill/SKILL.md"}, - } - entry = core.build_entry(data, "PreToolUse", stack) - assert entry["skill"] == "my-skill" - - -def test_given_flat_skill_path_when_build_entry_then_detects_skill(tmp_path): - """VS Code ships skills as ``skills//SKILL.md``, with no collection dir.""" - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - data = { - "hook_event_name": "PreToolUse", - "tool_name": "read_file", - "tool_input": {"filePath": r"/repo/.github/skills/chronicle/SKILL.md"}, - } - entry = core.build_entry(data, "PreToolUse", stack) - assert entry["skill"] == "chronicle" - - -def test_given_cli_pretooluse_payload_when_normalize_event_then_infers_pretooluse(): - # The CLI omits the event name and sends a tool payload without toolResult. - assert ( - core._normalize_event({"sessionId": "s", "toolName": "bash", "toolArgs": "{}"}) - == "PreToolUse" - ) - - -def test_given_cli_posttooluse_payload_when_normalize_event_then_infers_posttooluse(): - data = {"sessionId": "s", "toolName": "bash", "toolArgs": "{}", "toolResult": {}} - assert core._normalize_event(data) == "PostToolUse" - - -def test_given_snake_case_tool_response_when_normalize_event_then_infers_posttooluse(): - # VS Code names the tool output tool_response; without alias-aware - # inference this payload is mistaken for a PreToolUse. - data = {"session_id": "s", "tool_name": "read", "tool_response": {}} - assert core._normalize_event(data) == "PostToolUse" - - -def test_given_snake_case_stop_reason_when_normalize_event_then_infers_subagent_stop(): - data = {"session_id": "s", "agent_name": "explore", "stop_reason": "end_turn"} - assert core._normalize_event(data) == "SubagentStop" - - -def test_given_tool_response_payload_when_build_entry_then_measures_length(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - data = { - "session_id": "sid1", - "tool_name": "read", - "tool_response": {"text_result_for_llm": "abcde"}, - } - entry = core.build_entry(data, "PostToolUse", stack) - assert entry["tool_response_len"] == 5 - - -def test_given_vscode_subagent_payload_when_build_entry_then_names_agent(tmp_path): - # VS Code labels the subagent with agent_type and identifies it by agent_id. - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - data = {"session_id": "sid1", "agent_id": "sub-1", "agent_type": "Plan"} - entry = core.build_entry(data, "SubagentStart", stack) - assert entry["agent_name"] == "Plan" - assert entry["agent_id"] == "sub-1" - assert stack.active() == ["Plan"] - - -def test_given_cli_prompt_payload_when_normalize_event_then_infers_userpromptsubmit(): - assert core._normalize_event({"sessionId": "s", "prompt": "hi"}) == "UserPromptSubmit" - - -def test_given_cli_subagent_start_payload_when_normalize_event_then_infers_start(): - data = {"sessionId": "s", "agentName": "explore", "agentDescription": "x"} - assert core._normalize_event(data) == "SubagentStart" - - -def test_given_cli_subagent_stop_payload_when_normalize_event_then_infers_stop(): - data = {"sessionId": "s", "agentName": "explore", "stopReason": "end_turn"} - assert core._normalize_event(data) == "SubagentStop" - - -def test_given_cli_agent_stop_payload_when_normalize_event_then_infers_stop(): - # An agent turn end carries stopReason but no agentName. - assert core._normalize_event({"sessionId": "s", "stopReason": "end_turn"}) == "Stop" - - -def test_given_cli_session_end_payload_when_normalize_event_then_infers_sessionend(): - # A real sessionEnd payload carries a top-level reason and no stopReason. - data = {"sessionId": "s", "reason": "user_exit"} - assert core._normalize_event(data) == "SessionEnd" - - -@pytest.mark.parametrize("reason", ["complete", "error", "abort", "timeout", "user_exit"]) -def test_given_documented_reason_value_when_normalize_event_then_infers_sessionend(reason): - assert core._normalize_event({"sessionId": "s", "reason": reason}) == "SessionEnd" - - -def test_given_unrelated_reason_value_when_normalize_event_then_stays_unknown(): - # An arbitrary reason on an unrecognized payload must not be mistaken for - # a session end; only the documented sessionEnd values match. - assert core._normalize_event({"sessionId": "s", "reason": "tool_denied"}) == "unknown" - - -def test_given_cli_session_start_payload_when_normalize_event_then_infers_sessionstart(): - assert core._normalize_event({"sessionId": "s", "source": "startup"}) == "SessionStart" - - -def test_given_vscode_session_end_payload_when_normalize_event_then_infers_sessionend(): - data = {"hook_event_name": "SessionEnd", "session_id": "s", "reason": "complete"} - assert core._normalize_event(data) == "SessionEnd" - - -def test_given_explicit_event_name_when_normalize_event_then_shape_inference_skipped(): - # VS Code supplies the event name, so inference must not override it. - data = {"hook_event_name": "SessionStart", "toolName": "bash"} - assert core._normalize_event(data) == "SessionStart" - - -def test_given_cli_json_string_toolargs_when_build_entry_then_extracts_keys(tmp_path): - # The CLI serializes tool arguments as a JSON string; build_entry decodes it. - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - data = { - "sessionId": "s", - "toolName": "bash", - "toolArgs": json.dumps({"command": "ls", "description": "list"}), - } - entry = core.build_entry(data, "PreToolUse", stack) - assert sorted(entry["tool_input_keys"]) == ["command", "description"] - - -def test_given_session_end_when_build_entry_then_records_reason(tmp_path): - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - data = {"sessionId": "sid1", "reason": "user_exit"} - entry = core.build_entry(data, "SessionEnd", stack) - assert entry["event"] == "SessionEnd" - assert entry["reason"] == "user_exit" - assert "stop_reason" not in entry - - -def test_given_stop_event_with_reason_only_when_build_entry_then_falls_back_to_reason(tmp_path): - # A surface may name the event Stop yet send a session-end style payload. - stack = core._AgentStack(tmp_path / ".stacks", "sid1") - data = {"sessionId": "sid1", "reason": "complete"} - entry = core.build_entry(data, "Stop", stack) - assert entry["stop_reason"] == "complete" - - -def test_given_session_end_event_when_mode_collect_then_writes_entry_and_summary( - tmp_path, monkeypatch -): - tel_dir = tmp_path / "tel" - home = tmp_path / "home" - state_dir = home / "session-state" / "sid1" - state_dir.mkdir(parents=True) - _write_jsonl( - state_dir / "events.jsonl", - [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"model": "m", "outputTokens": 9}, - } - ], - ) - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - monkeypatch.setenv("COPILOT_HOME", str(home)) - # A real CLI sessionEnd payload: top-level reason, no event name, no stopReason. - payload = { - "sessionId": "sid1", - "timestamp": 1753195629801, - "cwd": str(tmp_path), - "reason": "complete", - } - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - - events = _session_records(tel_dir) - assert events[0]["event"] == "SessionEnd" - assert events[0]["reason"] == "complete" - assert any(e["event"] == "SessionSummary" for e in events) - - -def test_given_stop_event_when_mode_collect_then_writes_entry_and_summary(tmp_path, monkeypatch): - tel_dir = tmp_path / "tel" - home = tmp_path / "home" - state_dir = home / "session-state" / "sid1" - state_dir.mkdir(parents=True) - _write_jsonl( - state_dir / "events.jsonl", - [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"model": "m", "outputTokens": 9}, - } - ], - ) - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - monkeypatch.setenv("COPILOT_HOME", str(home)) - payload = { - "hook_event_name": "Stop", - "session_id": "sid1", - "timestamp": "2026-01-02T00:00:00Z", - } - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - - events = _session_records(tel_dir) - assert events[0]["event"] == "Stop" - assert any(e["event"] == "SessionSummary" for e in events) - - -def test_given_precompact_event_when_mode_collect_then_writes_summary(tmp_path, monkeypatch): - tel_dir = tmp_path / "tel" - home = tmp_path / "home" - state_dir = home / "session-state" / "sid1" - state_dir.mkdir(parents=True) - _write_jsonl( - state_dir / "events.jsonl", - [ - { - "type": "assistant.message", - "timestamp": "2026-01-01T00:00:00Z", - "data": {"model": "m", "outputTokens": 9}, - } - ], - ) - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - monkeypatch.setenv("COPILOT_HOME", str(home)) - payload = { - "hook_event_name": "PreCompact", - "session_id": "sid1", - "timestamp": "2026-01-02T00:00:00Z", - } - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - - events = _session_records(tel_dir) - assert events[0]["event"] == "PreCompact" - # PreCompact captures a summary before process logs rotate. - assert any(e["event"] == "SessionSummary" for e in events) - - -def test_given_tool_pair_when_mode_collect_then_carries_tool_use_id(tmp_path, monkeypatch): - """The id survives to disk on both halves so offline analysis can pair them. - - The bundled report correlates by tool name, which cannot separate two - concurrent calls to the same tool; the recorded id can. - """ - tel_dir = tmp_path / "tel" - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - for payload in ( - { - "hook_event_name": "PreToolUse", - "session_id": "sid1", - "tool_name": "read_file", - "toolUseId": "call-42", - }, - { - "hook_event_name": "PostToolUse", - "session_id": "sid1", - "tool_name": "read_file", - "tool_use_id": "call-42", - }, - ): - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - - events = _session_records(tel_dir) - pairs = [(e["event"], e.get("tool_use_id")) for e in events if e["event"].endswith("ToolUse")] - assert pairs == [("PreToolUse", "call-42"), ("PostToolUse", "call-42")] - - -def test_given_traversal_sid_when_mode_collect_then_rejects(tmp_path, monkeypatch): - tel_dir = tmp_path / "tel" - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - payload = {"hook_event_name": "SessionStart", "session_id": "../escape"} - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - # No telemetry written for a rejected sid. - assert not tel_dir.exists() or not _session_files(tel_dir) - - -@pytest.mark.parametrize( - "sid", - ["", "..", "../escape", "a/b", "a\\b", "C:", "C:x", "\\\\server\\share", "//server/share"], -) -def test_given_unsafe_sid_when_is_safe_sid_then_rejects(sid): - """A drive or UNC anchor holds no separator yet still re-anchors a join.""" - assert not core._is_safe_sid(sid) - - -@pytest.mark.parametrize("sid", ["sid1", "abc-123", "a.b"]) -def test_given_ordinary_sid_when_is_safe_sid_then_accepts(sid): - assert core._is_safe_sid(sid) - - -def test_given_escaped_session_dir_when_clear_then_removes_nothing(tmp_path): - """Containment is re-checked at the point of deletion, not only at parse.""" - stack_dir = tmp_path / ".stacks" - outside = tmp_path / "outside" - outside.mkdir() - (outside / "keep.txt").write_text("user data", encoding="utf-8") - stack = core._AgentStack(stack_dir, "sid1") - stack.session_dir = outside - stack.clear() - assert (outside / "keep.txt").exists() - - -def test_given_unicode_line_when_append_line_then_writes_bytes_verbatim(tmp_path): - target = tmp_path / "sessions.jsonl" - core.append_line(target, "caf\u00e9 \u65e5\u672c\u8a9e\n") - core.append_line(target, "\u00fcber\n") - # No BOM, and no CRLF expansion from Windows text mode. - assert target.read_bytes() == "caf\u00e9 \u65e5\u672c\u8a9e\n\u00fcber\n".encode() - - -def test_given_missing_parent_when_append_jsonl_then_creates_it(tmp_path): - target = tmp_path / "nested" / "sessions.jsonl" - core.append_jsonl(target, {"event": "PreToolUse"}) - assert json.loads(target.read_text(encoding="utf-8")) == {"event": "PreToolUse"} - - -def test_given_append_when_written_then_leaves_no_sidecar(tmp_path): - """The lock lives on a byte of the log itself, so no lock file is needed.""" - target = tmp_path / "sessions.jsonl" - core.append_jsonl(target, {"event": "Stop"}) - assert list(tmp_path.iterdir()) == [target] - - -def test_given_refused_lock_when_appending_then_falls_back_to_private_shard(tmp_path, monkeypatch): - """A filesystem without working locks must not let writers interleave.""" - target = tmp_path / "sessions-2026-01-01.jsonl" - core.append_jsonl(target, {"event": "first"}) - monkeypatch.setattr(core, "_lock_append", lambda fd: False) - core.append_jsonl(target, {"event": "second"}) - - files = sorted(tmp_path.glob("*.jsonl")) - assert len(files) == 2 - # The shard is a sibling of its day log, so the report's session glob and the - # cleanup allow-list both reach it without a separate rule. - shard = next(p for p in files if p != target) - assert shard.name.startswith("sessions-2026-01-01.") - # The refused write went to that shard, leaving the shared log untouched. - assert json.loads(target.read_text(encoding="utf-8")) == {"event": "first"} - records = [json.loads(p.read_text(encoding="utf-8")) for p in files] - assert {r["event"] for r in records} == {"first", "second"} - - -def test_given_raising_shared_append_when_appending_then_still_shards(tmp_path, monkeypatch): - """The shard is reached when the shared write raises, not only when it returns False.""" - - def boom(target, payload): - raise OSError("no space left on device") - - monkeypatch.setattr(core, "_append_shared", boom) - target = tmp_path / "sessions-2026-01-01.jsonl" - core.append_jsonl(target, {"event": "rescued"}) - - files = sorted(tmp_path.glob("*.jsonl")) - assert len(files) == 1 - assert json.loads(files[0].read_text(encoding="utf-8")) == {"event": "rescued"} - - -def test_given_unwritable_store_when_appending_then_returns_quietly(tmp_path, monkeypatch): - """A hook that raises stalls the turn, so the last resort is to drop the record.""" - - def boom(*args, **kwargs): - raise OSError("permission denied") - - monkeypatch.setattr(core, "_append_shared", boom) - monkeypatch.setattr(core, "_append_private", boom) - core.append_jsonl(tmp_path / "sessions-2026-01-01.jsonl", {"event": "dropped"}) - assert list(tmp_path.glob("*.jsonl")) == [] - - -def test_given_unwritable_store_when_mode_collect_then_still_succeeds(tmp_path, monkeypatch): - """The engine reports success so the wrapper's continue contract holds.""" - tel_dir = tmp_path / "tel" - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - monkeypatch.setattr( - core.Path, "mkdir", lambda *a, **k: (_ for _ in ()).throw(OSError("read-only fs")) - ) - payload = {"hook_event_name": "PreToolUse", "session_id": "sid1", "tool_name": "read"} - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - - -def test_given_short_write_when_append_line_then_finishes_the_record(tmp_path, monkeypatch): - """A signal or a full disk can cut os.write short and truncate a record.""" - real_write = os.write - calls = [] - - def stingy(fd, data): - calls.append(bytes(data)) - # Drip one byte at a time so a single unchecked write would truncate. - return real_write(fd, bytes(data)[:1]) - - monkeypatch.setattr(os, "write", stingy) - target = tmp_path / "sessions.jsonl" - core.append_jsonl(target, {"event": "Stop"}) - assert len(calls) > 1 - assert json.loads(target.read_text(encoding="utf-8")) == {"event": "Stop"} - - -def test_given_repeated_calls_when_fallback_stem_then_never_repeats(): - """Two live collectors sharing a stem would share a file and race on it.""" - stems = {core._fallback_stem() for _ in range(200)} - assert len(stems) == 200 - - -def _run_collectors(payloads, tel_dir, home): - """Run one collector per payload; return their exit codes for the caller.""" - env = { - **os.environ, - "HVE_TELEMETRY_DIR": str(tel_dir), - "HVE_HOME": str(home), - "COPILOT_HOME": str(home), - } - procs = [ - subprocess.Popen( - [sys.executable, core.__file__, "collect"], - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - env=env, - ) - for _ in payloads - ] - try: - for proc, data in zip(procs, payloads): - proc.stdin.write(json.dumps(data).encode("utf-8")) - # Close in a tight loop so the collectors contend on the same write. - for proc in procs: - proc.stdin.close() - return [proc.wait(timeout=60) for proc in procs] - finally: - for proc in procs: - if proc.poll() is None: - proc.kill() - - -@pytest.mark.slow -def test_given_concurrent_collectors_when_appending_then_every_event_lands(tmp_path): - """End-to-end check that N collector processes each record their event. - - They share one log per day, so this is the regression guard for the - overwrite the Windows CRT allows: it emulates append as seek-to-end plus - write, letting two writers resolve the same offset. - """ - tel_dir = tmp_path / "tel" - count = 16 - payloads = [ - { - "hook_event_name": "PreToolUse", - "session_id": "race", - "cwd": str(tmp_path), - "tool_name": f"tool_{i}", - # Vary the record length so a lost race tears a line rather than - # cleanly replacing an equally sized record. - "tool_input": {"pad": "x" * (100 + i * 37)}, - } - for i in range(count) - ] - assert _run_collectors(payloads, tel_dir, tmp_path / "home") == [0] * len(payloads) - - records = _session_records(tel_dir) - assert len(records) == count - assert {r["tool"] for r in records} == {f"tool_{i}" for i in range(count)} - if os.name != "nt": - # POSIX resolves O_APPEND in the kernel, so the lock is never refused and - # a shard here would be a real regression. Windows can legitimately shard - # under contention, and readers pick it up either way. - assert len(_session_files(tel_dir)) == 1 - - -@pytest.mark.slow -def test_given_collectors_on_one_day_when_appending_then_shares_one_day_log(tmp_path): - """Records are partitioned by UTC day so cleanup and reports stay day-scoped.""" - tel_dir = tmp_path / "tel" - payloads = [ - { - "hook_event_name": "PreToolUse", - "session_id": "day", - "cwd": str(tmp_path), - "tool_name": f"tool_{i}", - } - for i in range(3) - ] - assert _run_collectors(payloads, tel_dir, tmp_path / "home") == [0] * len(payloads) - - files = _session_files(tel_dir) - assert files - assert all(re.fullmatch(r"sessions-\d{4}-\d{2}-\d{2}(\..+)?\.jsonl", p.name) for p in files) - assert len(_session_records(tel_dir)) == 3 - - -@pytest.mark.slow -def test_given_concurrent_subagent_starts_when_pushing_then_keeps_every_push(tmp_path): - """Parallel subagents each record a start from their own collector process. - - The push is an append rather than a read-modify-write, so a simultaneous - batch cannot lose all but one entry and misattribute later tool calls. - Every racing collector also registers, because each sees ``first_write``. - """ - tel_dir = tmp_path / "tel" - home = tmp_path / "home" - count = 16 - payloads = [ - { - "hook_event_name": "SubagentStart", - "session_id": "race", - "cwd": str(tmp_path), - "agent_type": f"Agent{i}", - } - for i in range(count) - ] - assert _run_collectors(payloads, tel_dir, home) == [0] * len(payloads) - - stack = core._AgentStack(tel_dir / ".stacks", "race") - assert sorted(stack.active()) == sorted(f"Agent{i}" for i in range(count)) - - # Registering is a racing whole-file write; it must converge on one entry. - assert core.read_registry_dirs(home / "telemetry-dirs") == [str(tel_dir.resolve())] - - -def test_given_existing_file_when_write_text_atomic_then_replaces_without_residue(tmp_path): - target = tmp_path / "launcher.ps1" - target.write_text("old content") - core._write_text_atomic(target, "new content") - assert target.read_text() == "new content" - assert list(tmp_path.iterdir()) == [target] - - -def test_given_session_log_when_clean_telemetry_dir_then_removes_it(tmp_path): - tel_dir = tmp_path / "tel" - tel_dir.mkdir() - log = tel_dir / "sessions-2026-01-01.jsonl" - log.write_text("{}\n", encoding="utf-8") - removed = [] - core.clean_telemetry_dir(tel_dir, dry_run=False, removed=removed) - assert not log.exists() - assert removed == [str(log)] - - -def test_given_fallback_shard_when_clean_telemetry_dir_then_removes_it(tmp_path): - """A shard sits beside its day log, so the allow-list must reach both.""" - tel_dir = tmp_path / "tel" - tel_dir.mkdir() - shard = tel_dir / "sessions-2026-01-01.010203000000-42-abcdef.jsonl" - core.append_jsonl(shard, {"event": "Stop"}) - removed = [] - core.clean_telemetry_dir(tel_dir, dry_run=False, removed=removed) - assert not shard.exists() - assert removed == [str(shard)] - - -def test_given_unrelated_jsonl_when_clean_telemetry_dir_then_preserves_it(tmp_path): - """A user can point HVE_TELEMETRY_DIR at a directory holding other work.""" - tel_dir = tmp_path / "tel" - tel_dir.mkdir() - keep_file = tel_dir / "sessions-notes.jsonl" - keep_file.write_text("{}\n", encoding="utf-8") - keep_dir = tel_dir / "notes" - keep_dir.mkdir() - (keep_dir / "a.jsonl").write_text("{}\n", encoding="utf-8") - removed = [] - core.clean_telemetry_dir(tel_dir, dry_run=False, removed=removed) - assert keep_file.exists() - assert keep_dir.exists() - assert removed == [] - - -def test_given_legacy_registry_file_when_read_registry_dirs_then_imports_once(tmp_path): - """An upgraded install must not silently lose its registered stores.""" - registry = tmp_path / "telemetry-dirs" - legacy = tmp_path / "telemetry-dirs.txt" - first = tmp_path / "a" / "tel" - second = tmp_path / "b" / "tel" - first.mkdir(parents=True) - second.mkdir(parents=True) - legacy.write_text(f"{first.resolve()}\n\n{second.resolve()}\n", encoding="utf-8") - - assert core.read_registry_dirs(registry) == sorted( - [str(first.resolve()), str(second.resolve())] - ) - # The legacy file is retired rather than deleted, and is not re-read. - assert not legacy.exists() - assert (tmp_path / "telemetry-dirs.txt.migrated").is_file() - assert core.read_registry_dirs(registry) == sorted( - [str(first.resolve()), str(second.resolve())] - ) - - -def test_given_new_dir_when_register_telemetry_dir_then_records_absolute_path(tmp_path): - registry = tmp_path / "telemetry-dirs" - tel_dir = tmp_path / "proj" / "tel" - tel_dir.mkdir(parents=True) - core.register_telemetry_dir(tel_dir, registry) - assert core.read_registry_dirs(registry) == [str(tel_dir.resolve())] - - -def test_given_existing_entry_when_register_telemetry_dir_then_dedups(tmp_path): - registry = tmp_path / "telemetry-dirs" - tel_dir = tmp_path / "tel" - tel_dir.mkdir() - core.register_telemetry_dir(tel_dir, registry) - core.register_telemetry_dir(tel_dir, registry) - assert core.read_registry_dirs(registry) == [str(tel_dir.resolve())] - - -def test_given_several_stores_when_registered_then_each_gets_its_own_marker(tmp_path): - registry = tmp_path / "telemetry-dirs" - first = tmp_path / "a" - second = tmp_path / "b" - first.mkdir() - second.mkdir() - core.register_telemetry_dir(first, registry) - core.register_telemetry_dir(second, registry) - core.register_telemetry_dir(first, registry) - assert core.read_registry_dirs(registry) == sorted( - [str(first.resolve()), str(second.resolve())] - ) - assert len(list(registry.glob("*.path"))) == 2 - - -def test_given_session_start_when_mode_collect_then_registers_dir(tmp_path, monkeypatch): - tel_dir = tmp_path / "tel" - home = tmp_path / "home" - hve = tmp_path / "hve" - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - monkeypatch.setenv("COPILOT_HOME", str(home)) - monkeypatch.setenv("HVE_HOME", str(hve)) - payload = {"hook_event_name": "SessionStart", "session_id": "sid1"} - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - assert core.read_registry_dirs(hve / "telemetry-dirs") == [str(tel_dir.resolve())] - # A cross-project launcher for the host platform is emitted in the HVE home. - if core._is_windows(): - assert (hve / "generate-report.ps1").is_file() - assert (hve / "clean-telemetry.ps1").is_file() - else: - assert (hve / "generate-report.sh").is_file() - assert (hve / "clean-telemetry.sh").is_file() - - -def test_given_no_session_start_when_mode_collect_then_still_registers_dir(tmp_path, monkeypatch): - tel_dir = tmp_path / "tel" - hve = tmp_path / "hve" - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - monkeypatch.setenv("COPILOT_HOME", str(tmp_path / "home")) - monkeypatch.setenv("HVE_HOME", str(hve)) - payload = {"hook_event_name": "UserPromptSubmit", "session_id": "sid1", "prompt": "hi"} - monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload))) - assert core._mode_collect() == 0 - assert core.read_registry_dirs(hve / "telemetry-dirs") == [str(tel_dir.resolve())] - - -def test_given_stale_entries_when_mode_list_dirs_then_prunes_and_prints( - tmp_path, monkeypatch, capsys -): - hve = tmp_path / "hve" - hve.mkdir() - live = tmp_path / "live" - live.mkdir() - dead = tmp_path / "dead" - registry = hve / "telemetry-dirs" - core.register_telemetry_dir(live, registry) - core.register_telemetry_dir(dead, registry) - monkeypatch.setenv("HVE_HOME", str(hve)) - assert core._mode_list_dirs() == 0 - out = capsys.readouterr().out.splitlines() - assert out == [str(live.resolve())] - # Dead entry is pruned from the registry on read. - assert core.read_registry_dirs(registry) == [str(live.resolve())] - - -def test_given_posix_when_write_report_launchers_then_writes_sh_only(tmp_path, monkeypatch): - hve = tmp_path / "hve" - script_dir = tmp_path / "hook" - script_dir.mkdir() - monkeypatch.setenv("HVE_HOME", str(hve)) - monkeypatch.setattr(core, "_is_windows", lambda: False) - core.write_report_launchers(script_dir) - report_script = str(script_dir / "generate-telemetry-report.sh") - out_path = str(hve / "report.generated.html") - sh = (hve / "generate-report.sh").read_text(encoding="utf-8") - assert report_script in sh - assert out_path in sh - assert "--all-dirs" in sh - # No PowerShell launcher on POSIX. - assert not (hve / "generate-report.ps1").exists() - - -def test_given_windows_when_write_report_launchers_then_writes_ps1_only(tmp_path, monkeypatch): - hve = tmp_path / "hve" - script_dir = tmp_path / "hook" - script_dir.mkdir() - monkeypatch.setenv("HVE_HOME", str(hve)) - monkeypatch.setattr(core, "_is_windows", lambda: True) - core.write_report_launchers(script_dir) - report_ps1 = str(script_dir / "Invoke-TelemetryReport.ps1") - out_path = str(hve / "report.generated.html") - ps = (hve / "generate-report.ps1").read_text(encoding="utf-8") - # Native delegation to the PowerShell generator, no bash. - assert report_ps1 in ps - assert out_path in ps - assert "-AllDirs" in ps - assert "bash" not in ps - # No POSIX launcher on Windows. - assert not (hve / "generate-report.sh").exists() - - -def test_given_posix_when_write_report_launchers_then_clean_sh_delegates_to_bash( - tmp_path, monkeypatch -): - hve = tmp_path / "hve" - script_dir = tmp_path / "hook" - script_dir.mkdir() - monkeypatch.setenv("HVE_HOME", str(hve)) - monkeypatch.setattr(core, "_is_windows", lambda: False) - core.write_report_launchers(script_dir) - clean_script = str(script_dir / "clean-telemetry.sh") - sh = (hve / "clean-telemetry.sh").read_text(encoding="utf-8") - assert clean_script in sh - assert "--all-dirs" in sh - assert not (hve / "clean-telemetry.ps1").exists() - - -def test_given_windows_when_write_report_launchers_then_clean_ps1_is_native(tmp_path, monkeypatch): - hve = tmp_path / "hve" - script_dir = tmp_path / "hook" - script_dir.mkdir() - monkeypatch.setenv("HVE_HOME", str(hve)) - monkeypatch.setattr(core, "_is_windows", lambda: True) - core.write_report_launchers(script_dir) - clean_ps1 = str(script_dir / "Invoke-TelemetryClean.ps1") - ps = (hve / "clean-telemetry.ps1").read_text(encoding="utf-8") - # Native delegation to the PowerShell wrapper, no bash. - assert clean_ps1 in ps - assert "-AllDirs" in ps - assert "bash" not in ps - assert not (hve / "clean-telemetry.sh").exists() - - -def _seed_telemetry_store(tel_dir): - """Populate a telemetry directory with representative artifacts plus an - unrelated file that cleanup must preserve.""" - tel_dir.mkdir(parents=True) - core.append_jsonl(tel_dir / "sessions-2026-01-01.jsonl", {}) - core.append_jsonl(tel_dir / "sessions-2026-01-02.jsonl", {}) - (tel_dir / "raw-input.jsonl").write_text("{}\n", encoding="utf-8") - (tel_dir / "report.generated.html").write_text("", encoding="utf-8") - stacks = tel_dir / ".stacks" / "sid1" - stacks.mkdir(parents=True) - (stacks / "010203000000-42-aaaaaa.log").write_text( - '{"op": "push", "agent": "A"}\n', encoding="utf-8" - ) - keep = tel_dir / "keep-me.txt" - keep.write_text("user data", encoding="utf-8") - return keep - - -def test_given_store_when_clean_telemetry_dir_then_removes_only_artifacts(tmp_path): - tel_dir = tmp_path / "tel" - keep = _seed_telemetry_store(tel_dir) - removed = [] - core.clean_telemetry_dir(tel_dir, dry_run=False, removed=removed) - assert not (tel_dir / "sessions-2026-01-01.jsonl").exists() - assert not (tel_dir / "sessions-2026-01-02.jsonl").exists() - assert not (tel_dir / "raw-input.jsonl").exists() - assert not (tel_dir / "report.generated.html").exists() - assert not (tel_dir / ".stacks").exists() - # Unrelated files are preserved. - assert keep.exists() - assert len(removed) == 5 - - -def test_given_dry_run_when_clean_telemetry_dir_then_reports_without_deleting(tmp_path): - tel_dir = tmp_path / "tel" - _seed_telemetry_store(tel_dir) - removed = [] - core.clean_telemetry_dir(tel_dir, dry_run=True, removed=removed) - assert (tel_dir / "raw-input.jsonl").exists() - assert (tel_dir / ".stacks").exists() - assert (tel_dir / "sessions-2026-01-01.jsonl").exists() - assert len(removed) == 5 - - -def test_given_current_store_when_mode_clean_then_cleans_only_current(tmp_path, monkeypatch): - current = tmp_path / "current" - other = tmp_path / "other" - hve = tmp_path / "hve" - _seed_telemetry_store(current) - _seed_telemetry_store(other) - hve.mkdir() - registry = hve / "telemetry-dirs" - core.register_telemetry_dir(other, registry) - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(current)) - monkeypatch.setenv("HVE_HOME", str(hve)) - assert core._mode_clean(all_dirs=False, dry_run=False) == 0 - assert not (current / "raw-input.jsonl").exists() - # Without --all-dirs, registered stores and the registry are untouched. - assert (other / "raw-input.jsonl").exists() - assert registry.exists() - - -def test_given_all_dirs_when_mode_clean_then_cleans_registry_and_home(tmp_path, monkeypatch): - current = tmp_path / "current" - other = tmp_path / "other" - hve = tmp_path / "hve" - _seed_telemetry_store(current) - _seed_telemetry_store(other) - hve.mkdir() - registry = hve / "telemetry-dirs" - core.register_telemetry_dir(other, registry) - (hve / "report.generated.html").write_text("", encoding="utf-8") - (hve / "generate-report.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(current)) - monkeypatch.setenv("HVE_HOME", str(hve)) - assert core._mode_clean(all_dirs=True, dry_run=False) == 0 - assert not (current / "raw-input.jsonl").exists() - assert not (other / "raw-input.jsonl").exists() - assert not registry.exists() - assert not (hve / "report.generated.html").exists() - assert not (hve / "generate-report.sh").exists() - - -def test_given_clean_mode_when_main_dispatches_then_parses_flags(tmp_path, monkeypatch): - current = tmp_path / "current" - _seed_telemetry_store(current) - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(current)) - assert core.main(["clean", "--dry-run"]) == 0 - # Dry-run leaves artifacts in place. - assert (current / "raw-input.jsonl").exists() - - -class _ByteStdin: - """Stdin stand-in exposing a raw byte buffer, as the real stream does. - - Feeding bytes rather than str exercises the decode boundary where a host - locale would otherwise corrupt or reject the payload. - """ - - def __init__(self, data: bytes) -> None: - self.buffer = io.BytesIO(data) - - -def _collect_one(payload_bytes, tel_dir, monkeypatch): - """Run a single collect pass over raw stdin bytes and return the entries.""" - monkeypatch.setenv("HVE_TELEMETRY_DIR", str(tel_dir)) - monkeypatch.setattr("sys.stdin", _ByteStdin(payload_bytes)) - assert core._mode_collect() == 0 - return _session_records(tel_dir) - - -def test_given_non_ascii_utf8_stdin_bytes_when_mode_collect_then_preserves_text( - tmp_path, monkeypatch -): - payload = { - "hook_event_name": "UserPromptSubmit", - "session_id": "sid1", - "timestamp": "2026-01-02T00:00:00Z", - "cwd": "/home/M\u00fcller/\u30d7\u30ed\u30b8\u30a7\u30af\u30c8", - "prompt": "caf\u00e9 \u65e5\u672c\u8a9e", - } - events = _collect_one( - # ensure_ascii=False emits real multi-byte UTF-8 rather than \u escapes, - # which is what actually crosses the decode boundary at runtime. - json.dumps(payload, ensure_ascii=False).encode("utf-8"), - tmp_path / "tel", - monkeypatch, - ) - # cwd is a grouping key for cross-project reporting, so it must round-trip - # byte-for-byte rather than degrade to replacement characters. - assert events[0]["cwd"] == "/home/M\u00fcller/\u30d7\u30ed\u30b8\u30a7\u30af\u30c8" - assert events[0]["prompt"] == "caf\u00e9 \u65e5\u672c\u8a9e" - - -def test_given_invalid_utf8_stdin_bytes_when_mode_collect_then_still_records_event( - tmp_path, monkeypatch -): - # A lone 0xE9 is valid cp1252 but invalid UTF-8. UnicodeDecodeError - # subclasses ValueError, so an unguarded decode would drop the event as - # though the JSON were malformed. - raw = b'{"hook_event_name": "UserPromptSubmit", "session_id": "sid1", "prompt": "caf\xe9"}' - events = _collect_one(raw, tmp_path / "tel", monkeypatch) - assert events[0]["event"] == "UserPromptSubmit" - assert events[0]["prompt"] == "caf\ufffd" - - -def test_given_stdin_without_buffer_when_read_stdin_text_then_falls_back_to_read(monkeypatch): - monkeypatch.setattr("sys.stdin", io.StringIO("plain text")) - assert core._read_stdin_text() == "plain text" diff --git a/.github/hooks/shared/telemetry/uv.lock b/.github/hooks/shared/telemetry/uv.lock deleted file mode 100644 index 4a435967a..000000000 --- a/.github/hooks/shared/telemetry/uv.lock +++ /dev/null @@ -1,297 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.11" - -[[package]] -name = "atheris" -version = "3.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/58/5965955898e16bee17c8379eae12194993bf641c4629016991248b862069/atheris-3.0.0.tar.gz", hash = "sha256:1f0929c7bc3040f3fe4102e557718734190cf2d7718bbb8e3ce6d3eb56ef5bb3", size = 373239, upload-time = "2025-11-24T23:54:02.15Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/15/cf109e2e8696a54c8c4bc3ef79a79bec32361eceb64eaa36690a682e83a9/atheris-3.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8a5c8a781467c187da40fd29139784193e2647058831f837f675d0bb8cbd8746", size = 34805555, upload-time = "2025-11-24T23:53:53.477Z" }, - { url = "https://files.pythonhosted.org/packages/85/8c/e9960b996e70e5f6a523670431166b2b238de52fef094955515dcf854da1/atheris-3.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:510e502c57b6dc615fb174066407af620d4c7f73cf08a782c86e7761bf12c4eb", size = 34907016, upload-time = "2025-11-24T23:53:56.535Z" }, - { url = "https://files.pythonhosted.org/packages/db/48/df670f75f458cc7c1752a01a394fd59c830b08172dd59cf29d73f31050f9/atheris-3.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac", size = 34858525, upload-time = "2025-11-24T23:53:59.925Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "coverage" -version = "7.14.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/d7/477ad149490e6cb849f28abea1dabb9c823cea72e7500c81b4240ce619c0/coverage-7.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:478b5bcd63c2e1357c5c7e16c070690df7b07f676b1c114d7b93e533c664309f", size = 219848, upload-time = "2026-05-26T20:38:38.715Z" }, - { url = "https://files.pythonhosted.org/packages/91/82/a5eb47257c50601bb7b9a9d2857c67b7a3a85ad74180eb2c98bb1fbe0ce5/coverage-7.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a24a81f9715ee42ef59a316cc11611c98fe23920f7c81861315c9f3ff4a230f4", size = 220354, upload-time = "2026-05-26T20:38:40.232Z" }, - { url = "https://files.pythonhosted.org/packages/43/8b/78419b5391a5cb706b6544390507e469d83ffc9a8248b02c4011aceb9365/coverage-7.14.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:196a13319ad88d6d8ef5ab489ec4f44ddde2143c0c7d5b27786f6c3ffd56a7e1", size = 250771, upload-time = "2026-05-26T20:38:41.782Z" }, - { url = "https://files.pythonhosted.org/packages/77/63/e77aaacd491182210d639636b7a8bba23ffffa9b82aa3762da9431855fa9/coverage-7.14.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d452fd08b5c72c5167c93e6867b5c08500bd40f2a21e1e854a500550b6cc36f", size = 252683, upload-time = "2026-05-26T20:38:43.305Z" }, - { url = "https://files.pythonhosted.org/packages/65/1c/a022e3cfbec2ac241640003cb3a817e161d9c7f5aa9b49173756cdc03204/coverage-7.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23bf7fa51ac02e07fc7c96849b82946da47ae862dc8f86d183b2a4864fc38129", size = 254791, upload-time = "2026-05-26T20:38:45.361Z" }, - { url = "https://files.pythonhosted.org/packages/61/d6/967e408aca4c1ceb88cb0cc677169110ae7f5995fb5eaf5fb1f5a1bb8f5d/coverage-7.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcaa50684dcaadfa599ac48f81103c756d791cfd85c97203d2217c593d48b860", size = 256748, upload-time = "2026-05-26T20:38:46.91Z" }, - { url = "https://files.pythonhosted.org/packages/b8/be/869188f7fe28638078ec479331ace6dc5f7b40b7153eb616f47ab79404d8/coverage-7.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4ea1c034f95c9b056e856b794630b17f9fa3d57e4800ff1e503d3be0f9c9078c", size = 250907, upload-time = "2026-05-26T20:38:48.493Z" }, - { url = "https://files.pythonhosted.org/packages/07/aa/adb7d3b4278d690e68703abcd76ab1b948242e3668d921711551b78f9ddb/coverage-7.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e057326434e441306226fbeb5d1aaf14a2637efe97ba668306635835f32ad7", size = 252483, upload-time = "2026-05-26T20:38:50.074Z" }, - { url = "https://files.pythonhosted.org/packages/43/61/331c74103c62dcb0c4b9b3a0de9a61aca016208b0a90f109592a9f9ecc28/coverage-7.14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:59baf88468dbc8d63b1887afd92bda52e40bb1561696e5819670601403810cec", size = 250545, upload-time = "2026-05-26T20:38:51.613Z" }, - { url = "https://files.pythonhosted.org/packages/f6/b6/c5dae3c104d89be04828f61810e6b3473825482e4c288cc4ed04553e08ae/coverage-7.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d34d75f892b3ab73ba11cab5442cce7b3e168fd64162b16f0e1e0d09c508edef", size = 254310, upload-time = "2026-05-26T20:38:53.503Z" }, - { url = "https://files.pythonhosted.org/packages/ad/a1/2b9d5863e3b83c01ad8199e3c597802fbb3a9dc90b058885804c20296d31/coverage-7.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3a56abc20a472baf0304c455721bc601477440d28ecfde8a03dde79ede07e0df", size = 250266, upload-time = "2026-05-26T20:38:55.414Z" }, - { url = "https://files.pythonhosted.org/packages/7f/5e/0e511fbdb269359be26fe678a1c3fa1f2aa2a01573cc3f54268c8d6d4797/coverage-7.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a3cb83d1552c0cd1b4906655b6a33fd4a8473229633a901c6b73bf86914dee9", size = 251174, upload-time = "2026-05-26T20:38:57.141Z" }, - { url = "https://files.pythonhosted.org/packages/85/10/e55307b622b3dd9671cb321824502dc10f93e72f2802b9946159a8edadeb/coverage-7.14.1-cp311-cp311-win32.whl", hash = "sha256:10274a1fbeb8ec5d72966e17bb198a3104257aca4ac09d98667c5f8aca8c8548", size = 222354, upload-time = "2026-05-26T20:38:58.727Z" }, - { url = "https://files.pythonhosted.org/packages/71/cf/107421693cfb71e4f1ca5bf70443f64d4161878068d07a3e51c7ad21d17b/coverage-7.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:87ebdf787d4888e3f3f2d523eadc6e18c6d18c6d0eb173801a189641627fb37e", size = 223290, upload-time = "2026-05-26T20:39:00.413Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1d/3e3644585eb29e9dafefb19555078529a4d7cce12bd21929664eea989277/coverage-7.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:dd34767fa19848d35659ffc0a75314f58c7af3f1cd87ec521e8292a1238398a3", size = 221953, upload-time = "2026-05-26T20:39:02.159Z" }, - { url = "https://files.pythonhosted.org/packages/3d/b7/bdbb725ba02c5b42825b200c940f38b7a54fcad24627b7192f78f8110d76/coverage-7.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a06c76364a9360e33d6d23769aefdf7f66f38e2ffb60ceb1baaa4989d83b695c", size = 220022, upload-time = "2026-05-26T20:39:03.702Z" }, - { url = "https://files.pythonhosted.org/packages/72/81/fdc0898a55c6219223291ec1a1fe89966ef212ce82276aa0899df84b5de0/coverage-7.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fad54e871165f6ec2f536063ac74c3104508a12963e64072ba44bd822de52b0c", size = 220379, upload-time = "2026-05-26T20:39:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/de/72/de048c4a25e13bce59ac6a339351c10bdf2515e07459afcdaf04dc3143a2/coverage-7.14.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:84b535f00655ecafe1d929d1fb00ed5d6fa3051ea643ab2c161a3887b86f294b", size = 251888, upload-time = "2026-05-26T20:39:07.367Z" }, - { url = "https://files.pythonhosted.org/packages/28/30/300c343f68beb9d4cbb64ec81e58c5b6b80b56927f72d2b38654ac26e013/coverage-7.14.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6b6b0853b895fe0e98cbfc580d1ec3393d9302b4b1e96a77b3f5c91fdab899e6", size = 254624, upload-time = "2026-05-26T20:39:09.037Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ed/7b25642496e8170b6bac14adce00537c6e5fa2d586159401a4de3e8b49e6/coverage-7.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:442cc9c952b2df400cda54bb04ab87330cf2cd08a8692cbbea36773531eb6f37", size = 255739, upload-time = "2026-05-26T20:39:10.889Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a2/abd210b8c4e29c24e4624916db97bb519097a91034aaeb767f937e7da794/coverage-7.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8270544c361ed405a27a060dbc9ed2c124b084d96dfdc2d9a2510482aef981ad", size = 257998, upload-time = "2026-05-26T20:39:12.722Z" }, - { url = "https://files.pythonhosted.org/packages/7f/24/7c50beed3792fe62f6ce0545c6686ce83379719e2c0276179333d97eae92/coverage-7.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:48b283b1dd6372e8de2a7a9a4c4d5dc06f4d4fd209b876f3c88a7a205a0c8f84", size = 252296, upload-time = "2026-05-26T20:39:14.259Z" }, - { url = "https://files.pythonhosted.org/packages/15/05/0f874628ebcbfc77ead559ff210281ef06a97db08481832e7dd39274a135/coverage-7.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5b0c99ba93a07d56f6df340bb79be53202a082b2fdb81bfe6190b741a3470d54", size = 253658, upload-time = "2026-05-26T20:39:15.923Z" }, - { url = "https://files.pythonhosted.org/packages/99/6f/ca6ad067364b337ef997802115e7ecad2abd2248b05471464b0dea02b4d4/coverage-7.14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e471bc5769ff073b058cfadb0d736b56ce067c8560eabeb0da88462df98c23e7", size = 251803, upload-time = "2026-05-26T20:39:17.537Z" }, - { url = "https://files.pythonhosted.org/packages/c0/30/b9b4d377cd9f40baf228068f5a81faf8450c6228503011bd499708483a50/coverage-7.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f497a1ea81d4cd7c10ddcaa685135b9aabd291af3d55775a9ddf3cb7a364cdd9", size = 255873, upload-time = "2026-05-26T20:39:19.414Z" }, - { url = "https://files.pythonhosted.org/packages/3c/21/7c721a9e5e6bb88547d30a787aefb97512d3f54c1324c7488d9b3743f7f9/coverage-7.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2222be86d0b54f5dd5a38f45f17f315f737245e857bf0bdedc70734f84a13c02", size = 251372, upload-time = "2026-05-26T20:39:21.169Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f8ae5a2200130e1503cd7661a6cd3b2b7bacef98277fbf3571fb13f8b766/coverage-7.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85e85586565842f6932abebd4c18bcb1074223dc0b3576e7d173ca710622813a", size = 253245, upload-time = "2026-05-26T20:39:23.097Z" }, - { url = "https://files.pythonhosted.org/packages/34/62/70a9024672a5f6910517d9628c52c9afbdd3cf8f46426af52bb148a56fff/coverage-7.14.1-cp312-cp312-win32.whl", hash = "sha256:4a28fd227808366b196a75476dced2eb35b351d6766ba9c858dc93319e87f4f1", size = 222567, upload-time = "2026-05-26T20:39:24.868Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/8b7cd386839b039ebe1855733b9f9449a8dec5d79564018234f185a7fa70/coverage-7.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:54acdb6674a4661768d7bf7db32dfb9f46ab1d764f8aba6df75ce1a6a088724e", size = 223372, upload-time = "2026-05-26T20:39:26.603Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ba/b44d472022f620d289d95fa830143235c0c36461c6f2437ea8d51e5481ed/coverage-7.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:99cd41ff91afd94896fea3bc002706b6ae4ce95727d06e4a0f39c0a8d8bd8b1a", size = 221989, upload-time = "2026-05-26T20:39:28.242Z" }, - { url = "https://files.pythonhosted.org/packages/8a/9e/5f6d56327c62b185225d145191c607e07515294a0aa6338e58805cd4a5ac/coverage-7.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793", size = 220044, upload-time = "2026-05-26T20:39:29.902Z" }, - { url = "https://files.pythonhosted.org/packages/75/92/e82aca356744cbbc0f77a0b623e38918c1872361963413a3bab5d0340393/coverage-7.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d", size = 220412, upload-time = "2026-05-26T20:39:31.561Z" }, - { url = "https://files.pythonhosted.org/packages/27/c9/385bde0bf7ed0f4bf3a7ee5367060a86b5d218718cfd6fb943c0f836b34f/coverage-7.14.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247", size = 251412, upload-time = "2026-05-26T20:39:33.337Z" }, - { url = "https://files.pythonhosted.org/packages/51/8c/23faf6a2343a0d17f960a4bd56c43bc7eb4cf312f774dd6ceebd82c7d8fc/coverage-7.14.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d", size = 254008, upload-time = "2026-05-26T20:39:35.009Z" }, - { url = "https://files.pythonhosted.org/packages/42/06/36f4aa9ca8a815e6036156e80706a67828bb97bd826948244f6996dda957/coverage-7.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b", size = 255241, upload-time = "2026-05-26T20:39:36.71Z" }, - { url = "https://files.pythonhosted.org/packages/ca/79/95266316352f90f6b1c6736bb413302edfde2453fb32422d3911642691b3/coverage-7.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be", size = 257373, upload-time = "2026-05-26T20:39:38.412Z" }, - { url = "https://files.pythonhosted.org/packages/e3/9c/58316d1f66c488b5fca8a0eb3e98348807813efa8a0d0833b9021be27488/coverage-7.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43", size = 251635, upload-time = "2026-05-26T20:39:40.268Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5a/ca2398a568e16fed7bb713e84ba3603a7164fb65779abe645c565ec890d5/coverage-7.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901", size = 253373, upload-time = "2026-05-26T20:39:42.145Z" }, - { url = "https://files.pythonhosted.org/packages/6e/2c/0396562c32deaebe7be51d865b3a41e9a87d7561acafe1a28f53b07e019a/coverage-7.14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff", size = 251341, upload-time = "2026-05-26T20:39:43.907Z" }, - { url = "https://files.pythonhosted.org/packages/fd/8f/a94f9221184c9cae1ee115820e3798e48b6b17777a9f19e46fb9a0c8dc74/coverage-7.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4", size = 255497, upload-time = "2026-05-26T20:39:46.166Z" }, - { url = "https://files.pythonhosted.org/packages/71/69/505d70e47db1eaebcd002c39759707621ef184cd6b1ae084d9f41293f323/coverage-7.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d", size = 251159, upload-time = "2026-05-26T20:39:48.03Z" }, - { url = "https://files.pythonhosted.org/packages/e0/aa/58681c383aa33a9d2ed40a02d7a22fbf780d1fa4d575396365777828198c/coverage-7.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33", size = 252934, upload-time = "2026-05-26T20:39:49.872Z" }, - { url = "https://files.pythonhosted.org/packages/eb/fd/11c928cd6bdffc7074bb5965c173d9ebf517fb00205e1da524b98d29ef92/coverage-7.14.1-cp313-cp313-win32.whl", hash = "sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c", size = 222584, upload-time = "2026-05-26T20:39:51.68Z" }, - { url = "https://files.pythonhosted.org/packages/6f/92/fb416fc26d340dcba19518c418d6048e913186e17243982c5e435e41fa7a/coverage-7.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416", size = 223394, upload-time = "2026-05-26T20:39:53.472Z" }, - { url = "https://files.pythonhosted.org/packages/73/c6/02d56e3867972f77d5036de924643f26c056e848f00452cafb4dbc3c29b4/coverage-7.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42", size = 222015, upload-time = "2026-05-26T20:39:55.374Z" }, - { url = "https://files.pythonhosted.org/packages/4d/9e/fcc77914050df73f7662fa1f00902774c79c075a8388ab334074574bf77e/coverage-7.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d", size = 220733, upload-time = "2026-05-26T20:39:57.189Z" }, - { url = "https://files.pythonhosted.org/packages/f7/67/2963cbdaf5cbadec44efa3a1e39eaa1f02df4079585f05387607a221e126/coverage-7.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5", size = 221086, upload-time = "2026-05-26T20:39:59.019Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c5/8701645574e11881f2f47d8930f98bc48b5d43b25eb5b4430dfc4a2f9f48/coverage-7.14.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52", size = 262381, upload-time = "2026-05-26T20:40:00.822Z" }, - { url = "https://files.pythonhosted.org/packages/7c/28/7a64d73598263e0c5abd5084211a8474488d31b3c552ff531c719dfcff62/coverage-7.14.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a", size = 264458, upload-time = "2026-05-26T20:40:02.506Z" }, - { url = "https://files.pythonhosted.org/packages/fa/d8/4969179db9f7eb4df218e69540adf829d1c835f59452513d065d15446802/coverage-7.14.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a", size = 266884, upload-time = "2026-05-26T20:40:04.421Z" }, - { url = "https://files.pythonhosted.org/packages/a6/78/a45d5794dbc9bafd97afc96a4377c86c7820d78b6cf51b89bc1d4e919275/coverage-7.14.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2", size = 268022, upload-time = "2026-05-26T20:40:06.298Z" }, - { url = "https://files.pythonhosted.org/packages/21/cb/4f5e354e9e3e67af96bd4e57113e6db6b22298c7168b13eec408a549903d/coverage-7.14.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e", size = 261631, upload-time = "2026-05-26T20:40:08.226Z" }, - { url = "https://files.pythonhosted.org/packages/ec/49/eced49af4cb996d5d8b7e94e736175c513e4facd3398507b89892b4326d8/coverage-7.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d", size = 264443, upload-time = "2026-05-26T20:40:10.137Z" }, - { url = "https://files.pythonhosted.org/packages/f1/d8/5603a88a7c5913a6b54f6cb1a8c46f7b39cbb30f27cd3f492908da09b2d7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb", size = 262069, upload-time = "2026-05-26T20:40:11.999Z" }, - { url = "https://files.pythonhosted.org/packages/f0/59/2ae3cb79da554a06c8619d6c88ea19dd1e4aed4b834b6a83bb1fa243bdc5/coverage-7.14.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d", size = 265780, upload-time = "2026-05-26T20:40:13.858Z" }, - { url = "https://files.pythonhosted.org/packages/af/5f/b130c1dc999031f2648bd25317fbce505ad8d5562079b4ed81e736a84967/coverage-7.14.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69", size = 260970, upload-time = "2026-05-26T20:40:16.142Z" }, - { url = "https://files.pythonhosted.org/packages/87/d1/ec13ccddeb48ec963bdfa72a11224bac2584bd045ba13beca82f8113e9c7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54", size = 263157, upload-time = "2026-05-26T20:40:18.382Z" }, - { url = "https://files.pythonhosted.org/packages/cf/c2/cd91ead503045161092d3845f7bb95ea2f25131ce96d3e314dd835d91b9c/coverage-7.14.1-cp313-cp313t-win32.whl", hash = "sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1", size = 223259, upload-time = "2026-05-26T20:40:20.381Z" }, - { url = "https://files.pythonhosted.org/packages/71/9f/1e28d97e6bd2c76b07f38b7c02870f1371255ff6717f54eca578fcbbdd0e/coverage-7.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce", size = 224320, upload-time = "2026-05-26T20:40:22.316Z" }, - { url = "https://files.pythonhosted.org/packages/a9/e0/d936e908f0e1efa55e52b91e01b52f1055cef5e1ab2718493390ed8e2fb8/coverage-7.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1", size = 222577, upload-time = "2026-05-26T20:40:24.894Z" }, - { url = "https://files.pythonhosted.org/packages/d6/34/fc2f101b151af3799a101f0550b0454aa008afdc0add677394ec4aa8ea10/coverage-7.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee", size = 220091, upload-time = "2026-05-26T20:40:27.249Z" }, - { url = "https://files.pythonhosted.org/packages/3d/a7/1ebae2ab5b961b5c79bb09fe7b3ac99edb190d8be4a8c510b2cf66f46468/coverage-7.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500", size = 220421, upload-time = "2026-05-26T20:40:30.084Z" }, - { url = "https://files.pythonhosted.org/packages/5e/90/92aca9cf0acc95123c96cd1eb1f08917897a7f5dee01e15738922971ec31/coverage-7.14.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906", size = 251466, upload-time = "2026-05-26T20:40:32.542Z" }, - { url = "https://files.pythonhosted.org/packages/26/2b/78048cbe3b999f6cbf9cc0d90abba6a88a3e0863a8c1c6cbc762f3f8802f/coverage-7.14.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42", size = 253973, upload-time = "2026-05-26T20:40:34.473Z" }, - { url = "https://files.pythonhosted.org/packages/8e/21/c2e33b29d1cfde484a19d437afc343c6cd30b08d78cbbf9f5aff14e57b2b/coverage-7.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8", size = 255318, upload-time = "2026-05-26T20:40:38.154Z" }, - { url = "https://files.pythonhosted.org/packages/8e/ee/aad2f108d63b769121005302f16bf66db8625c88ceaba466942e09a2607e/coverage-7.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851", size = 257633, upload-time = "2026-05-26T20:40:40.164Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f8/11a2c29b4fd76d9849f81d0bb812ec0017a9396df3217214e38934a8c837/coverage-7.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034", size = 251488, upload-time = "2026-05-26T20:40:42.631Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b8/9a5820de4b8ac2b71d85e3b5fb49108d7469c665f0e2ad0dd7569023e305/coverage-7.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c", size = 253329, upload-time = "2026-05-26T20:40:45.208Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ff/f33e4823667e27548e8fd8df44217515303f9808d0ff29817db56f87d990/coverage-7.14.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36", size = 251291, upload-time = "2026-05-26T20:40:47.502Z" }, - { url = "https://files.pythonhosted.org/packages/68/9b/489db0ebb209054766b90a9014a45f6d26eb724c02ec21311c3733b5a644/coverage-7.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5", size = 255564, upload-time = "2026-05-26T20:40:49.372Z" }, - { url = "https://files.pythonhosted.org/packages/27/b5/16bc2d4c2409b23c7737edb68c83bc89e345f378050549fe1d75ac7d34d5/coverage-7.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4", size = 251107, upload-time = "2026-05-26T20:40:51.677Z" }, - { url = "https://files.pythonhosted.org/packages/7d/0c/2629997469a00cd069d588a41c9dc887610f2775ae89d250c4791e65272a/coverage-7.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d", size = 252764, upload-time = "2026-05-26T20:40:54.267Z" }, - { url = "https://files.pythonhosted.org/packages/d2/ee/f78d63c8f079e0d7211c7e2401fa17e311514534ba61bae03e4b287ce4ab/coverage-7.14.1-cp314-cp314-win32.whl", hash = "sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee", size = 222837, upload-time = "2026-05-26T20:40:56.496Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b9/be539854f93a70dfbeec69117f33ec70dc42ff0b65b5b07ab8d40d04228e/coverage-7.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7", size = 223650, upload-time = "2026-05-26T20:40:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/fe/9e/24e2842fef40f35ac82ba3a7719c8023d011bf3bf652d0675316a9d088a1/coverage-7.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343", size = 222218, upload-time = "2026-05-26T20:41:00.321Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1d/ac0a9df5fe31c1e8bdd658074905fc12844a05c1a7e3fdb8417e97c31e23/coverage-7.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1", size = 220822, upload-time = "2026-05-26T20:41:02.281Z" }, - { url = "https://files.pythonhosted.org/packages/32/cf/f964fd9aff20323f9f1a726c97135f8a76bcd87b92dad141a456a43f3c64/coverage-7.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b", size = 221084, upload-time = "2026-05-26T20:41:04.593Z" }, - { url = "https://files.pythonhosted.org/packages/d8/5e/7e5ef2aba844de2b80d678619fcf0841b42e3f37f16411226f3fe4c1016f/coverage-7.14.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474", size = 262454, upload-time = "2026-05-26T20:41:06.641Z" }, - { url = "https://files.pythonhosted.org/packages/64/62/75809bded87015cc4935524218a2a8ed8dd1a8498bfed30a2f4f7a4b4d34/coverage-7.14.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86", size = 264578, upload-time = "2026-05-26T20:41:08.556Z" }, - { url = "https://files.pythonhosted.org/packages/f3/42/d33392dc14633525012d2d504fa1a33b05538bf535f5c1d64675e5754b78/coverage-7.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e", size = 266981, upload-time = "2026-05-26T20:41:10.824Z" }, - { url = "https://files.pythonhosted.org/packages/2a/49/0157c4428c2aca7f1e09d5565930586fd5ae36f1655f08b0daa7cf1fcae1/coverage-7.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65", size = 268112, upload-time = "2026-05-26T20:41:12.966Z" }, - { url = "https://files.pythonhosted.org/packages/96/26/86b9ce71f4092b1ed325ce1421698081df1286b833400b6836912834d6e0/coverage-7.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e", size = 261558, upload-time = "2026-05-26T20:41:15Z" }, - { url = "https://files.pythonhosted.org/packages/20/4c/c311210c5472cf5401d8422b0d7812cdd520f24417673afabda6c323faca/coverage-7.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8", size = 264447, upload-time = "2026-05-26T20:41:17.369Z" }, - { url = "https://files.pythonhosted.org/packages/fb/71/59513f8710ed3e6b0ac0a050a5b7e977bb9c9e880354863b5d00d8809256/coverage-7.14.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07", size = 262048, upload-time = "2026-05-26T20:41:19.309Z" }, - { url = "https://files.pythonhosted.org/packages/84/8d/bceed32dc494f5bbf50f775cd2e78ca814953942b5ea28d3c1c3ac316f14/coverage-7.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de", size = 265781, upload-time = "2026-05-26T20:41:21.559Z" }, - { url = "https://files.pythonhosted.org/packages/e7/c5/9348fe40dbfd4991aaf78df2c6c3098bfb2cc834d1fd362a64b4efef855a/coverage-7.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890", size = 260896, upload-time = "2026-05-26T20:41:23.428Z" }, - { url = "https://files.pythonhosted.org/packages/ca/92/1ea0f03929da7cf87206b1fa24f4c8e9c158be0455481af29ec0a1f3503f/coverage-7.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd", size = 263214, upload-time = "2026-05-26T20:41:25.419Z" }, - { url = "https://files.pythonhosted.org/packages/f6/a9/b2493c054c0e01a643266742ab45e15744e60743f9260cd930c7142b1124/coverage-7.14.1-cp314-cp314t-win32.whl", hash = "sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e", size = 223624, upload-time = "2026-05-26T20:41:27.795Z" }, - { url = "https://files.pythonhosted.org/packages/fc/bd/3e1e6a57fccd2d7c83fcdf338e93ba98eb85c6e877dd34731ac585375490/coverage-7.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c", size = 224728, upload-time = "2026-05-26T20:41:30.098Z" }, - { url = "https://files.pythonhosted.org/packages/bb/d7/31066cf1d2f0c6c797fce911bcfa01dd35642dc6da992a950256097c5860/coverage-7.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af", size = 222752, upload-time = "2026-05-26T20:41:32.123Z" }, - { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, -] - -[package.optional-dependencies] -toml = [ - { name = "tomli", marker = "python_full_version <= '3.11'" }, -] - -[[package]] -name = "hve-telemetry" -version = "0.0.0" -source = { virtual = "." } - -[package.dev-dependencies] -dev = [ - { name = "pytest" }, - { name = "pytest-cov" }, - { name = "ruff" }, -] -fuzz = [ - { name = "atheris" }, -] - -[package.metadata] - -[package.metadata.requires-dev] -dev = [ - { name = "pytest", specifier = ">=9.0" }, - { name = "pytest-cov", specifier = ">=7.0" }, - { name = "ruff", specifier = ">=0.15" }, -] -fuzz = [{ name = "atheris", specifier = ">=3.0" }] - -[[package]] -name = "iniconfig" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, -] - -[[package]] -name = "packaging" -version = "26.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, -] - -[[package]] -name = "pluggy" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, -] - -[[package]] -name = "pygments" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, -] - -[[package]] -name = "pytest" -version = "9.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, -] - -[[package]] -name = "pytest-cov" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coverage", extra = ["toml"] }, - { name = "pluggy" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, -] - -[[package]] -name = "ruff" -version = "0.15.16" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/5f7ec371001337d8fa61701c186ff8b613ecac1651848c5950f4c4d5f2e9/ruff-0.15.16.tar.gz", hash = "sha256:d05e78d38c78caf020b03789e25106c93017db5a0cb6e2819885018c61343b78", size = 4714267, upload-time = "2026-06-04T16:33:09.974Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/42/53ef1c3953f157956db9bf7861e3bc50b9b887ce93300aa48cdba8336fe6/ruff-0.15.16-py3-none-linux_armv6l.whl", hash = "sha256:6ac3c0b3969cc6cf6b158c4e2f8f682acb58e7d700d8a44b65ecdc72d66ab0b2", size = 10709025, upload-time = "2026-06-04T16:32:51.935Z" }, - { url = "https://files.pythonhosted.org/packages/93/9a/a79159346f19134a956607754e57d8d128f7a4c00f4ad2f7514d224c172c/ruff-0.15.16-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:197c207ed75ffba54a0dec23db4aa939a27a3053073e085e0042433cbdc58e4a", size = 11063550, upload-time = "2026-06-04T16:32:42.24Z" }, - { url = "https://files.pythonhosted.org/packages/bc/72/3ce2ac000a5299ec238e01f51397b3b653c93b077d9b1bfe8715bb895f20/ruff-0.15.16-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3a39fec45ab316cc23e7558f23fea4a70403ddb5648ea9a4a3854a16973d0071", size = 10421345, upload-time = "2026-06-04T16:32:37.251Z" }, - { url = "https://files.pythonhosted.org/packages/b0/c2/cc7fad3ec9169373f5b6a18f1917b91080feec40c3f9658334a1d28e2f03/ruff-0.15.16-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba93191d79003116b95128c9d306e045200fdbd0bccb782b110f3cd1d4abc5cf", size = 10757217, upload-time = "2026-06-04T16:32:54.722Z" }, - { url = "https://files.pythonhosted.org/packages/69/d2/3474009eaa0a65b31fa7152a2fad5e2f050c640ceb1e6b02ee6922e94c82/ruff-0.15.16-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6ee4b90520630120ef032aa5cc10db483852dff950e78b1d717e2993a61ac8d", size = 10507035, upload-time = "2026-06-04T16:33:05.343Z" }, - { url = "https://files.pythonhosted.org/packages/ca/81/b7ae6ccbd11f0c8dc3d5d67fc4be9b57ff57ca86ba56152021378e1277f2/ruff-0.15.16-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e4215bc938bc3c8215c1472c1aa437e310fee20cd427335fec9d7e609563628", size = 11255291, upload-time = "2026-06-04T16:32:49.49Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e1/46e526f1a7cc90857ce6ddf25fbb77eb6568651ac38d71b033af07076dd5/ruff-0.15.16-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c8d26be963b090f10e29abc8b3e74a2a321f6fa34e02424e30b5af89350ecbb", size = 12124922, upload-time = "2026-06-04T16:33:07.821Z" }, - { url = "https://files.pythonhosted.org/packages/1a/da/5c791b088b596b24d0deb967fa28ae02ad751a140c0b9ea81c5ab915d6c0/ruff-0.15.16-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f198cf4123602a2280ed46c307bcbafe41758d6fee5b456b6b6058ca1514b3b4", size = 11332186, upload-time = "2026-06-04T16:33:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/72/11/5da87abe20047c8962361473923ebb2f62b595250126aadfad8c20649c1e/ruff-0.15.16-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb27515fa6240fb586ae82b901a59e67d24acff86f2190b433dc542fe0435aeb", size = 11373541, upload-time = "2026-06-04T16:32:47.007Z" }, - { url = "https://files.pythonhosted.org/packages/fe/2a/8554754c23a854ae3fd6b507e36ad61ddb121e298c6d5d617dec94ed0f14/ruff-0.15.16-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a267c46ba1593fc26b8eecbea050b39d40c0b6bb7781ee11c90a02cd10032951", size = 11353014, upload-time = "2026-06-04T16:32:34.795Z" }, - { url = "https://files.pythonhosted.org/packages/62/25/62ea41529ec89f742ea3fed9cb1059c72877ec7cf9b9e99ac9cf3294d1d9/ruff-0.15.16-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:528c68f39a91498a8d50e91ff5985df3d105782bab49cc378e73ac26bff083e8", size = 10737467, upload-time = "2026-06-04T16:32:26.348Z" }, - { url = "https://files.pythonhosted.org/packages/90/17/334d3ad9de4d40f9dd58fdd09e35ce64553bb501e2f19a839e2fb6be14fc/ruff-0.15.16-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7ed55c58950df60589a9a7a5d2f8fa5f54ebd287163be805adfe6ee95a9de123", size = 10521910, upload-time = "2026-06-04T16:32:32.54Z" }, - { url = "https://files.pythonhosted.org/packages/4d/bd/3ac7c6ae77a885c1004b3dda2446ea401768d24f851c14b4ad4b24f6639c/ruff-0.15.16-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d482feaf51512b50f9790ceb417a56a61dd1e9d9bf967662b9ed27c01b34f53a", size = 10979190, upload-time = "2026-06-04T16:32:57.492Z" }, - { url = "https://files.pythonhosted.org/packages/33/d7/609546e6a413c3f216fbf2a50c928f97c80939154f6a0503114094a86191/ruff-0.15.16-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1e15bc8c94513dae2a40cc9ef07c94fdd4ecc9e29dabebeebe170f952322c9e3", size = 11477014, upload-time = "2026-06-04T16:32:44.687Z" }, - { url = "https://files.pythonhosted.org/packages/74/0d/f2cd247ad32633a5c36e97141a2c21b11c6279f7957bc2ff360b1e08fddd/ruff-0.15.16-py3-none-win32.whl", hash = "sha256:580378f7bd4aa25f72e74aa54948a9622f142b1e509521dd10902e886681cc1e", size = 10735541, upload-time = "2026-06-04T16:32:30.145Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9e/02e845ef151b1dee585e55c4739f8e1734ae1d9f1221dff65761c162208b/ruff-0.15.16-py3-none-win_amd64.whl", hash = "sha256:408256017284eddf98fff77b29aa4fb30f586042d535b2d9befc6512f400aaec", size = 11843403, upload-time = "2026-06-04T16:32:39.76Z" }, - { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, -] - -[[package]] -name = "tomli" -version = "2.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, - { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, - { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, - { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, - { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, - { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, - { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, - { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, - { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, - { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, - { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, - { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, - { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, - { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, - { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, - { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, - { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, - { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, - { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, - { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, - { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, - { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, - { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, - { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, - { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, - { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, - { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, - { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, - { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, - { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, - { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, - { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, - { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, - { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, - { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, - { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, - { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, - { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, - { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, - { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, - { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, - { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, - { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, -] From aa53f75aee6fa2d28a1d6b9029c57cc640113467 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 13:49:13 -0700 Subject: [PATCH 03/28] refactor(scripts): drop hook recipes and synthesize hook test fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remove the hooks key and its maturity entry from affected package recipes - register copilot-otel-metrics in the experimental package - replace fixtures that referenced the real manifest with synthetic samples Refs #2717 πŸ”§ - Generated by Copilot --- .github/plugin/marketplace.json | 4 -- .../src/data/__tests__/packageCards.test.ts | 18 +++++-- .../Cross-ChannelProjection.Tests.ps1 | 50 +++++++++++++++---- .../PluginHelpers.Materialization.Tests.ps1 | 49 ++++++++++++------ 4 files changed, 86 insertions(+), 35 deletions(-) diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index d50037923..d7a6b6b3d 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -486,7 +486,6 @@ "skills/shared/pr-reference", "skills/shared/telemetry-foundations" ], - "hooks": "hooks/shared/telemetry.json", "x-hve": { "componentMaturity": { "agents/coding-standards/code-review.agent.md": "experimental", @@ -502,7 +501,6 @@ "prompts/hve-core/evals-import.prompt.md": "experimental", "prompts/hve-core/pr-review.prompt.md": "experimental", "prompts/hve-core/vally-test-write.prompt.md": "experimental", - "hooks/shared/telemetry.json": "experimental", "instructions/coding-standards/code-review/diff-computation.instructions.md": "experimental", "instructions/coding-standards/code-review/review-artifacts.instructions.md": "experimental", "instructions/experimental/mural/mural-bootstrap.instructions.md": "experimental", @@ -779,7 +777,6 @@ "skills/shared/pr-reference", "skills/shared/telemetry-foundations" ], - "hooks": "hooks/shared/telemetry.json", "x-hve": { "componentMaturity": { "agents/accessibility/accessibility-planner.agent.md": "experimental", @@ -859,7 +856,6 @@ "prompts/security/vex-implement.prompt.md": "experimental", "prompts/security/vex-scan.prompt.md": "experimental", "prompts/security/vex-triage.prompt.md": "experimental", - "hooks/shared/telemetry.json": "experimental", "instructions/accessibility/accessibility-identity.instructions.md": "experimental", "instructions/accessibility/accessibility-license-posture.instructions.md": "experimental", "instructions/coding-standards/code-review/diff-computation.instructions.md": "experimental", diff --git a/docs/docusaurus/src/data/__tests__/packageCards.test.ts b/docs/docusaurus/src/data/__tests__/packageCards.test.ts index 6f371158f..b57a928dd 100644 --- a/docs/docusaurus/src/data/__tests__/packageCards.test.ts +++ b/docs/docusaurus/src/data/__tests__/packageCards.test.ts @@ -78,7 +78,7 @@ describe("loadPackageCards against the canonical catalog", () => { describe("countMarketplaceComponents", () => { it("counts a string component field as one component", () => { expect( - countMarketplaceComponents({ hooks: "hooks/shared/telemetry.json" }), + countMarketplaceComponents({ hooks: "hooks/sample/sample-hook.json" }), ).toBe(1); }); @@ -95,7 +95,7 @@ describe("countMarketplaceComponents", () => { commands: ["c.md"], rules: [], skills: ["s"], - hooks: "hooks/shared/telemetry.json", + hooks: "hooks/sample/sample-hook.json", }), ).toBe(5); }); @@ -110,11 +110,21 @@ describe("countMarketplaceComponents", () => { expect(countMarketplaceComponents({ name: "empty" })).toBe(0); }); - it("counts a catalog package that declares hooks as a single string", () => { + it("counts every catalog package that declares hooks as a single string", () => { const withStringHooks = catalog.plugins.filter( (entry) => typeof entry.hooks === "string", ); - expect(withStringHooks.length).toBeGreaterThan(0); + + // No catalog package declares hooks as a bare string today, so iterating + // the filtered set alone cannot fail. The synthetic entry keeps the rule + // under test, and asserting the current count states the end state + // explicitly instead of quietly passing over an empty list. + expect(withStringHooks).toHaveLength(0); + + const synthetic = { name: "synthetic", agents: ["a.md"], hooks: "h.json" }; + expect(countMarketplaceComponents(synthetic)).toBe( + countMarketplaceComponents({ ...synthetic, hooks: undefined }) + 1, + ); for (const entry of withStringHooks) { const withoutHooks = { ...entry, hooks: undefined }; diff --git a/scripts/tests/extension/Cross-ChannelProjection.Tests.ps1 b/scripts/tests/extension/Cross-ChannelProjection.Tests.ps1 index a236a8342..45bdc6627 100644 --- a/scripts/tests/extension/Cross-ChannelProjection.Tests.ps1 +++ b/scripts/tests/extension/Cross-ChannelProjection.Tests.ps1 @@ -109,30 +109,58 @@ Describe 'Cross-channel canonical projection' -Tag 'Unit' { Describe 'Cross-channel hook exclusion' -Tag 'Unit' { BeforeAll { - $script:HookProjections = @($script:Projections | Where-Object { @($_.HookSources).Count -gt 0 }) + # Hook components are plugin-only and the catalog may declare none, so the + # exclusion is proven against a fixture recipe rather than live membership. + $script:HookFixtureSource = '.github/hooks/sample/sample-hook.json' + $script:HookFixtureRecipe = @( + @{ Kind = 'hook'; Field = 'hooks'; PackagePath = 'hooks/sample/sample-hook.json'; SourcePath = $script:HookFixtureSource } + @{ Kind = 'agent'; Field = 'agents'; PackagePath = 'agents/sample/sample.agent.md'; SourcePath = '.github/agents/sample/sample.agent.md' } + ) + $script:HookFixtureSources = @($script:HookFixtureRecipe | Where-Object { $_.Kind -eq 'hook' } | ForEach-Object { $_.SourcePath }) } It 'Resolves a non-empty hook component set before asserting the exclusion' { - @($script:HookProjections).Count | Should -BeGreaterThan 0 - @($script:HookProjections | ForEach-Object { $_.HookSources } | Sort-Object -Unique).Count | Should -BeGreaterThan 0 + @($script:HookFixtureSources).Count | Should -BeGreaterThan 0 } It 'Keeps hooks out of every VS Code contribution bucket' { - foreach ($projection in $script:HookProjections) { - $contributions = Get-ExtensionContributions -Items $projection.Recipe - $extensionSources = Get-ExtensionCanonicalSource -Contribution $contributions - foreach ($hookSource in $projection.HookSources) { - $extensionSources | Should -Not -Contain $hookSource - } + $contributions = Get-ExtensionContributions -Items $script:HookFixtureRecipe + $extensionSources = Get-ExtensionCanonicalSource -Contribution $contributions + foreach ($hookSource in $script:HookFixtureSources) { + $extensionSources | Should -Not -Contain $hookSource } } It 'Keeps hook sources under the hooks root only' { - foreach ($projection in $script:HookProjections) { + foreach ($hookSource in $script:HookFixtureSources) { + $hookSource | Should -BeLike '.github/hooks/*' + } + } + + It 'Excludes every catalog-declared hook source when the catalog declares one' { + $catalogHookSources = @($script:Projections | ForEach-Object { $_.HookSources } | Sort-Object -Unique) + $withHooks = @($script:Projections | Where-Object { @($_.HookSources).Count -gt 0 }) + + # The catalog may declare no hook components at all. Iterating an empty + # set proves nothing, so the current end state is asserted explicitly: + # either the catalog declares hooks and each is excluded, or it declares + # none and this states that as a fact rather than passing silently. + if ($withHooks.Count -eq 0) { + @($catalogHookSources).Count | + Should -Be 0 -Because 'no catalog package declares a hook component today' + return + } + + foreach ($projection in $withHooks) { + $contributions = Get-ExtensionContributions -Items $projection.Recipe + $extensionSources = Get-ExtensionCanonicalSource -Contribution $contributions foreach ($hookSource in $projection.HookSources) { - $hookSource | Should -BeLike '.github/hooks/*' + $extensionSources | Should -Not -Contain $hookSource } } + foreach ($hookSource in $catalogHookSources) { + $hookSource | Should -BeLike '.github/hooks/*' + } } } diff --git a/scripts/tests/plugins/PluginHelpers.Materialization.Tests.ps1 b/scripts/tests/plugins/PluginHelpers.Materialization.Tests.ps1 index 2e9058def..52c65640e 100644 --- a/scripts/tests/plugins/PluginHelpers.Materialization.Tests.ps1 +++ b/scripts/tests/plugins/PluginHelpers.Materialization.Tests.ps1 @@ -401,32 +401,49 @@ Describe 'Write-PluginDirectory' -Tag 'Unit' { Describe 'Hook plugin root fallback' -Tag 'Unit' { BeforeAll { - $script:hookRepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../../..')).Path - $script:hookSourceText = Get-Content -LiteralPath (Join-Path $script:hookRepoRoot '.github/hooks/shared/telemetry.json') -Raw -Encoding utf8 - $script:hookRootExpression = '[string]::IsNullOrWhiteSpace($env:CLAUDE_PLUGIN_ROOT) ? ''.github'' : $env:CLAUDE_PLUGIN_ROOT' - $script:hookRepositoryBash = '${CLAUDE_PLUGIN_ROOT:-.github}/hooks/shared/telemetry/telemetry-collector.sh' - $script:hookRepositoryPwsh = "& (Join-Path ($script:hookRootExpression) 'hooks/shared/telemetry/Invoke-TelemetryCollector.ps1')" - $script:hookInstalledBash = '${CLAUDE_PLUGIN_ROOT}/hooks/shared/telemetry/telemetry-collector.sh' - $script:hookInstalledPwsh = '& (Join-Path $env:CLAUDE_PLUGIN_ROOT ''hooks/shared/telemetry/Invoke-TelemetryCollector.ps1'')' + $script:hookRepositoryBash = '${CLAUDE_PLUGIN_ROOT:-.github}/hooks/sample/sample-hook/sample-hook.sh' + $script:hookRepositoryPwsh = "& (Join-Path ($script:hookRootExpression) 'hooks/sample/sample-hook/Invoke-SampleHook.ps1')" + $script:hookInstalledBash = '${CLAUDE_PLUGIN_ROOT}/hooks/sample/sample-hook/sample-hook.sh' + $script:hookInstalledPwsh = '& (Join-Path $env:CLAUDE_PLUGIN_ROOT ''hooks/sample/sample-hook/Invoke-SampleHook.ps1'')' $script:hookEventNames = @( 'sessionStart', 'userPromptSubmitted', 'userPromptSubmit', 'preToolUse', 'postToolUse', 'subagentStart', 'subagentStop', 'sessionEnd', 'stop', 'agentStop', 'preCompact' ) + # Synthesize the repository-form manifest so the transform under test is + # exercised against the generic hook contract rather than any one package. + $hookEventMap = [ordered]@{} + foreach ($eventName in $script:hookEventNames) { + $hookEventMap[$eventName] = @( + [ordered]@{ + type = 'command' + bash = $script:hookRepositoryBash + powershell = $script:hookRepositoryPwsh + timeoutSec = 10 + } + ) + } + $script:hookSourceText = [ordered]@{ + version = 1 + description = 'Sample lifecycle hook used to exercise plugin materialization.' + hooks = $hookEventMap + } | ConvertTo-Json -Depth 10 + # Produce the installed form through the production writer so the # transform under test is exercised rather than a hand-written copy. $hookRepo = Join-Path $TestDrive 'hook-fallback' New-PluginFixtureRepository -Path $hookRepo -Version '9.9.9' | Out-Null - Add-PluginFixtureFile -RepoRoot $hookRepo -RelativePath '.github/hooks/shared/telemetry.json' -Content $script:hookSourceText | Out-Null - Add-PluginFixtureFile -RepoRoot $hookRepo -RelativePath '.github/hooks/shared/telemetry/telemetry-collector.sh' -Content "#!/usr/bin/env bash`nexit 0`n" | Out-Null - Add-PluginFixtureFile -RepoRoot $hookRepo -RelativePath '.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1' -Content "exit 0`n" | Out-Null + Add-PluginFixtureFile -RepoRoot $hookRepo -RelativePath '.github/hooks/sample/sample-hook.json' -Content $script:hookSourceText | Out-Null + Add-PluginFixtureFile -RepoRoot $hookRepo -RelativePath '.github/hooks/sample/sample-hook/sample-hook.sh' -Content "#!/usr/bin/env bash`nexit 0`n" | Out-Null + Add-PluginFixtureFile -RepoRoot $hookRepo -RelativePath '.github/hooks/sample/sample-hook/Invoke-SampleHook.ps1' -Content "exit 0`n" | Out-Null - Write-PluginDirectory -Entry ([ordered]@{ name = 'shared'; description = 'Shared telemetry package'; version = '9.9.9'; license = 'MIT' }) ` - -Items @(@{ Kind = 'hook'; Field = 'hooks'; PackagePath = 'hooks/shared/telemetry.json'; SourcePath = '.github/hooks/shared/telemetry.json' }) ` + Write-PluginDirectory -Entry ([ordered]@{ name = 'sample'; description = 'Sample hook package'; version = '9.9.9'; license = 'MIT' }) ` + -Items @(@{ Kind = 'hook'; Field = 'hooks'; PackagePath = 'hooks/sample/sample-hook.json'; SourcePath = '.github/hooks/sample/sample-hook.json' }) ` -PluginsDir (Join-Path $hookRepo 'plugins') -RepoRoot $hookRepo -Version '9.9.9' | Out-Null - $script:hookInstalledText = Get-Content -LiteralPath (Join-Path $hookRepo 'plugins/shared/hooks/shared/telemetry.json') -Raw -Encoding utf8 + $script:hookRepoRoot = $hookRepo + $script:hookInstalledText = Get-Content -LiteralPath (Join-Path $hookRepo 'plugins/sample/hooks/sample/sample-hook.json') -Raw -Encoding utf8 $script:hookRepositoryEvents = (ConvertFrom-Json $script:hookSourceText -AsHashtable)['hooks'] $script:hookInstalledEvents = (ConvertFrom-Json $script:hookInstalledText -AsHashtable)['hooks'] } @@ -448,8 +465,8 @@ Describe 'Hook plugin root fallback' -Tag 'Unit' { } It 'Resolves the fallback root to tracked collector scripts' { - Test-Path -LiteralPath (Join-Path $script:hookRepoRoot '.github/hooks/shared/telemetry/telemetry-collector.sh') -PathType Leaf | Should -BeTrue - Test-Path -LiteralPath (Join-Path $script:hookRepoRoot '.github/hooks/shared/telemetry/Invoke-TelemetryCollector.ps1') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $script:hookRepoRoot '.github/hooks/sample/sample-hook/sample-hook.sh') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $script:hookRepoRoot '.github/hooks/sample/sample-hook/Invoke-SampleHook.ps1') -PathType Leaf | Should -BeTrue } It 'Resolves the same root as bash for ' -ForEach @( @@ -485,7 +502,7 @@ Describe 'Hook plugin root fallback' -Tag 'Unit' { } Context 'when the manifest is materialized into a plugin' { - It 'Preserves every lifecycle event and its telemetry contract' { + It 'Preserves every lifecycle event and its hook contract' { @($script:hookInstalledEvents.Keys | Sort-Object) | Should -Be @($script:hookEventNames | Sort-Object) foreach ($eventName in $script:hookInstalledEvents.Keys) { @($script:hookInstalledEvents[$eventName]) | Should -HaveCount @($script:hookRepositoryEvents[$eventName]).Count From 017b7a159001f22b5f02e6471c390cec74b850e5 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 13:49:16 -0700 Subject: [PATCH 04/28] docs: point telemetry guidance at the opt-in skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the local telemetry guide with the copilot-otel-metrics guide and updates the security model and transparency note to describe opt-in capture. Refs #2717 πŸ“ - Generated by Copilot --- TRANSPARENCY-NOTE.md | 3 +- docs/contributing/hooks.md | 15 +- docs/customization/README.md | 27 +- docs/customization/copilot-otel-metrics.md | 126 ++++-- docs/customization/local-telemetry.md | 403 ------------------ .../experimental/copilot-otel-metrics.md | 2 +- docs/security/security-model.md | 28 +- 7 files changed, 128 insertions(+), 476 deletions(-) delete mode 100644 docs/customization/local-telemetry.md diff --git a/TRANSPARENCY-NOTE.md b/TRANSPARENCY-NOTE.md index 5d8217c53..f30ce36a4 100644 --- a/TRANSPARENCY-NOTE.md +++ b/TRANSPARENCY-NOTE.md @@ -56,8 +56,7 @@ HVE Core ships text files and supporting tools. When you load an HVE Core file i 2. You work with the host's model, now shaped by the file's instructions. 3. The model's replies come back through the normal Copilot surface. -HVE Core has no model, no API, and no network calls while you author or install it. It ships one optional local telemetry hook that is disabled by default and, when you turn it on, records Copilot session lifecycle events to plaintext files on your own disk with no network egress. -The processed event stream stores derived signals (such as tool-input key names and a truncated prompt preview) rather than full payloads; a separate, explicit opt-in is required before any verbatim prompt or tool input is captured. See the [Local Telemetry guide](docs/customization/local-telemetry.md) for exactly what is captured and how to disable or remove it. +HVE Core has no model, no API, and no network calls while you author or install it. It ships no telemetry collector that runs on your machine by default. Optional OpenTelemetry export is a Copilot Chat feature you configure yourself, and the `copilot-otel-metrics` skill only generates the settings, local stack, and infrastructure templates you choose to apply. Nothing is collected until you enable that export and run a receiver you control. Validation tools (linters, frontmatter checks, Pester tests, plugin generation) run in CI on pull requests. Nothing runs on your machine unless you install the VS Code extension or run a packaged script yourself. Most skills are pure authoring or validation helpers with no independent Responsible AI surface and are not called out individually. A few skills warrant specific mention because they assemble media outputs or depend on external services: diff --git a/docs/contributing/hooks.md b/docs/contributing/hooks.md index 08858103e..cf0487420 100644 --- a/docs/contributing/hooks.md +++ b/docs/contributing/hooks.md @@ -37,10 +37,7 @@ Hooks use package-oriented source folders. Use this structure for hook contribut Manifests live one package level down (`.github/hooks//`). A flat `.github/hooks/.json` is treated as a repo-specific artifact and is excluded from distribution. -The telemetry hook is the current reference implementation: - -* `.github/hooks/shared/telemetry.json` -* `.github/hooks/shared/telemetry/` +The repository ships no hook manifest today. The generic contract below stays supported, and the validator accepts a repository with zero manifests. ## Implementing a New Hook @@ -79,8 +76,6 @@ For reliability and portability, hook scripts should follow these rules: * Keep runtime short and respect `timeoutSec` values in the manifest. * Support both bash and PowerShell paths when practical. -Telemetry follows this model with a no-op gate and structured JSONL append behavior. - ## Manifest Schema and Validation Manifests are validated against `scripts/linting/schemas/hook-manifest.schema.json`, the authoritative contract. The schema enforces the allowed top-level keys (`version`, `description`, `hooks`), the ten CLI-lowercase event names (`sessionStart`, `sessionEnd`, `userPromptSubmit`, `userPromptSubmitted`, `preToolUse`, `postToolUse`, `preCompact`, `subagentStart`, `subagentStop`, `stop`), and the permitted command properties. @@ -101,10 +96,9 @@ when a hook persists payloads to disk: * Write to local, gitignored locations and never to committed paths. * Document exactly what is captured, where it is written, and how to remove it. -The telemetry hook applies this pattern: its processed `sessions-*.jsonl` stream -stores only tool-input key names and a truncated prompt preview, while the -verbatim `raw-input.jsonl` dump is a separate opt-in (`HVE_TELEMETRY_RAW=1`, -off by default). See [Local Telemetry](../customization/local-telemetry#sensitive-data-and-privacy). +A hook that persists derived signals should keep any verbatim payload capture +behind its own environment gate, default it off, and name the exact file it +writes so an operator can delete it. ## Event Compatibility Guidance @@ -149,7 +143,6 @@ When your hook includes scripts, also run the relevant script linters and tests * [Custom Agents](custom-agents) * [Instructions](instructions) * [Common Standards](ai-artifacts-common) -* [Local Telemetry](../customization/local-telemetry) --- diff --git a/docs/customization/README.md b/docs/customization/README.md index fda605a90..db33c6577 100644 --- a/docs/customization/README.md +++ b/docs/customization/README.md @@ -98,18 +98,18 @@ with type-specific examples. Each HVE role benefits from different customization techniques. The table below maps the nine roles to the guides most relevant to their workflow. -| Role | Recommended Guides | Rationale | -|--------------------------|-------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------| -| Engineer | [Instructions](instructions.md), [Agents](custom-agents.md) | Coding standards and specialized review agents accelerate daily development | -| TPM | [Prompts](prompts.md), [Marketplace Packages](packages.md) | Reusable planning prompts and curated bundles standardize project workflows | -| Tech Lead / Architect | [Instructions](instructions.md), [Agents](custom-agents.md), [Skills](skills.md) | Standards enforcement, architecture review agents, and deep domain knowledge | -| Security Architect | [Skills](skills.md), [Instructions](instructions.md) | Compliance knowledge packages and security-focused coding conventions | -| Data Scientist | [Skills](skills.md), [Prompts](prompts.md) | Analytical domain bundles and repeatable notebook workflows | -| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md), [Local Telemetry](local-telemetry.md) | Infrastructure conventions, DevContainer tuning, and local telemetry workflows | -| Platform / Observability | [Copilot OTel Metrics](copilot-otel-metrics.md), [Local Telemetry](local-telemetry.md) | Agent usage, token cost, and latency measurement through OpenTelemetry | -| Business Program Manager | [Prompts](prompts.md), [Team Adoption](team-adoption.md) | Sprint-planning prompts and governance patterns for stakeholder alignment | -| New Contributor | [Instructions](instructions.md), [Environment](environment.md) | Quick onboarding through conventions and a ready-to-use development environment | -| Utility | [Marketplace Packages](packages.md), [Build System](build-system.md) | Cross-cutting tooling assembly and validation pipeline customization | +| Role | Recommended Guides | Rationale | +|--------------------------|----------------------------------------------------------------------------------|---------------------------------------------------------------------------------| +| Engineer | [Instructions](instructions.md), [Agents](custom-agents.md) | Coding standards and specialized review agents accelerate daily development | +| TPM | [Prompts](prompts.md), [Marketplace Packages](packages.md) | Reusable planning prompts and curated bundles standardize project workflows | +| Tech Lead / Architect | [Instructions](instructions.md), [Agents](custom-agents.md), [Skills](skills.md) | Standards enforcement, architecture review agents, and deep domain knowledge | +| Security Architect | [Skills](skills.md), [Instructions](instructions.md) | Compliance knowledge packages and security-focused coding conventions | +| Data Scientist | [Skills](skills.md), [Prompts](prompts.md) | Analytical domain bundles and repeatable notebook workflows | +| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md) | Infrastructure conventions and DevContainer tuning | +| Platform / Observability | [Copilot OTel Metrics](copilot-otel-metrics.md) | Agent usage, token cost, and latency measurement through OpenTelemetry | +| Business Program Manager | [Prompts](prompts.md), [Team Adoption](team-adoption.md) | Sprint-planning prompts and governance patterns for stakeholder alignment | +| New Contributor | [Instructions](instructions.md), [Environment](environment.md) | Quick onboarding through conventions and a ready-to-use development environment | +| Utility | [Marketplace Packages](packages.md), [Build System](build-system.md) | Cross-cutting tooling assembly and validation pipeline customization | ## File Index @@ -122,8 +122,7 @@ Each HVE role benefits from different customization techniques. The table below 7. [Forking and Extending](forking.md): Full fork-and-extend customization 8. [Environment Customization](environment.md): DevContainers, VS Code settings, MCP servers 9. [Team Adoption and Governance](team-adoption.md): Governance, naming, onboarding, change management -10. [Local Telemetry](local-telemetry.md): Enable local telemetry, review capture and storage schema mechanics, and generate reports -11. [Copilot OpenTelemetry Metrics](copilot-otel-metrics.md): Export Copilot OTel signals to a local Grafana stack, or to a fleet-wide Azure pipeline, and query agent, token, and latency data +10. [Copilot OpenTelemetry Metrics](copilot-otel-metrics.md): Export Copilot OTel signals to a local Grafana stack, or to a fleet-wide Azure pipeline, and query agent, token, and latency data ## Related Resources diff --git a/docs/customization/copilot-otel-metrics.md b/docs/customization/copilot-otel-metrics.md index c0616c29d..0a13a9f6d 100644 --- a/docs/customization/copilot-otel-metrics.md +++ b/docs/customization/copilot-otel-metrics.md @@ -3,7 +3,7 @@ title: Copilot OpenTelemetry Metrics description: Capture GitHub Copilot Chat OpenTelemetry signals on your own machine in a local Grafana stack, or across an organization through Azure sidebar_position: 11 author: Microsoft -ms.date: 2026-07-29 +ms.date: 2026-08-13 ms.topic: how-to keywords: - opentelemetry @@ -20,7 +20,7 @@ estimated_reading_time: 18 GitHub Copilot Chat can export traces, metrics, and events over OpenTelemetry. Point it at a collector you run yourself and you get a measured view of your own agent sessions: which models you use, how long calls take, which tools run most, how many tokens you burn, and how much of that is cache. -Everything up to [Configuring this for an organization](#configuring-this-for-an-organization) runs on one machine, in one container, and sends nothing anywhere. You do not need an administrator to try it. The sections after that cover pushing the same configuration to a fleet and collecting it in Azure, which does need one. +Everything up to [Configuring this for an organization](#configuring-this-for-an-organization) runs on one machine, in two containers, and sends nothing anywhere. You do not need an administrator to try it. The sections after that cover pushing the same configuration to a fleet and collecting it in Azure, which does need one. :::tip Let the skill do it The [`copilot-otel-metrics` skill](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SKILL.md) walks this guide for you. @@ -67,10 +67,11 @@ The split between metrics and traces matters more than it first appears, and the ```mermaid graph LR - A["VS Code
Copilot Chat"] -->|OTLP :4318| B["OTel Collector"] - B --> C["Prometheus
metrics"] - B --> D["Tempo
traces"] - B --> E["Loki
events"] + A["VS Code
Copilot Chat"] -->|OTLP :4318| B["OTel Collector
allow-list filter"] + B -->|OTLP, internal network| G["LGTM container"] + G --> C["Prometheus
metrics"] + G --> D["Tempo
traces"] + G --> E["Loki
events"] C --> F["Grafana
:3000"] D --> F E --> F @@ -78,42 +79,105 @@ graph LR ## Start the stack -The Grafana OTel-LGTM image bundles Grafana, Prometheus, Tempo, Loki, and an OpenTelemetry Collector in a single container with the datasources pre-provisioned. +Two containers. The Grafana OTel-LGTM image bundles Grafana, Prometheus, Tempo, and Loki with the datasources pre-provisioned. In front of it sits an OpenTelemetry Collector that decides what is allowed to reach any of them. + +The Collector is not decoration. It is the only OTLP listener published on your machine, and LGTM's own OTLP ports are deliberately unmapped. If both listened, any local process could write straight past the filter, and binding to loopback would not help, because everything on your machine is already on loopback. ```yaml services: + otel-collector: + # otel/opentelemetry-collector-contrib:0.158.0 + image: otel/opentelemetry-collector-contrib@sha256:c5918f78992ee73b0d6f0e599423ac5ec52dd5d9726733114d6eca53d5a32ed5 + container_name: copilot-otel-collector + restart: unless-stopped + command: ["--config=/etc/otelcol-contrib/config.yaml"] + ports: + - "127.0.0.1:4317:4317" # OTLP gRPC + - "127.0.0.1:4318:4318" # OTLP HTTP + volumes: + - ./otel-collector-local.yaml:/etc/otelcol-contrib/config.yaml:ro + networks: [copilot-otel] + depends_on: + lgtm: + condition: service_healthy + lgtm: - image: grafana/otel-lgtm:0.29.2 + # grafana/otel-lgtm:0.29.2 + image: grafana/otel-lgtm@sha256:af7242c1a9608faf6d26e6f235392fd0c32b67258228f9a3cfc96e724974930c container_name: copilot-otel-lgtm restart: unless-stopped ports: - "127.0.0.1:3000:3000" # Grafana - - "127.0.0.1:4317:4317" # OTLP gRPC - - "127.0.0.1:4318:4318" # OTLP HTTP - "127.0.0.1:9090:9090" # Prometheus - "127.0.0.1:3200:3200" # Tempo + networks: [copilot-otel] environment: + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_AUTH_DISABLE_LOGIN_FORM: "false" + GF_AUTH_BASIC_ENABLED: "true" + GF_SECURITY_ADMIN_USER: ${COPILOT_OTEL_GRAFANA_USER:?set COPILOT_OTEL_GRAFANA_USER before starting the stack} + GF_SECURITY_ADMIN_PASSWORD: ${COPILOT_OTEL_GRAFANA_PASSWORD:?set COPILOT_OTEL_GRAFANA_PASSWORD before starting the stack} PROMETHEUS_EXTRA_ARGS: "--enable-feature=otlp-deltatocumulative --storage.tsdb.retention.time=120d" volumes: - copilot-otel-data:/data +networks: + copilot-otel: + driver: bridge + volumes: copilot-otel-data: external: true ``` +The Collector configuration it mounts is an allow-list, not a delete-list: + +```yaml +processors: + redaction: + allow_all_keys: false + allowed_keys: + - service.name + - gen_ai.request.model + - gen_ai.usage.input_tokens + - gen_ai.usage.output_tokens + - copilot_chat.mode_name + # ...the rest of the keys the dashboards actually read + summary: silent +``` + +The allow-list is not the whole story, and the difference is worth knowing before you rely on it. It governs **attributes**, plus a map-valued log body. It does not reach span names, span status messages, span event names, trace state, non-map log bodies, severity text, metric metadata, span links, or metric exemplars. A second `transform/scrub` processor closes the ones nothing in the shipped dashboards reads. + +Three things still reach the store unfiltered, and they are recorded as gaps rather than described away: + +* **Span names and metric metadata**, because dashboard queries match on them. A span name is composed by the emitter, so this is the residual worth watching. +* **Span link attributes and metric exemplar attributes**, because this Collector distribution offers no way to reach them. OTTL has no `spanlink` context, refuses to index `links`, and silently ignores an assignment to `datapoint.exemplars`. + +None of that is inferred from reading the configuration. `tests/test_collector_carriers.py` starts the pinned Collector, sends a distinct marker through each of 28 carriers, and records what survives, with a paired control run proving each marker is visible when nothing filters it. The recorded map is asserted, so a change that opens a carrier fails the test suite. + +That direction is the whole point. A delete-list removes the content attributes someone already knows about; it passes through the one a future extension release adds. An allow-list drops anything it has not been told to keep, so an unfamiliar attribute is a dropped attribute rather than a stored one. The full file is in the skill's examples directory. + ```bash +export COPILOT_OTEL_GRAFANA_USER=admin +read -rs -p 'Grafana password: ' COPILOT_OTEL_GRAFANA_PASSWORD; echo +export COPILOT_OTEL_GRAFANA_PASSWORD docker volume create copilot-otel-data docker compose up -d ``` -Four details in that file are deliberate: +The password is read rather than typed inline so it does not land in the terminal scrollback or the shell history file. In PowerShell, `Read-Host -AsSecureString` serves the same purpose. A Compose `.env` file reaches the containers but not the bundled Python helpers, which read the same two variables from the environment of the shell that runs them. + +Several details in that file are deliberate: -* Ports bind to `127.0.0.1` because Grafana ships with default credentials and nothing needs to reach this stack from off-host. +* Grafana credentials are required and have no default. The LGTM image otherwise enables anonymous access with the Admin role and hides the login form, which would leave every panel and every stored prompt fragment readable by anything that can reach port 3000. The `${VAR:?message}` form fails the `up` rather than starting on a guessable credential. +* Both images are pinned by digest, with the tag kept in a comment. A tag is mutable, so a tag-pinned stack can change under a configuration you already reviewed. +* Ports bind to `127.0.0.1` because nothing here should be reachable from off-host. * The volume is declared `external` so Compose binds the volume you created instead of making a project-prefixed duplicate and orphaning your history. * `otlp-deltatocumulative` is set defensively. Prometheus drops delta-temporality metrics by default, and a dropped delta metric can fail the entire batched write, taking unrelated cumulative metrics with it. The flag converts instead of dropping and is inert when traffic is already cumulative. * Retention is raised well past the 15 day default, because a monthly total would otherwise truncate silently at fifteen days. +The filter decides what is stored. It does not change what the extension emits, so content attributes still leave the editor and still cross the loopback hop in plaintext before the Collector drops them. + Everything in this guide is copy-pasteable, but the dashboard is not: it is roughly nineteen kilobytes of JSON. Runnable copies of the compose file, the dashboard, and the verification helpers live in the [copilot-otel-metrics skill](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SKILL.md), @@ -134,25 +198,29 @@ Add these to your **user** `settings.json`, then reload the window. ``` > [!IMPORTANT] -> These settings are application-scoped. They cannot live in workspace `.vscode/settings.json`, and they do not take effect until you run **Developer: Reload Window**. If you enable export and see nothing, the reload is almost always why. +> These settings do not take effect until you run **Developer: Reload Window**. If you enable export and see nothing, the reload is almost always why. -Application scope has a second consequence that is easier to miss. These keys resolve from the **default profile no matter which profile is active**, so editing `User/profiles//settings.json` accomplishes nothing and tells you nothing: no error, no warning, no telemetry. -Run **Preferences: Open Application Settings (JSON)** from the command palette and VS Code opens the exact file these settings come from. If you run both stable and Insiders you have two of these files, and only the one belonging to the build you are testing counts. +Which settings file they resolve from depends on the scope the installed build declares, and that is worth checking rather than assuming. +In the build verified for this page, none of the OTel keys declares a scope, so they take VS Code's default `window` scope and the active profile's settings file applies. +If your build declares them `scope: application`, they resolve from the **default profile no matter which profile is active**, and editing `User/profiles//settings.json` accomplishes nothing and tells you nothing: no error, no warning, no telemetry. + +Run **Preferences: Open Application Settings (JSON)** from the command palette and VS Code opens the global file. That target is correct either way, so it is the safe place to write when you are unsure. If you run both stable and Insiders you have two of these files, and only the one belonging to the build you are testing counts.
-All eleven settings, and the environment variables that override them +All seven settings, and the environment variables that override them + +Three keys turn export on. The other four tune it. Every name below is prefixed with `github.copilot.chat.otel.`. -Three keys turn export on. The other eight tune it. Every name below is prefixed with `github.copilot.chat.otel.`. +Verified against GitHub Copilot Chat 0.52.0. +An earlier revision of this page listed eleven settings, adding `protocol`, `headers`, `serviceName`, and `resourceAttributes`, verified against build `0.59.2026072702` where all eleven were application-scoped. +Both observations were accurate for their own build, which is the real lesson: this surface moves with the extension. +Writing a key your build does not declare gives you a setting that is silently inert, so check your own installed extension before using any of them. | Setting | Type | Default | Sets what | |--------------------------|---------|---------------------------|------------------------------------------------------------------------| | `enabled` | boolean | `false` | Master switch for trace, metric, and log emission | | `exporterType` | string | `"otlp-http"` | One of `otlp-grpc`, `otlp-http`, `console`, `file` | | `otlpEndpoint` | string | `"http://localhost:4318"` | Where the data goes | -| `protocol` | string | `""` | `""`, `http/json`, `http/protobuf`, or `grpc`; empty means `http/json` | -| `headers` | object | `{}` | Extra OTLP headers, such as auth tokens, sent by the exporter | -| `serviceName` | string | `""` | The `service.name` resource attribute | -| `resourceAttributes` | object | `{}` | Extra resource attributes, merged per key with the environment | | `captureContent` | boolean | `false` | Prompts, responses, system instructions, and tool definitions on spans | | `maxAttributeSizeChars` | integer | `0` | Truncation limit in characters; `0` disables truncation | | `outfile` | string | `""` | JSON-lines output path; setting it forces the `file` exporter | @@ -165,10 +233,6 @@ Resolution order is enterprise policy, then environment variable, then user sett | `enabled` | `COPILOT_OTEL_ENABLED` | | `captureContent` | `COPILOT_OTEL_CAPTURE_CONTENT` | | `otlpEndpoint` | `OTEL_EXPORTER_OTLP_ENDPOINT` | -| `protocol` | `OTEL_EXPORTER_OTLP_PROTOCOL` | -| `serviceName` | `OTEL_SERVICE_NAME` | -| `resourceAttributes` | `OTEL_RESOURCE_ATTRIBUTES` | -| `headers` | `OTEL_EXPORTER_OTLP_HEADERS` | | `maxAttributeSizeChars` | `COPILOT_OTEL_MAX_ATTRIBUTE_SIZE_CHARS` | That second table is the practical route for a devcontainer or a CI runner, where editing a global settings file is awkward. `exporterType`, `outfile`, and `dbSpanExporter.enabled` declared no environment variable in the build inspected for this page, so those three come from settings or policy. Check your own installed extension before relying on either table. @@ -379,10 +443,11 @@ That command returns your own prompt text. Do not paste its output into a shared > [!WARNING] > With content capture disabled I still observed six attributes populated in plaintext on spans: `copilot_chat.user_request`, `gen_ai.input.messages`, > `gen_ai.output.messages`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`, and `gen_ai.system_instructions`. -> A seventh, `copilot_chat.reasoning_content`, was present but marked `[encrypted]`. **Treat the store as holding your prompt text regardless of the setting.** -> On a local-only stack the data stays on your machine, but the volume is unencrypted, traces have no configured expiry, and `docker compose down` preserves it -> deliberately, so any local user with Docker or filesystem access can read it. The skill's threat model rates that Medium residual risk and tracks it as an open gap. -> It stops being a local question entirely the moment `otlpEndpoint` points at a shared or hosted collector. Treat both the endpoint and the volume as sensitive. +> A seventh, `copilot_chat.reasoning_content`, was present but marked `[encrypted]`. **The extension emits these regardless of the setting.** +> The Collector's allow-list is what keeps them out of the store: none of them is allowed, so they are dropped before Prometheus or Tempo sees them. That covers these seven, which are attributes. It does not cover every carrier: span names and metric metadata reach storage unfiltered because dashboard queries read them, and span links and metric exemplars reach it because no processor in this distribution can address them. +> What the filter cannot change is that they leave the editor and cross the loopback hop in plaintext, so anything reading the OTLP port directly still sees them. +> The volume itself is unencrypted and `docker compose down` preserves it deliberately, so any local user with Docker or filesystem access can read whatever +> survived filtering. It stops being a local question entirely the moment `otlpEndpoint` points at a shared or hosted collector. Treat both the endpoint and the volume as sensitive. ## Configuring this for an organization @@ -523,8 +588,7 @@ To stop exporting, set `github.copilot.chat.otel.enabled` to `false` and reload ## Related reading * The [copilot-otel-metrics skill](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SKILL.md) walks this guide for you and generates the local stack, both dashboards, and the Azure collector and infrastructure templates. It is invoked explicitly and never activates on its own. -* The skill's [examples directory](https://github.com/microsoft/hve-core/tree/main/.github/skills/experimental/copilot-otel-metrics/examples) holds runnable copies of everything on this page: the compose file, both dashboards, the helper scripts, and the [Azure templates](https://github.com/microsoft/hve-core/tree/main/.github/skills/experimental/copilot-otel-metrics/examples/azure) with their own deploy guide. -* [Local Telemetry](local-telemetry) covers the hook-based JSONL capture, which records session lifecycle events rather than OTel signals. +* The skill's [examples directory](https://github.com/microsoft/hve-core/tree/main/.github/skills/experimental/copilot-otel-metrics/examples) holds runnable copies of everything on this page: the compose file, the Collector configuration, both dashboards, the helper scripts, and the [Azure templates](https://github.com/microsoft/hve-core/tree/main/.github/skills/experimental/copilot-otel-metrics/examples/azure) with their own deploy guide. * [Monitor agent usage with OpenTelemetry](https://code.visualstudio.com/docs/agents/guides/monitoring-agents) is the upstream reference for signal names and settings. * [Manage AI settings in enterprise environments](https://code.visualstudio.com/docs/enterprise/ai-settings) documents the managed settings channels in full. * [Visualize Azure Monitor data with Grafana](https://learn.microsoft.com/azure/azure-monitor/visualize/visualize-grafana-overview) compares the two Grafana products in Microsoft's own words. diff --git a/docs/customization/local-telemetry.md b/docs/customization/local-telemetry.md deleted file mode 100644 index 0c8b76494..000000000 --- a/docs/customization/local-telemetry.md +++ /dev/null @@ -1,403 +0,0 @@ ---- -title: Local Telemetry -description: Enable local Copilot session telemetry, understand capture mechanics, and generate local reports -sidebar_position: 10 -author: Microsoft -ms.date: 2026-08-02 -ms.topic: how-to -keywords: - - telemetry - - hooks - - local reporting - - copilot -estimated_reading_time: 7 ---- - -## What This Captures - -The local telemetry hook captures Copilot lifecycle events into local JSONL files. It is intended for local analysis and troubleshooting of your own sessions. - -The telemetry manifest is at `.github/hooks/shared/telemetry.json`. - -Events currently captured include: - -* session start -* user prompt submission -* pre-tool and post-tool use -* subagent start and stop -* agent stop and session end -* pre-compact events - -At stop time, telemetry also appends a session summary with model and token usage, but only when the surface you are running on writes a usage log it can read. See [Enrichment Coverage by Surface](#enrichment-coverage-by-surface) for what each surface provides. - -## Prerequisites - -All telemetry processing runs in Python. The shell entry points are thin wrappers that gate on opt-in and hand stdin to `_telemetry_core.py`. - -| Requirement | Needed for | Notes | -|-----------------|-----------------------------------------------------------|------------------------------------------------------------------------------------------| -| Python 3.11+ | Event collection, reports, cleanup | Collectors try `python3` then `python`; the report and cleanup scripts require `python3` | -| Bash 3.2+ | Bash entry points | macOS system Bash works; no Bash 4 features used | -| PowerShell 5.1+ | `Invoke-TelemetryCollector.ps1` | Written for Windows PowerShell; no `#Requires` floor | -| PowerShell 7.4+ | `Invoke-TelemetryReport.ps1`, `Invoke-TelemetryClean.ps1` | Enforced by `#Requires -Version 7.4` | -| `jq` | `generate-telemetry-report.sh` | Not needed by `Invoke-TelemetryReport.ps1` | - -You only need one shell family. Copilot selects the Bash or PowerShell entry point based on the host. - -Only `telemetry-collector.sh` checks the interpreter version, skipping a `python3` older than 3.11 and falling back to `python`. Where Python is missing or too old, the collector warns and continues without recording events, so collection never blocks a session. The report and cleanup scripts are run by hand and fail loudly instead. - -## Enable Local Telemetry - -Telemetry is opt-in. Enable it with either an environment variable or a repository marker file. - -### Option 1: Environment Variable - -```bash -export HVE_TELEMETRY=1 -``` - -```powershell -$env:HVE_TELEMETRY = "1" -``` - -### Option 2: Repository Marker File - -Create `.hve-telemetry` at the repository root: - -```bash -touch .hve-telemetry -``` - -Either option enables collection. If both are absent, the hook exits in no-op mode. - -### Optional: Verbatim Raw Payload Capture - -Processed telemetry never stores full prompt text or full tool inputs (see -[Sensitive Data and Privacy](#sensitive-data-and-privacy)). A separate, -explicit opt-in records the first few hook payloads **verbatim** to -`raw-input.jsonl` for deep diagnostics. It is off by default, even when -telemetry is enabled, and both the Bash and PowerShell collectors honor it: - -```bash -export HVE_TELEMETRY_RAW=1 -``` - -Leave this unset unless you are actively debugging the hook payload shape, and -remove the captured file afterward. Because it stores prompts and tool inputs in -the clear, treat any session run with it enabled as potentially sensitive. - -## View Reports - -Generate a report with the script in ~/.hve (created at session start): - -```bash -bash ~/.hve/generate-report.sh -``` - -Which will generate a `report.generated.html` for viewing. - -The generated report path is printed when report generation completes. - -The report is self-contained: it embeds every selected JSONL file (session -events plus model and token enrichment) inline. Combined cross-project reports -over a long history (`--all-dirs --date all`) can therefore grow to several -megabytes. Narrow the scope with a specific `--date`, or report a single project -without `--all-dirs`, when a smaller file is preferred. - -## Disable Local Telemetry - -Disable collection by removing both enablement gates: - -1. Unset `HVE_TELEMETRY` -2. Remove `.hve-telemetry` from repository root - -## Where Data Is Written - -Default output directory: - -`/.copilot-tracking/telemetry` - -Override with `HVE_TELEMETRY_DIR` when needed. - -Key files and folders: - -| Path | Purpose | -|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------| -| `sessions-YYYY-MM-DD.jsonl` | Daily event stream with hook events and session summaries | -| `sessions-YYYY-MM-DD.--.jsonl` | Fallback shard written when a collector could not take the day log's lock; read alongside it | -| `raw-input.jsonl` | First few hook payloads stored verbatim; written only when `HVE_TELEMETRY_RAW=1` is set | -| `.stacks//ops.log` | Append-only agent push and pop records for one session, replayed to attribute events to the calling agent | -| `report.generated.html` | Optional self-contained report output | - -## Data Captured and Storage Schema - -This section describes the mechanics of what local telemetry collects and where each class of data comes from. - -### Collection Pipeline - -1. Copilot lifecycle events invoke the telemetry hook from `.github/hooks/shared/telemetry.json`. -2. Shell entry points (`telemetry-collector.sh` and `Invoke-TelemetryCollector.ps1`) enforce opt-in gates. -3. Event payloads are normalized and appended to daily JSONL files. -4. On stop events, a `SessionSummary` record is appended with model and token aggregates when available. - -### Core Record Types - -The daily JSONL stream contains two primary record types: - -| Record Type | Trigger | Purpose | -|--------------------|----------------------------------------|-------------------------------------------------| -| Hook event records | Session/tool/subagent lifecycle events | Timeline of what happened during a session | -| `SessionSummary` | Stop event (`Stop`) | Aggregated usage totals and model-level summary | - -### Common Fields - -Most hook event records include: - -* `ts`: Event timestamp (ISO 8601) -* `sid`: Session identifier -* `event`: Canonical event name (for example, `PreToolUse`, `PostToolUse`, `SessionStart`) -* `cwd`: Working directory at capture time - -Additional fields are event-specific. Examples: - -* Prompt events: truncated prompt preview -* Tool events: tool name, selected input keys, response length, inferred agent attribution -* Subagent events: agent name and display name -* Stop events: stop reason - -Tool events also carry two fields the bundled report does not render, kept for -offline analysis of a full session timeline. A tool payload names no agent, and a -turn's tool calls run in parallel, so an active subagent is not evidence that it -issued any particular call: - -* `agent`: Present only when no subagent was in flight, holding `root`. This is - the one case where the caller is certain. -* `agents`: The candidate list (`root` plus every started-but-not-stopped - subagent) when one or more subagents were running. The two fields are mutually - exclusive; neither resolves the caller once work overlaps. -* `tool_use_id`: The client's identifier for one tool invocation, pairing a - `PreToolUse` record with its `PostToolUse` record exactly, where the report's - by-name correlation can only approximate the pairing under concurrency. - -Token attribution does not have this problem and the report does render it; see -[Session Summary Fields](#session-summary-fields). - -### Session Summary Fields - -When available at stop time, `SessionSummary` includes: - -* `models`: Model usage map observed during the session -* `model_usage`: Per-model aggregate usage counters -* `input_tokens`, `output_tokens` -* `cache_read_tokens`, `cache_write_tokens` -* `total_nano_aiu` -* `turns`, `messages` -* `token_source`: provenance of the token numbers (`process_log` or `state_fallback`) -* Optional `reasoning_effort`, `subagent_map`, and `client` -* `agent_usage`: Per-agent token counters, present only under `process_log` - -Unlike tool attribution, the token split is exact. Every process-log request -records the agent that issued it, so the collector partitions requests by agent -and resolves each id to a display name through `subagent_map`. The report shows -the split under Token Usage, as a share of AIU (or of output tokens where the -host reports no AIU), and rolls it up to a Subagent Share card. - -Where a host emits no process log, the report reconstructs the same split by -crediting each merged subagent session its own totals and treating the balance -as the root agent's. - -### Data Sources by Layer - -| Data Category | Source | Availability | -|---------------------------|-------------------------------------------------|--------------------------------| -| Hook lifecycle events | Copilot hook payloads via the collector scripts | Any surface that runs hooks | -| Session summaries | `~/.copilot/session-state//events.jsonl` | Copilot CLI only | -| Precise per-request usage | `~/.copilot/logs/process-*.log` | Copilot CLI only | -| Report-time enrichment | VS Code `debug-logs/**/*.jsonl` (`llm_request`) | VS Code builds that write them | - -### Enrichment Coverage by Surface - -Event capture and usage enrichment are separate concerns. Hook events are recorded on any surface that runs hooks. Model and token data comes from surface-specific logs that telemetry reads but does not create, and the two sources are not interchangeable. - -#### Copilot CLI - -The CLI maintains its own session state under `~/.copilot/session-state//` (honoring `COPILOT_HOME`). -When that directory contains an `events.jsonl`, the collector appends a `SessionSummary` inline at `Stop`, `SessionEnd`, and `PreCompact`, and the report re-derives summaries through the `aggregate-session` pass. -Precise per-request usage comes from `~/.copilot/logs/process-*.log`, located through the session lock PID and, once the lock is gone, by scanning for logs that reference the session's interaction ids. -Not every state directory has an `events.jsonl`; sessions that never reached a recorded turn produce no summary. - -#### VS Code - -VS Code sessions do not appear under `~/.copilot/session-state`, so the CLI path contributes nothing for them. Their only enrichment source is the Copilot Chat debug log, discovered by globbing `debug-logs/**/*.jsonl` beneath the workspace storage roots for `.vscode-server-insiders`, `.vscode-server`, `.vscode`, and the platform user-data directories for Code - Insiders, Code, and VSCodium. - -> [!IMPORTANT] -> Debug logs are not written by every VS Code build. Public (stable) VS Code on macOS has been confirmed to produce none. On such a host, telemetry still records the full event timeline, but the report shows no model, token, or cost data for those sessions, and no `SessionSummary` record is written. This is a source-availability gap, not a telemetry failure. - -Missing enrichment is never fatal. Both aggregation passes return a non-zero status when they find nothing, the generator silently skips the corresponding layer, and the report renders from the hook event stream alone. - -### Summary Provenance - -Each `SessionSummary` carries a `token_source` field recording where its numbers came from: - -| `token_source` | Meaning | -|------------------|----------------------------------------------------------------| -| `process_log` | Per-request `assistant_usage` metrics from the CLI process log | -| `state_fallback` | Summed `session.shutdown` model metrics from `events.jsonl` | - -Under `state_fallback`, shutdown metrics are summed across every run segment because a resumed session resets its counters on each resume. A live session that has not yet ended a segment knows only per-message output tokens, so input, cache, and AIU totals are reported as unknown rather than zero, keeping an in-progress session distinguishable from a free one. - -### Event Naming Normalization - -The pipeline normalizes different casing variants of event names to canonical names used in stored telemetry records. This keeps mixed-client event surfaces queryable from one schema. - -### Storage and Retention Behavior - -* Data is stored locally under `.copilot-tracking/telemetry` by default. -* Records append to date-partitioned files (`sessions-YYYY-MM-DD.jsonl`). - Concurrent hook events append to that one file: POSIX resolves `O_APPEND` in the kernel, and on Windows the collector takes a byte-range lock first, because the C runtime emulates append as seek-then-write and would otherwise let one writer overwrite another. - A collector that cannot take the lock writes a sibling shard named `sessions-YYYY-MM-DD.--.jsonl`, which the report generators pick up alongside the day log. -* A small verbatim raw payload sample is stored in `raw-input.jsonl` only when `HVE_TELEMETRY_RAW=1` is explicitly set; see [Sensitive Data and Privacy](#sensitive-data-and-privacy). -* Per-session agent stacks are maintained under `.stacks//ops.log`, an append-only record of agent pushes and pops that is replayed to attribute each event, and removed on session stop. - -### Sensitive Data and Privacy - -Local telemetry writes plaintext JSONL to local disk only. It makes no network -calls and the default output directory (`.copilot-tracking/telemetry`) is -gitignored, so data is not committed. The risk is local-disk exposure, not a -committed leak. Be aware of what each layer records: - -* **Processed stream (`sessions-*.jsonl`)** stores a truncated prompt preview - (first 200 characters of each submitted prompt) and, for tool events, only - the tool input *key names* plus selected fields such as file paths and - subagent names. It does not store full tool input *values* (file contents or - shell command strings). A secret pasted into the start of a prompt can still - appear in the 200-character preview. -* **Verbatim raw dump (`raw-input.jsonl`)** stores the first few hook payloads - exactly as received, including the full prompt and the full tool input (file - contents being written, shell command strings). It is off by default and only - written when `HVE_TELEMETRY_RAW=1` is set. -* **User-level locations** under `~/.hve` and `~/.copilot` (honoring `HVE_HOME`) - hold the report generator and directory registry. Generated reports embed the - captured JSONL inline. - -To reduce exposure: keep `HVE_TELEMETRY_RAW` unset, avoid pasting secrets into -prompts while telemetry is enabled, and remove captured files when you are done -(`bash ~/.hve/clean-telemetry.sh` or delete the telemetry directory). - -## Generate a Report - -Run the report generator directly: - -```bash -bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --help -bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --date all -bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --open -``` - -On Windows (or any PowerShell host) the native equivalent needs no `bash`: - -```powershell -pwsh .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 -Date all -pwsh .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 -Open -``` - -## Cross-Project Reports - -Telemetry is captured per project, so each repository keeps its own store under -`/.copilot-tracking/telemetry`. To view sessions across every project in a -single report, each store is recorded once per session in a user-level registry -at `~/.hve/telemetry-dirs` (honoring `HVE_HOME`). - -Generate a combined, cross-project report with `--all-dirs`: - -```bash -bash .github/hooks/shared/telemetry/generate-telemetry-report.sh --all-dirs --date all -``` - -The PowerShell generator takes `-AllDirs` for the same cross-project report: - -```powershell -pwsh .github/hooks/shared/telemetry/Invoke-TelemetryReport.ps1 -AllDirs -Date all -``` - -The registry self-populates as you work across repositories, so no manual setup -is required. Stale directories (deleted or moved repositories) are pruned -automatically when the report runs. Each session is labeled with its originating -project in the report, so combined output still reads per project. - -To report on a single store, pass `--path` (`-Path`). An explicit path overrides -`--all-dirs`, so the generated cross-project launcher can also be scoped down: - -```bash -bash ~/.hve/generate-report.sh --path /path/to/repo/.copilot-tracking/telemetry --date all -``` - -Cleanup follows the same precedence: `clean-telemetry.sh --path DIR` restricts -removal to that one store and leaves the registry, launchers, and every other -project untouched, even when `--all-dirs` is also present. Both entry points -report the override on stderr, so a narrowed destructive scope is never silent. - -> [!NOTE] -> **Registry-driven cleanup is name-constrained.** `clean-telemetry.sh -> --all-dirs` iterates every path in `~/.hve/telemetry-dirs` and, in each -> directory, removes only a fixed allow-list of artifact names: -> `raw-input.jsonl`, `report.generated.html`, date-shaped -> `sessions-YYYY-MM-DD*.jsonl` logs and their fallback shards, and the -> `.stacks/` directory. It never deletes a directory -> wholesale. A tampered registry can therefore, at most, delete those specific -> names in an attacker-chosen directory, not arbitrary files. The `.stacks/` -> entry is removed recursively, but symlinked artifacts are unlinked rather than -> followed, so the target of a symlink is never deleted. The registry lives in -> the user-owned HVE home (`~/.hve`, honoring `HVE_HOME`), so an attacker able -> to tamper with it already holds the user's filesystem privileges; the risk is -> low and the blast radius is bounded. - -## Reports Without the Repository (Extension Users) - -When telemetry runs from the VS Code extension rather than this repository, the -report generator lives at a version-pinned extension path that is awkward to -locate. To bridge this, a cross-project launcher is written into the HVE home -directory (`~/.hve`, honoring `HVE_HOME`) at session start, next to the registry -it reads: - -* `~/.hve/generate-report.sh`: for unix shells and Git Bash on Windows -* `~/.hve/generate-report.ps1`: for PowerShell, runs natively (no `bash` required) - -Run the launcher from the HVE home directory without knowing the extension path. -It defaults to a combined, cross-project report written to -`~/.hve/report.generated.html`: - -```bash -bash ~/.hve/generate-report.sh -bash ~/.hve/generate-report.sh --date all -``` - -From PowerShell, run the native launcher: - -```powershell -~/.hve/generate-report.ps1 -~/.hve/generate-report.ps1 -Date all -``` - -The launchers are regenerated every session, so they self-heal after an -extension upgrade. They forward any extra arguments to the report generator. - -## Troubleshooting - -Common issues: - -* No events captured: verify one enablement gate is set and your hook manifest is active. -* Events captured but no model, token, or cost data: the surface produced no usable usage log. On the CLI, confirm `~/.copilot/session-state//events.jsonl` exists for the session. In VS Code, confirm a `debug-logs` directory exists under your workspace storage; stable builds may not write one. See [Enrichment Coverage by Surface](#enrichment-coverage-by-surface). -* Token totals look approximate: check `token_source` on the `SessionSummary` record. `state_fallback` means the CLI process log was unavailable and totals were summed from shutdown metrics. -* Report generation fails: ensure `python3` is available for enrichment. The bash generator also needs `jq`; the PowerShell generator (`Invoke-TelemetryReport.ps1`) does not. - -## Related Guides - -* [Contributing Hooks](../contributing/hooks) -* [Environment Customization](environment) -* [Managing Packages](packages) - ---- - -*πŸ€– Crafted with precision by ✨Copilot following brilliant human instruction, -then carefully refined by our team of discerning human reviewers.* diff --git a/docs/reference/skills/experimental/copilot-otel-metrics.md b/docs/reference/skills/experimental/copilot-otel-metrics.md index 5e474061d..781bc135b 100644 --- a/docs/reference/skills/experimental/copilot-otel-metrics.md +++ b/docs/reference/skills/experimental/copilot-otel-metrics.md @@ -38,7 +38,7 @@ It has four independent modes. Name one directly, or describe the outcome you wa Reach for something else when: * You want seat-level Copilot adoption figures across an organization. The GitHub Copilot Metrics API answers that far more cheaply, though it returns no spans, tokens, or latency. -* You want session lifecycle events rather than OTel signals. [Local Telemetry](../../../customization/local-telemetry) covers the hook-based JSONL capture. +* You want session lifecycle events rather than OTel signals. This skill covers OTel signals only. * You are instrumenting your own service rather than measuring Copilot. [telemetry-foundations](../shared/telemetry-foundations) carries the OpenTelemetry naming conventions for that. ## Example usage diff --git a/docs/security/security-model.md b/docs/security/security-model.md index be9cb73e2..3a72bf07c 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -3,7 +3,7 @@ title: Security Assurance Case and Security Model description: Comprehensive security model and security assurance documentation demonstrating enterprise security practices sidebar_position: 2 author: Microsoft -ms.date: 2026-08-07 +ms.date: 2026-08-13 ms.topic: reference keywords: - security @@ -1661,19 +1661,19 @@ Most skills are markdown knowledge packs with no runtime and are covered by the Skills that ship an executable runtime (network egress, credential handling, subprocess execution, or untrusted document/content parsing) carry their own per-skill STRIDE threat model in a `SECURITY.md` next to their `SKILL.md`. Those models follow a shared structure (assets, adversaries, trust buckets with per-bucket STRIDE mitigations, and an Enterprise Readiness Gaps register) and are the authoritative source for each skill's residual risk. -| Skill | Runtime surface | Primary residual gaps | Security model | -|-------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| -| jira | REST CLI; environment credentials | No token revocation; best-effort redaction; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/jira/SECURITY.md) | -| gitlab | REST CLI; environment credentials; git-remote subprocess | Untrusted CI-trace egress; insecure-transport opt-out; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/gitlab/SECURITY.md) | -| mural (experimental) | REST CLI; embedded stdio MCP server; OAuth token store | OAuth audit gaps; keyring backend toggle is code-execution surface | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/mural/SECURITY.md) | -| tts-voiceover (experimental) | Azure Speech egress; key/Entra credentials; SSML + PPTX parsing | Content egress to Azure region; broad credential chain | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/tts-voiceover/SECURITY.md) | -| accessibility | Arbitrary-URL scan egress; version-pinned `npx @axe-core/cli@4.12.1` subprocess; design-intent verification adapter | Runtime integrity is best-effort without lockfile/integrity hash; no egress allow-list (SSRF); headless-browser surface; consuming-project validation boundary is repository-scoped | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/accessibility/accessibility/SECURITY.md) | -| powerpoint (experimental) | Sandboxed `content-extra.py` execution; LibreOffice/MuPDF document parsing | Denylist confinement is not OS-level; external-parser CVE exposure | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/powerpoint/SECURITY.md) | -| video-to-gif (experimental) | Local CLI (bash + PowerShell); FFmpeg/ffprobe subprocess; untrusted media parsing | Inherited FFmpeg decoder CVE exposure; bare-filename search resolution | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/video-to-gif/SECURITY.md) | -| copilot-otel-metrics (experimental) | Diff-approved per-key write into the user's global settings.json; loopback OTLP telemetry ingest into a containerized Grafana/Prometheus/Tempo stack; four stdlib Python reference helpers querying local APIs; generated collector configuration, Bicep, Terraform, and Azure CLI templates the operator deploys | Prompt content present in spans despite the documented capture default; shared fleet-wide ingest credential with no per-user binding or in-place rotation; unauthenticated loopback ingest; the no-execution boundary on Docker and infrastructure commands is advisory rather than enforced | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SECURITY.md) | -| gh-code-scanning | GitHub code-scanning read via `gh` CLI subprocess; stdout only | Unpinned `gh`/`jq` PATH dependencies; TLS delegated to `gh` | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/gh-code-scanning/SECURITY.md) | -| customer-card-render (experimental) | Local Python CLI; regex parse of untrusted DT markdown; YAML emission | Inherited powerpoint build toolchain; confidential DT prose egress | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/customer-card-render/SECURITY.md) | -| vex | Local Python gate (`vex_gate.py`); anchored-regex parse of untrusted detection-issue body; `json.loads` of local OpenVEX doc; exit code only | Gate-suppression by issue-edit access; forced-proceed AI-credit consumption | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/vex/SECURITY.md) | +| Skill | Runtime surface | Primary residual gaps | Security model | +|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| jira | REST CLI; environment credentials | No token revocation; best-effort redaction; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/jira/SECURITY.md) | +| gitlab | REST CLI; environment credentials; git-remote subprocess | Untrusted CI-trace egress; insecure-transport opt-out; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/gitlab/SECURITY.md) | +| mural (experimental) | REST CLI; embedded stdio MCP server; OAuth token store | OAuth audit gaps; keyring backend toggle is code-execution surface | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/mural/SECURITY.md) | +| tts-voiceover (experimental) | Azure Speech egress; key/Entra credentials; SSML + PPTX parsing | Content egress to Azure region; broad credential chain | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/tts-voiceover/SECURITY.md) | +| accessibility | Arbitrary-URL scan egress; version-pinned `npx @axe-core/cli@4.12.1` subprocess; design-intent verification adapter | Runtime integrity is best-effort without lockfile/integrity hash; no egress allow-list (SSRF); headless-browser surface; consuming-project validation boundary is repository-scoped | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/accessibility/accessibility/SECURITY.md) | +| powerpoint (experimental) | Sandboxed `content-extra.py` execution; LibreOffice/MuPDF document parsing | Denylist confinement is not OS-level; external-parser CVE exposure | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/powerpoint/SECURITY.md) | +| video-to-gif (experimental) | Local CLI (bash + PowerShell); FFmpeg/ffprobe subprocess; untrusted media parsing | Inherited FFmpeg decoder CVE exposure; bare-filename search resolution | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/video-to-gif/SECURITY.md) | +| copilot-otel-metrics (experimental) | Schema-enforced, backed-up, audited write into the user's global settings.json; loopback OTLP ingest through a fail-closed filtering Collector in front of a containerized Grafana/Prometheus/Tempo stack; five stdlib Python reference helpers plus a shared input-policy module querying local APIs; generated collector configuration, Bicep, Terraform, and Azure CLI templates the operator deploys | Prompt content present in spans before the Collector filters it and in plaintext across the loopback hop; content carriers the Collector governs incompletely, namely span names and metric metadata preserved for shipped dashboard queries, and span links and metric exemplars unreachable by any processor in this distribution, and the instrumentation scope and schema fields left unfiltered on an assumption about emitter behavior; shared fleet-wide ingest credential with no per-user binding or in-place rotation, delivered to the extension exporter but not the agent host, whose telemetry a per-workstation loopback relay carries at the cost of an unauthenticated local listener; unauthenticated Prometheus and Tempo query APIs; exporter-side certificate validation outside skill control; retention is the only deletion mechanism; the no-execution boundary on Docker and infrastructure commands is advisory rather than enforced | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SECURITY.md) | +| gh-code-scanning | GitHub code-scanning read via `gh` CLI subprocess; stdout only | Unpinned `gh`/`jq` PATH dependencies; TLS delegated to `gh` | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/gh-code-scanning/SECURITY.md) | +| customer-card-render (experimental) | Local Python CLI; regex parse of untrusted DT markdown; YAML emission | Inherited powerpoint build toolchain; confidential DT prose egress | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/customer-card-render/SECURITY.md) | +| vex | Local Python gate (`vex_gate.py`); anchored-regex parse of untrusted detection-issue body; `json.loads` of local OpenVEX doc; exit code only | Gate-suppression by issue-edit access; forced-proceed AI-credit consumption | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/vex/SECURITY.md) | Skills whose scripts perform only local validation with no external surface (for example `adr-author` and `vally-tests`) do not require a dedicated model; their risk is bounded by the repository-level controls. When a new skill adds an executable runtime with any of the surfaces above, add a `SECURITY.md` following the shared structure and register it in this table and in the [security documentation index](README.md#skill-security-models). From a158fa5f054aa25c6dc2d7d63a88f0b3ca49a7b3 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 14:28:37 -0700 Subject: [PATCH 05/28] chore(settings): add OpenTelemetry terms to the spell dictionary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit otelcol, OTTL, and spanlink appear in the copilot-otel-metrics guide. Refs #2717 πŸ“– - Generated by Copilot --- .cspell.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cspell.json b/.cspell.json index e465976c9..089c1f667 100644 --- a/.cspell.json +++ b/.cspell.json @@ -176,6 +176,8 @@ "onclick", "optimi", "ordinally", + "otelcol", + "OTTL", "overbroad", "packageable", "parseable", @@ -210,6 +212,7 @@ "skillmd", "SLSA", "smol", + "spanlink", "SPOF", "SSSC", "stdlib", From 208a212801025b3f8e8f9da1494f0e3a53f6f533 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 17:11:07 -0700 Subject: [PATCH 06/28] style(skills): apply ruff format to copilot-otel-metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI runs ruff format --check as of #2709; six files disagreed with the formatter. Refs #2717 🎨 - Generated by Copilot --- .../examples/inspect_metrics.py | 1 - .../examples/validate_dashboard.py | 7 +-- .../tests/test_collector_carriers.py | 43 ++++++------------- .../tests/test_helpers.py | 8 +--- .../tests/test_local_config.py | 20 +++------ .../tests/test_settings_upsert.py | 7 +-- 6 files changed, 24 insertions(+), 62 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py b/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py index ec1e09f94..b9c72e904 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/inspect_metrics.py @@ -85,4 +85,3 @@ def main() -> int: if __name__ == "__main__": raise SystemExit(main()) - diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py b/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py index df49671d4..9c61836b0 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py @@ -173,8 +173,7 @@ def validate_panels( n = len(rows) except urllib.error.HTTPError as exc: print( - f" [ERROR] {title:<44} {label:<28} " - f"{exc.code} {exc.read().decode()[:70]}" + f" [ERROR] {title:<44} {label:<28} {exc.code} {exc.read().decode()[:70]}" ) continue except Exception as exc: # noqa: BLE001 @@ -236,8 +235,7 @@ def main(argv: list[str] | None = None) -> int: ("localhost", "127.", "::1") ): print( - f"warning: importing into {grafana_url} overwrites any dashboard " - "sharing the same uid.", + f"warning: importing into {grafana_url} overwrites any dashboard sharing the same uid.", file=sys.stderr, ) @@ -285,4 +283,3 @@ def store(url: str) -> dict: if __name__ == "__main__": raise SystemExit(main()) - diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py index 70220260e..ad1ea23d0 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py @@ -264,8 +264,7 @@ class Carrier: "CARRIERMARKERMETRICNAME", PASSED_THROUGH, NO_MECHANISM, - note="Preserved: every Prometheus query in the shipped dashboard " - "selects by metric name.", + note="Preserved: every Prometheus query in the shipped dashboard selects by metric name.", ), Carrier("metric description", "CARRIERMARKERMETRICDESCRIPTION", PASSED_THROUGH, NO_MECHANISM), Carrier("metric unit", "CARRIERMARKERMETRICUNIT", PASSED_THROUGH, NO_MECHANISM), @@ -301,10 +300,7 @@ def traces_payload() -> dict: "attributes": [ _attr("service.name", "copilot-chat"), _attr("probe.resource.attr", "CARRIERMARKERRESOURCEATTR"), - *( - _attr(key, marker) - for key, marker in RESTORED_DETECTION_KEYS.items() - ), + *(_attr(key, marker) for key, marker in RESTORED_DETECTION_KEYS.items()), ] }, "schemaUrl": "https://example.invalid/CARRIERMARKERRESOURCESCHEMAURL", @@ -313,9 +309,7 @@ def traces_payload() -> dict: "scope": { "name": "CARRIERMARKERSCOPENAME", "version": "CARRIERMARKERSCOPEVERSION", - "attributes": [ - _attr("probe.scope.attr", "CARRIERMARKERSCOPEATTR") - ], + "attributes": [_attr("probe.scope.attr", "CARRIERMARKERSCOPEATTR")], }, "schemaUrl": "https://example.invalid/CARRIERMARKERSCOPESCHEMAURL", "spans": [ @@ -359,9 +353,7 @@ def traces_payload() -> dict: "spanId": "bbbbbbbbbbbbbbbb", "traceState": "probe=CARRIERMARKERLINKTRACESTATE", "attributes": [ - _attr( - "probe.link.attr", "CARRIERMARKERLINKATTR" - ) + _attr("probe.link.attr", "CARRIERMARKERLINKATTR") ], } ], @@ -390,9 +382,7 @@ def logs_payload() -> dict: "severityText": "CARRIERMARKERLOGSEVERITYTEXT", "eventName": "CARRIERMARKERLOGEVENTNAME", "body": {"stringValue": "CARRIERMARKERLOGBODYSCALAR"}, - "attributes": [ - _attr("probe.log.attr", "CARRIERMARKERLOGATTR") - ], + "attributes": [_attr("probe.log.attr", "CARRIERMARKERLOGATTR")], }, { "timeUnixNano": "1700000000000000000", @@ -411,9 +401,7 @@ def logs_payload() -> dict: "timeUnixNano": "1700000000000000000", "body": { "arrayValue": { - "values": [ - {"stringValue": "CARRIERMARKERLOGBODYARRAY"} - ] + "values": [{"stringValue": "CARRIERMARKERLOGBODYARRAY"}] } }, }, @@ -423,9 +411,7 @@ def logs_payload() -> dict: }, { "timeUnixNano": "1700000000000000000", - "body": { - "stringValue": "password=CARRIERMARKERMASKLOGBODY" - }, + "body": {"stringValue": "password=CARRIERMARKERMASKLOGBODY"}, }, ], } @@ -467,9 +453,7 @@ def metrics_payload() -> dict: { "timeUnixNano": "1700000001000000000", "asInt": "1", - "traceId": ( - "5b8efff798038103d269b633813fc60c" - ), + "traceId": ("5b8efff798038103d269b633813fc60c"), "spanId": "eee19b7ec3c1b174", "filteredAttributes": [ _attr( @@ -723,8 +707,11 @@ def run_probe(runtime: ContainerRuntime, *, remove_content_processors: bool) -> def _await_all_signals(runtime: ContainerRuntime, container: str, *, timeout: float) -> str: """Wait until the exporter has rendered all three signals, then return them.""" - expected = ('"otelcol.signal": "traces"', '"otelcol.signal": "logs"', - '"otelcol.signal": "metrics"') + expected = ( + '"otelcol.signal": "traces"', + '"otelcol.signal": "logs"', + '"otelcol.signal": "metrics"', + ) deadline = time.monotonic() + timeout output = "" while time.monotonic() < deadline: @@ -799,9 +786,7 @@ def test_a_masked_expectation_declares_what_masking_looks_like(self) -> None: def test_a_passed_through_carrier_declares_no_mechanism(self) -> None: """A carrier cannot claim a control and pass through at the same time.""" contradictory = [ - c.name - for c in CARRIERS - if c.expected == PASSED_THROUGH and c.mechanism != NO_MECHANISM + c.name for c in CARRIERS if c.expected == PASSED_THROUGH and c.mechanism != NO_MECHANISM ] assert contradictory == [] diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py index 64b1618e8..eca8992a5 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py @@ -373,8 +373,7 @@ def fake_open_url(request, **kwargs): # noqa: ANN001, ANN202 authorized = [r for r in sent if "Authorization" in r.headers] assert len(sent) > 1, "the dashboard walk issued no store queries" assert len(authorized) == 1, ( - "exactly one request may carry the Grafana credential; " - f"{len(authorized)} did" + f"exactly one request may carry the Grafana credential; {len(authorized)} did" ) assert authorized[0].full_url.startswith("http://localhost:3000/") for request in sent: @@ -388,9 +387,7 @@ def fake_open_url(request, **kwargs): # noqa: ANN001, ANN202 class TestRedirectCredentialHandling: """A redirect may not carry a credential to an origin it was not issued for.""" - def _handler( - self, *, allow_remote: bool = False - ) -> urllib.request.HTTPRedirectHandler: + def _handler(self, *, allow_remote: bool = False) -> urllib.request.HTTPRedirectHandler: handler = open_url.__globals__["_PolicyRedirectHandler"] return handler(allow_remote=allow_remote, allowed_ports=DEFAULT_ALLOWED_PORTS) @@ -509,4 +506,3 @@ def test_no_bundled_helper_opens_a_url_outside_the_policy_module(self, name: str f"{name} opens a URL directly; every helper must go through open_url " "so scheme, authority, port, redirect, and remote-opt-in rules apply" ) - diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py index b82e05b0f..4248fc5f2 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py @@ -131,9 +131,7 @@ def shipped_consumers() -> ShippedConsumers: for label in labels if not label.startswith("$") and label not in PROMETHEUS_RESERVED_LABELS } - trace_fields = { - field for field in ("rootTraceName", "durationMs") if field in baseline_source - } + trace_fields = {field for field in ("rootTraceName", "durationMs") if field in baseline_source} return ShippedConsumers( metric_names=frozenset(metric_names), @@ -497,9 +495,7 @@ def statement_target(statement: str) -> str: # Intrinsics a shipped consumer reads directly. `span.name` is here because # three TraceQL queries match on it; the metric intrinsics are here because # every Prometheus query selects by metric name. -PROTECTED_INTRINSICS = frozenset( - {"span.name", "metric.name", "metric.description", "metric.unit"} -) +PROTECTED_INTRINSICS = frozenset({"span.name", "metric.name", "metric.description", "metric.unit"}) ATTRIBUTE_TARGET = re.compile(r"attributes\[\"([^\"]+)\"\]") @@ -534,9 +530,7 @@ class TestScrubStaysInsideTheInventory: def test_the_scrub_processor_declares_statements(self, collector: dict[str, Any]) -> None: assert scrub_statements(collector), "the content scrub declares no statements" - def test_no_scrub_statement_targets_a_shipped_consumer( - self, collector: dict[str, Any] - ) -> None: + def test_no_scrub_statement_targets_a_shipped_consumer(self, collector: dict[str, Any]) -> None: findings = overreaching_statements(scrub_statements(collector), shipped_consumers()) assert findings == [], f"a scrub rule targets telemetry a shipped panel reads: {findings}" @@ -561,9 +555,7 @@ def test_a_condition_that_reads_a_protected_field_is_not_an_overreach(self) -> N class TestExporterAlias: """The exporter uses the alias the pinned Collector asks for.""" - def test_no_pipeline_references_the_deprecated_alias( - self, collector: dict[str, Any] - ) -> None: + def test_no_pipeline_references_the_deprecated_alias(self, collector: dict[str, Any]) -> None: exporters = set(collector["exporters"]) assert not any(name == "otlp" or name.startswith("otlp/") for name in exporters), ( "the pinned Collector logs a deprecation warning once per signal for " @@ -572,9 +564,7 @@ def test_no_pipeline_references_the_deprecated_alias( for name, pipeline in collector["service"]["pipelines"].items(): assert set(pipeline["exporters"]) <= exporters, f"{name} names an undeclared exporter" - def test_the_internal_hop_is_unchanged_by_the_rename( - self, collector: dict[str, Any] - ) -> None: + def test_the_internal_hop_is_unchanged_by_the_rename(self, collector: dict[str, Any]) -> None: exporter = collector["exporters"]["otlp_grpc/lgtm"] assert exporter["endpoint"] == "lgtm:4317" assert exporter["tls"]["insecure"] is True diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py index 6a2bbe070..a5b3e72ed 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py @@ -175,12 +175,7 @@ def test_a_nested_key_of_the_same_name_is_not_targeted(self) -> None: break silently under a later change: the write would land in a `"[python]"` block and the user's real setting would stay unchanged. """ - nested = ( - "{\n" - ' "[python]": { "' + ENABLED + '": false },\n' - ' "' + ENABLED + '": false\n' - "}\n" - ) + nested = '{\n "[python]": { "' + ENABLED + '": false },\n "' + ENABLED + '": false\n}\n' assert set(top_level_spans(nested)) == {"[python]", ENABLED} parsed = json.loads(strip_jsonc(upsert(nested, {ENABLED: True}))) From 06aedd4d1624489e8c95225439ce8ec24d9a20ec Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 17:58:45 -0700 Subject: [PATCH 07/28] fix(skills): add pytest-cov and a fuzz corpus to copilot-otel-metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI runs pytest with --cov and requires tests/corpus/; the skill shipped neither. Seeds cover collector config, loopback and rejected URLs, JSONC settings, and redaction keys. Refs #2717 πŸ§ͺ - Generated by Copilot --- .../copilot-otel-metrics/pyproject.toml | 2 + .../copilot-otel-metrics/tests/corpus/0_empty | 0 .../tests/corpus/1_collector_config | 11 + .../tests/corpus/2_loopback_url | 1 + .../tests/corpus/3_settings_jsonc | 5 + .../tests/corpus/4_redaction_keys | 5 + .../tests/corpus/5_rejected_url | 1 + .../experimental/copilot-otel-metrics/uv.lock | 189 ++++++++++++++++++ 8 files changed, 214 insertions(+) create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/corpus/0_empty create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/corpus/1_collector_config create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/corpus/2_loopback_url create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/corpus/3_settings_jsonc create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/corpus/4_redaction_keys create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/corpus/5_rejected_url diff --git a/.github/skills/experimental/copilot-otel-metrics/pyproject.toml b/.github/skills/experimental/copilot-otel-metrics/pyproject.toml index 10932a0fc..960e0748b 100644 --- a/.github/skills/experimental/copilot-otel-metrics/pyproject.toml +++ b/.github/skills/experimental/copilot-otel-metrics/pyproject.toml @@ -7,10 +7,12 @@ dependencies = [] [dependency-groups] dev = [ "pytest>=9.0", + "pytest-cov>=7.0", "pyyaml>=6.0", "ruff>=0.15", ] fuzz = [ + # Atheris ships manylinux-only wheels; keep separate from dev so uv sync works on macOS. "atheris>=3.0", ] diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/corpus/0_empty b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/0_empty new file mode 100644 index 000000000..e69de29bb diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/corpus/1_collector_config b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/1_collector_config new file mode 100644 index 000000000..3c3ac9e92 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/1_collector_config @@ -0,0 +1,11 @@ +receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:4318 +processors: + redaction: + allow_all_keys: false + allowed_keys: + - service.name + - http.url diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/corpus/2_loopback_url b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/2_loopback_url new file mode 100644 index 000000000..078c1133f --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/2_loopback_url @@ -0,0 +1 @@ +http://127.0.0.1:3000/api/dashboards/db diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/corpus/3_settings_jsonc b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/3_settings_jsonc new file mode 100644 index 000000000..7780b30c8 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/3_settings_jsonc @@ -0,0 +1,5 @@ +{ + // trailing comment form the upsert must preserve + "github.copilot.chat.otel.enabled": true, + "github.copilot.chat.otel.captureContent": false /* inline */ +} diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/corpus/4_redaction_keys b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/4_redaction_keys new file mode 100644 index 000000000..69235ee19 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/4_redaction_keys @@ -0,0 +1,5 @@ +service.name +http.url +prompt.text +gen_ai.completion +0123456789abcdef0123456789abcdef01234567 diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/corpus/5_rejected_url b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/5_rejected_url new file mode 100644 index 000000000..1b8bcb5b3 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/corpus/5_rejected_url @@ -0,0 +1 @@ +https://user:secret@example.com:9999/../etc/passwd diff --git a/.github/skills/experimental/copilot-otel-metrics/uv.lock b/.github/skills/experimental/copilot-otel-metrics/uv.lock index 892a68d94..e249a9e2e 100644 --- a/.github/skills/experimental/copilot-otel-metrics/uv.lock +++ b/.github/skills/experimental/copilot-otel-metrics/uv.lock @@ -29,6 +29,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ { name = "pytest" }, + { name = "pytest-cov" }, { name = "pyyaml" }, { name = "ruff" }, ] @@ -41,11 +42,131 @@ fuzz = [ [package.metadata.requires-dev] dev = [ { name = "pytest", specifier = ">=9.0" }, + { name = "pytest-cov", specifier = ">=7.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "ruff", specifier = ">=0.15" }, ] fuzz = [{ name = "atheris", specifier = ">=3.0" }] +[[package]] +name = "coverage" +version = "7.15.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/c3/4f2195f512fb172aa425a8803a874b2baa9ba7f80ff7b6080998761fc701/coverage-7.15.4.tar.gz", hash = "sha256:0548198fff07ccf4faf469520bce1c2eceb1ce3e62891921138dec10907f9d00", size = 936952, upload-time = "2026-08-06T13:50:24.442Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/66/edcec7d7a0b524aa8923e22925fde6fe50ce005a113dca13ae1581455c4c/coverage-7.15.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbac5abad70df71019988f83f26ac7092ff2642975def4429e98dc7585ef3490", size = 222367, upload-time = "2026-08-06T13:47:15.578Z" }, + { url = "https://files.pythonhosted.org/packages/e6/c6/ab8de429e2e8548faf58ec7e1674a4ce00414b4113942d3fe87109cf0f68/coverage-7.15.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:357a173465c7ce028d07a95cc2b63b5bf59f50ecdd5ad75c5cbb78ada984048e", size = 222874, upload-time = "2026-08-06T13:47:16.961Z" }, + { url = "https://files.pythonhosted.org/packages/be/c4/3b7b49587e8a6b9af79b3eb468d443d6042b6d65b47aa26586846a0d6566/coverage-7.15.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:21b803935e2efc3acebe9697197a294fccf5dc4e5382bd6369542ff7a7d2a1d7", size = 253287, upload-time = "2026-08-06T13:47:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/fb/65/ec03b743a2a229c72cc1eff3e57be9d3564e9c6b4d5aba2d70744a3fc0d8/coverage-7.15.4-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7a2b580774a4786c1053157c0165e04476e03ff293993d7c148eee784a94bae6", size = 255199, upload-time = "2026-08-06T13:47:19.765Z" }, + { url = "https://files.pythonhosted.org/packages/41/4b/5163729e4b6582d61975cfd3ccab45b4ec53e21cf156d9941cb025188468/coverage-7.15.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9464451c4efffe8d47ace5a540b10b0dc10e879066290f8600872b7f54a419d", size = 257308, upload-time = "2026-08-06T13:47:21.206Z" }, + { url = "https://files.pythonhosted.org/packages/86/08/2167a0f08fb87d702fa423a48578a32865464b7c9e1db3911ad7812ab414/coverage-7.15.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de602f34123c2f4af1c1869c6dbbbd60da6d5983bf01937367295d135cccbfce", size = 259268, upload-time = "2026-08-06T13:47:22.503Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e5/68eebae3053dbd48508edea559c21b23fbdf3460784f91370c83a86a6acd/coverage-7.15.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6879ded16a27f3eeca19b900c147e81616e7054db451471a611b2755ee5249f7", size = 253392, upload-time = "2026-08-06T13:47:23.88Z" }, + { url = "https://files.pythonhosted.org/packages/1a/46/fd4ced40a2b691c774e515c9b69500bfa64c7960b67fcee4b2f6fad97fc3/coverage-7.15.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:986be58c3ab54aae8d3496a6225eea74f760fdbe739b38bd442c7e8d133aa53b", size = 255001, upload-time = "2026-08-06T13:47:25.469Z" }, + { url = "https://files.pythonhosted.org/packages/53/25/ae2e5fa710bb6957a9aadeb9e3598d3b3e4af6587ce857ad42e8639a3f30/coverage-7.15.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6103639613fe6c1e989082948419bc77a2d26b6c825c99d7fad25f7d3d87afc", size = 253061, upload-time = "2026-08-06T13:47:26.845Z" }, + { url = "https://files.pythonhosted.org/packages/d7/31/67ddc0365db2c6e93ac8580bc4bbc50f65273262f973f63ebcdbc15c0495/coverage-7.15.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d3af93dddb5659276c63bc16ac6466ac2033a70ca816097bbc06345b8ccdf571", size = 256831, upload-time = "2026-08-06T13:47:28.217Z" }, + { url = "https://files.pythonhosted.org/packages/f6/78/82b8fd18f57fb13f12d98fe874995bb2c4f9f17be8aff762c426323fdb96/coverage-7.15.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b10075e5421d04265766a6d1dac809bbeb8a946fbb23c8f82c227409b2190719", size = 252781, upload-time = "2026-08-06T13:47:29.712Z" }, + { url = "https://files.pythonhosted.org/packages/0a/eb/6c74ef4dd12b252e573c49bdef9e2ac265bf3dbb79b8d7feb3266e084e9e/coverage-7.15.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a67a9f78b2942d87ba8ce3059c642164d2aedd65337377fb52fe9803656bc5c7", size = 253692, upload-time = "2026-08-06T13:47:31.192Z" }, + { url = "https://files.pythonhosted.org/packages/5a/66/eb9aed1c3fd2d36ee00eb173f434b14fa607fc056739c9a89ff4244010ea/coverage-7.15.4-cp311-cp311-win32.whl", hash = "sha256:69484d1aca26e322e1c3ce03f09341e84524ababad2d7202161738d83cc9f82e", size = 224461, upload-time = "2026-08-06T13:47:32.572Z" }, + { url = "https://files.pythonhosted.org/packages/e2/6d/81fa4161dfb3ed9d74e40d58647eff83a56b7612e78352581280fce2f477/coverage-7.15.4-cp311-cp311-win_amd64.whl", hash = "sha256:63fd6fcd1dd6e158f7eb78606e72933b3f6d01e7b747f99c6c12d764307a0fdc", size = 224937, upload-time = "2026-08-06T13:47:34.205Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c1/d8dacf683c6cad3cf85ce68fd3774a6774ec402128822fdfaed920f11e6a/coverage-7.15.4-cp311-cp311-win_arm64.whl", hash = "sha256:ea82116c9893fa89e929b7f197ee5a1950a76e91cc5c85ba503fc02379d04890", size = 224479, upload-time = "2026-08-06T13:47:36.118Z" }, + { url = "https://files.pythonhosted.org/packages/1d/48/bc8d4ba7b37551a767bd863f15b3f80182b271c2f55975356f5f7dbe94c2/coverage-7.15.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4fedd1f7f428f9fe83b1ead5e7cc87a43427be31aadafbac3ac0636dc7abb22", size = 222543, upload-time = "2026-08-06T13:47:37.562Z" }, + { url = "https://files.pythonhosted.org/packages/20/dd/88d6f83f1fffc974a3691a34a97951c5b12df7512a6782c5963883cbc058/coverage-7.15.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:37e2f0cdf58e2e1fed4e4d5a8f8786ae2f7eb80b478016876667dc4a01d60a97", size = 222905, upload-time = "2026-08-06T13:47:38.927Z" }, + { url = "https://files.pythonhosted.org/packages/bd/5c/54ee0d4748585bb0acab9891cd8d92f2d3593165b4e59fc9de113bfb3140/coverage-7.15.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fb55d0e70bb15f2e81477613627286581414693d74ac7963c93a790dd453ca9d", size = 254407, upload-time = "2026-08-06T13:47:40.488Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3f/f0642a372f494bd0d7dad3b497083b910194a5f1c88be2c94fef707c3b59/coverage-7.15.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:899b9da30f3c6c336566e3707495bb23e8302d39d862f01fa78c48b99b9437e2", size = 257145, upload-time = "2026-08-06T13:47:41.931Z" }, + { url = "https://files.pythonhosted.org/packages/71/17/8b46d0ed68251016002ec972c8fc0119961a765d0984cafb8bf317c43758/coverage-7.15.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d15715e8c46552827e5e4f30a35575a2dbcad14454cf3284c54483946bd16931", size = 258257, upload-time = "2026-08-06T13:47:43.527Z" }, + { url = "https://files.pythonhosted.org/packages/30/b8/8498a0e72d0adbe15477dd07463d2b3bb2c9f6a4815e8589e50939e2c3ae/coverage-7.15.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:002a438859f7b430bc99afeaf01a6d187dad1d0dc907b64cdeffc632a5db8fd8", size = 260517, upload-time = "2026-08-06T13:47:45.121Z" }, + { url = "https://files.pythonhosted.org/packages/41/e1/7dce19c3bdb1e3dd63e769508216500edad81bd5f69a26d724e32aceaf78/coverage-7.15.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4193a04b518f7968f3099755f5509ee7cccc6dc2b92a6b14841934d22e222c9", size = 254785, upload-time = "2026-08-06T13:47:46.541Z" }, + { url = "https://files.pythonhosted.org/packages/dd/b1/e1494703c675a2561723cd9b89f45c9168782c31280c611b1f767851e57c/coverage-7.15.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e98dcc55d572b38e69d117da7e8e8efb8500f1f5eaf81ecd460a63220790b839", size = 256176, upload-time = "2026-08-06T13:47:48.155Z" }, + { url = "https://files.pythonhosted.org/packages/73/76/a5629d270fb638a43a4b10466f51e2f49d532c1aa4da2913cbbb150bbe0a/coverage-7.15.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:af6c538498ce66c10d3fd541c2a8d5b03da5850355add34e6cba564210cb9e72", size = 254321, upload-time = "2026-08-06T13:47:49.757Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4f/9c44447218435d5766b911534f9d798144a5560f85e9a54ebe5f3f5d19f9/coverage-7.15.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1d10025d96ea89fc2f73714dbc4cbd433fe012c1ac9e23f895d7728b238b6e52", size = 258390, upload-time = "2026-08-06T13:47:51.248Z" }, + { url = "https://files.pythonhosted.org/packages/de/36/c1e127616fb3fa18a9ff71e76c417f2fd7424332a4870015ac224ef4c039/coverage-7.15.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d802e1947603162ded419bff83ac7489820355d2b856dfb09206574e3a37ac0c", size = 253894, upload-time = "2026-08-06T13:47:52.816Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b9/fdb92c8ae7a8bb9b850cc253b7b3b9c8526f68130002048b5671cd510d09/coverage-7.15.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c2de40895718f91951b86712b4c5b694acaf9a0a49be13874896f599a1eed3f4", size = 255763, upload-time = "2026-08-06T13:47:54.296Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/a7d51b2587c7bdb76e71b0896d2565bf7d60436b5122fc83e511adb1f7cd/coverage-7.15.4-cp312-cp312-win32.whl", hash = "sha256:5c3431b2161279b7db5c2a1aa58ae02e5cb8c3c42d93a5094be3f5537bd5b11b", size = 224597, upload-time = "2026-08-06T13:47:56.074Z" }, + { url = "https://files.pythonhosted.org/packages/49/b9/5c5f80cc55f5acaaca6dee677626bfcec8c87204a7809b438b08e84f4571/coverage-7.15.4-cp312-cp312-win_amd64.whl", hash = "sha256:6befeab5fb2b51c958ca4ac6c5d141a1e8240f4f76e46350f1911963deda49cd", size = 225135, upload-time = "2026-08-06T13:47:57.52Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/2a4561f89ff6bf7c925c287d0f2cce8bdf139c3a33735c87e3203401cf94/coverage-7.15.4-cp312-cp312-win_arm64.whl", hash = "sha256:67bc345491ab55b837277d76f5775d057e8c7f1ac44d890d8c2c82adde258c6f", size = 224515, upload-time = "2026-08-06T13:47:58.977Z" }, + { url = "https://files.pythonhosted.org/packages/f1/84/651a9310859673aaa3b3203f1aa1641ca60fcf2494683e1c9474c7172780/coverage-7.15.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c705b28feb2775dc82a25f1d473a370bc37ff93f5177f4e29ce2425f560f6921", size = 222565, upload-time = "2026-08-06T13:48:00.796Z" }, + { url = "https://files.pythonhosted.org/packages/82/f9/4dcf700137e8af550670f4d74d1b63828ce93e1e2b05e5f10710eb2ea987/coverage-7.15.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ff205ab5e3ecc670f6a4dd19d9cbf12ede53dd41cfc1e15716ec961ea6d314e", size = 222936, upload-time = "2026-08-06T13:48:02.391Z" }, + { url = "https://files.pythonhosted.org/packages/07/4a/612ff1e780b3fbfd637486f542f84adc5503873d8b5d279dec1ffeef9414/coverage-7.15.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5172326e861a38b48b48befca15e0f477a26b283337a33a739c8fed229934e36", size = 253926, upload-time = "2026-08-06T13:48:04.382Z" }, + { url = "https://files.pythonhosted.org/packages/b0/04/d1cff1c2ead4708a6a79c01d3736b6a25bd38a36678398f72a8dd33dfad9/coverage-7.15.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:12b59c90084e3234fb11184886bf4a40f4f16a8c8f867be2e087b81f8e8868d4", size = 256523, upload-time = "2026-08-06T13:48:05.996Z" }, + { url = "https://files.pythonhosted.org/packages/b9/80/d34e13fb4b293cbdb9665838cf5522077b8ad14ef947550631a4bced36a5/coverage-7.15.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349062d66f00b40fa2c1c222438bad25fabf755631b5d82937fe985c8008615c", size = 257759, upload-time = "2026-08-06T13:48:08.036Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e7/2c5fe7636fdb0732fe0f09f308a5b066864078b7fc61f6678e8478554f2e/coverage-7.15.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4256ced708e598e05209bc1a8ab4074e04a51dba4c62fb45926a229af675ace7", size = 259890, upload-time = "2026-08-06T13:48:09.834Z" }, + { url = "https://files.pythonhosted.org/packages/92/28/9689f0858dfff59c2ea688938ab9fa2925631235df67126a42b6c5c70ae1/coverage-7.15.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d80f974b20782d9612c8b4c9beeca867074c7cf4079d1419843fa25a26428b25", size = 254121, upload-time = "2026-08-06T13:48:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/785077c230c157243eb5aa9a26c3be260ecd02001bead54a3cada3df8e03/coverage-7.15.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2e179f19bfe1d31f8eeeaa12990194d761c4f62f0759661000bca6cd8729f40b", size = 255891, upload-time = "2026-08-06T13:48:13.209Z" }, + { url = "https://files.pythonhosted.org/packages/d4/90/e20371b17b40f912f21305c2db2f30efa3de306f7320fc916804872c85a4/coverage-7.15.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8bc16bb47b7679670eceff71d78bfb7d6e5b143f6c2cd117487ec7c75e0d4b78", size = 253859, upload-time = "2026-08-06T13:48:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/05/49/25371987ee459a5f67c0427fb75c74f9358e65f2c71fe75bf41c1b6c5fcb/coverage-7.15.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd685005cd2c4200adfc14cf39a603b9320efab3f18a8f7f156d20c9cc3345f", size = 258011, upload-time = "2026-08-06T13:48:16.464Z" }, + { url = "https://files.pythonhosted.org/packages/30/6e/32e67467f6154bf4f1c4f63b05acc5097cba4237d45bbeeea446b52e8ac1/coverage-7.15.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:337399ad2c93b3acd2a937627dae8b3e86b66707cd3d3e856347999aadf1ef8d", size = 253676, upload-time = "2026-08-06T13:48:18.493Z" }, + { url = "https://files.pythonhosted.org/packages/03/c1/8b24192e89286399765155251f99ee9f070a9d637109018ac23d99b99f6f/coverage-7.15.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:96e257121228ec5cd2bb919276e94ac11074471bc37d68dbae0e8308cce15fff", size = 255453, upload-time = "2026-08-06T13:48:20.057Z" }, + { url = "https://files.pythonhosted.org/packages/16/6f/8b41ebdf67c87854e17c035336a90f1cfbad0c14c2a584301be6ff148718/coverage-7.15.4-cp313-cp313-win32.whl", hash = "sha256:c65a9e0dfc6143491879da4e13b5e30f8be192055de508d737fb14601edbd22c", size = 224605, upload-time = "2026-08-06T13:48:21.655Z" }, + { url = "https://files.pythonhosted.org/packages/e0/e2/2946c7f0b42b152ecb21ff1bdad72e3d301e790c0c487e4a86e8c9f69347/coverage-7.15.4-cp313-cp313-win_amd64.whl", hash = "sha256:2ff8f5e9b8f7a94f0c11c45631eee103dbcb7d63274edd12c56efe1be690b3b4", size = 225148, upload-time = "2026-08-06T13:48:23.376Z" }, + { url = "https://files.pythonhosted.org/packages/9e/83/3f4a69957f48ae7a0aba76c34743f88963d607b19e03f3f8e66f91cae0f9/coverage-7.15.4-cp313-cp313-win_arm64.whl", hash = "sha256:6e0a8a5083b096487d6cfced94cdd514d8f5db6f113610fb36c0620edb1028cf", size = 224536, upload-time = "2026-08-06T13:48:25.117Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ac/748cf29eeb2d6be34a3176ce26a4f49e38085ee08e8935f05f6f26ed7e0f/coverage-7.15.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:770e9325ab5ea6d56f77e59b29ecfe0ac20b57a82a601876f90494a4dda0386f", size = 222608, upload-time = "2026-08-06T13:48:26.806Z" }, + { url = "https://files.pythonhosted.org/packages/0b/02/1abbf5c984677b0aa439cdacaccbf38d248939d8ef8fe1cc7a50d73edb77/coverage-7.15.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d12b33a3a50a1676b7784dc8d00a0c6d66a9f2add4b85a041c19b6a7e53ef23c", size = 222940, upload-time = "2026-08-06T13:48:28.432Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e1/ff8f9f53d9fcf586125b55d0b1f04ec1c14955fee41e83d5814bee141bb5/coverage-7.15.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5669c8378ebde86f5def7a25d29586631b58acc27ffde04399f678f3dfc6e082", size = 253985, upload-time = "2026-08-06T13:48:29.995Z" }, + { url = "https://files.pythonhosted.org/packages/a1/26/595759762e514e81be1d7d01ed03444303bcd152226a6529998d253f9201/coverage-7.15.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ff97a14362eef486483ed44042ca2027ea257df6ff768e62358ee0c9776925ac", size = 256492, upload-time = "2026-08-06T13:48:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/b79aabac54d482be23b5fcdd4f4662bff24a78edc4ee29201726929936d5/coverage-7.15.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a325e815318638aed1655d9c06e6d7c2d3d46c09231ce988070428a8762d734", size = 257837, upload-time = "2026-08-06T13:48:33.186Z" }, + { url = "https://files.pythonhosted.org/packages/09/0f/bf7f297885a5bf6fd71e5782404e0ff059ca09e8711ceb3a08544abde45a/coverage-7.15.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:474223409d88eb20d2d6a0d37ea60e8647a65a90cc008dc1f0410af5f64f1e0d", size = 260152, upload-time = "2026-08-06T13:48:34.75Z" }, + { url = "https://files.pythonhosted.org/packages/fd/f1/296744e854ff8368542343457414380465e9ceefb9192342feb9d3bc461d/coverage-7.15.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f2f62ae3cd189dd2e13aece758c57b3eecbd27be070dbd4cbd10936049e5dbf", size = 253978, upload-time = "2026-08-06T13:48:36.434Z" }, + { url = "https://files.pythonhosted.org/packages/55/b0/bbdb2e9057493e66220a2e149ca2d301ba0e3a58a83bd6b90de9826d16f3/coverage-7.15.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39ece820e29e0a2ba34b3ecb3be83c27e997eed8926f2ba6fe7ce7a0bda5843b", size = 255846, upload-time = "2026-08-06T13:48:38.317Z" }, + { url = "https://files.pythonhosted.org/packages/96/e4/38015b2b6d21258713bd17e76b59d033b191efb5703589cffd037dfbca20/coverage-7.15.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f21b56dcace11dfe013014201f577dcd592b2a9b72182d930361b47cf6f73f25", size = 253808, upload-time = "2026-08-06T13:48:39.993Z" }, + { url = "https://files.pythonhosted.org/packages/0b/64/0d515c1e60ee6fbfd1a0e79c07cd87d388a233b7adc37758735677203808/coverage-7.15.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:93a3a0b662abcc10c73a47cbc72cd60f63618d6989fb2d1286e50eacd974f303", size = 258081, upload-time = "2026-08-06T13:48:41.971Z" }, + { url = "https://files.pythonhosted.org/packages/91/71/04d9e7a3642146c6351338aef4ef85ab11dbbb54744c13245caba1aad1c0/coverage-7.15.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:141fae2cabf5569b782c10afc4c850ce10f618c13f8db54765cba99cc839da1f", size = 253624, upload-time = "2026-08-06T13:48:43.731Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a7/6c28b74c81ebff66987b0e2522ba5cffa3e90b0c33cb6a2eb264d4ee8cf1/coverage-7.15.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:81294c7e6ab30c5f74c0353b11b2fd6320e72d9bee6ac73b357caa8b916323a5", size = 255280, upload-time = "2026-08-06T13:48:45.58Z" }, + { url = "https://files.pythonhosted.org/packages/52/af/bc19996a7014b98d7bbb0f0939453c67074af65784a3aa16a789a07381fa/coverage-7.15.4-cp314-cp314-win32.whl", hash = "sha256:7bbd7d6418e0dab31a206af5203bd43ae36edb8e7fba1940b055d3e9249290d7", size = 224768, upload-time = "2026-08-06T13:48:47.525Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/219484e476d6e101ba0a444852579e05f5b75c37c611a42ed1190f73ef62/coverage-7.15.4-cp314-cp314-win_amd64.whl", hash = "sha256:f0204ed122758782970526057093f448051a39db9d810d4e344bb87a3546f425", size = 225259, upload-time = "2026-08-06T13:48:49.513Z" }, + { url = "https://files.pythonhosted.org/packages/b7/66/fa77daf4e383e5f776dac62c2409b6af81910ae6fe326bd5170dba74cc63/coverage-7.15.4-cp314-cp314-win_arm64.whl", hash = "sha256:9e71e7bc71c686a123347ae47a0de33a175e797a85bb57b791492adf4eec8ed8", size = 224684, upload-time = "2026-08-06T13:48:51.235Z" }, + { url = "https://files.pythonhosted.org/packages/58/5b/f03bf0ce362bbf3f785fa5219620d00778d4ac6fc9e407734828e9c672f6/coverage-7.15.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7c922735321eef3f87c280a3d39afff6b646723a2880b862cda4ac7a093b8aa8", size = 223338, upload-time = "2026-08-06T13:48:52.896Z" }, + { url = "https://files.pythonhosted.org/packages/0f/76/e77d0ae22501831cc9f92193e8a957a5caa1dd177f90a6d1d9b106242d92/coverage-7.15.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f41c17c4668a655ce96d090d8d5ffdc24ef64b5a02f9753884d08483e8a4a41a", size = 223609, upload-time = "2026-08-06T13:48:54.688Z" }, + { url = "https://files.pythonhosted.org/packages/82/1a/b1f089da8d38ac612fa2dd6dc7f4a1a7657d12f3e261d2996edd3a838d0b/coverage-7.15.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:46822e9b6ff1c6a72b518c162c44a8f45a61a1d609c51084bf5b16c023c5037b", size = 264970, upload-time = "2026-08-06T13:48:56.403Z" }, + { url = "https://files.pythonhosted.org/packages/bf/31/e66d98d6e9c7fcc88470f1e234eaf6b1950dc0dfbf797f7282c1c861da24/coverage-7.15.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d6f4955b73b5445271379a59e3792b0d978f42d4a01e0cf7a67d9c33a3bb0a5", size = 267088, upload-time = "2026-08-06T13:48:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/ae94eb2c541add426378408379f233591e069040b1e2cdb33df9498a0682/coverage-7.15.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3fc9e047706fb4a9abb54f719d3aa643e80e5bb3818182c40aee01ac0f0247ba", size = 269508, upload-time = "2026-08-06T13:49:00.42Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c7/88a10694a1c6a213569766aba9f25847b28155d4ac731b13226db216356d/coverage-7.15.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05e491d4f3165d62d4f5c8fd48dfeabf2ae8f42cbbd484319af33ea851b78982", size = 270629, upload-time = "2026-08-06T13:49:02.234Z" }, + { url = "https://files.pythonhosted.org/packages/b3/34/d8b8232e5e55169933b59aabcef2fedfa4b9d8897361bb80fcbda146505f/coverage-7.15.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:226c66e80ec0598d3b9b4874123df167ccca342aca8714f77cac6829688ee09c", size = 264043, upload-time = "2026-08-06T13:49:04.102Z" }, + { url = "https://files.pythonhosted.org/packages/7e/35/58b009dbf8c471c7224716478b9fed4a7e1af15320e1ed41660978504663/coverage-7.15.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac41cc14bebda0dbfb0628036b7f75706935c95bcc07fefe9a0f93614aa60a57", size = 266963, upload-time = "2026-08-06T13:49:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/62/aa/57fbda1b42c892968273c56b6ee9dc0f1310850859230a507bc7873b1f65/coverage-7.15.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8af623e5cd92080acddd02b38f2f406a2c3a0893c38950b211890361448fbf26", size = 264569, upload-time = "2026-08-06T13:49:07.706Z" }, + { url = "https://files.pythonhosted.org/packages/98/8a/360e6e7f24d477b7e889703af0afa878d15b6d4d8d2a822b2835c169a879/coverage-7.15.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07545711d4f0f32852a18f18ad11f76f0109909d09e78b9008b4cfc67e829429", size = 268299, upload-time = "2026-08-06T13:49:09.587Z" }, + { url = "https://files.pythonhosted.org/packages/4e/89/6f701261aee21b6b5fa8f7872229406dc917e125069448292223bf213606/coverage-7.15.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a0865421cfdc53654b342d515e5a233187590882d20b95752150e53f65460017", size = 263413, upload-time = "2026-08-06T13:49:11.604Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0f/6f04036edc260ed425af83e834f627fad48941ce97b50bfe6edd8b6fa623/coverage-7.15.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:460115e32ee40566476db5048f9bec1e842c127ad8e6f8be745aad3ac9cbc839", size = 265725, upload-time = "2026-08-06T13:49:13.38Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ce/d19b5d4d5c49a7bfb925fd74310fee7d28bc99520ac3367ccbc54e662518/coverage-7.15.4-cp314-cp314t-win32.whl", hash = "sha256:cbde877ef9dd7baf272b9bfef2b8a25edd45d9170fc326951dd20eb480335e85", size = 225079, upload-time = "2026-08-06T13:49:15.265Z" }, + { url = "https://files.pythonhosted.org/packages/26/bb/7aa1b3b173faee0679037ca950bbbe1247273656697994d8d13f80f8d4b4/coverage-7.15.4-cp314-cp314t-win_amd64.whl", hash = "sha256:3da9e92d1c551fd7563833e9ade686efb0c4b7363ab7681a94283958c950bf5e", size = 225911, upload-time = "2026-08-06T13:49:17.279Z" }, + { url = "https://files.pythonhosted.org/packages/81/1c/4ea9e47426d80038d9222db3c4534cb6021a74b237d3ff97ffd33b6600dd/coverage-7.15.4-cp314-cp314t-win_arm64.whl", hash = "sha256:3a54f5a0d85050c73a38f6793090ee83974531e67fe5e57a1da9bee11398aa5e", size = 225219, upload-time = "2026-08-06T13:49:19.293Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c4/dc5d2ac8f9142e7ec7de66e7bf0591db29d78955a040bd915870d9c0e657/coverage-7.15.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:2c9872e4d9dc5d3cf616bf4b382f5a00359305a5be666a3dd0b5cdb4e49597f9", size = 222604, upload-time = "2026-08-06T13:49:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/70/39/33e63df81fe2ee100897451841c821467635923e58e37c6bd4b46dd8106c/coverage-7.15.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:e101dbb4b9b72f0cddd8cdc8c9c5b47f456766f5e0ac82dbfb75e5c55409b78a", size = 222944, upload-time = "2026-08-06T13:49:23.187Z" }, + { url = "https://files.pythonhosted.org/packages/99/1f/ef3ffb5557febc75a0d97aa459d0266d7d741110265121cc6d8539343d44/coverage-7.15.4-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7d1abebdb047729e852b9c77a00497dfbeb11eb3a117e037d7dbc3ac8e5f5c54", size = 254050, upload-time = "2026-08-06T13:49:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/6f/f5/1f0f6f77698c3601ca0ae7431e34b24c62ca2f06fecb23b73ed1f651d2be/coverage-7.15.4-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d28a4a899354d0ea6214cc59b4fa19eefbce1b9ff1688ab579acf49e894bd3fb", size = 256967, upload-time = "2026-08-06T13:49:26.896Z" }, + { url = "https://files.pythonhosted.org/packages/03/7a/2ed9bed79925f4367c83c77f66a89e5ca7229c288d2d19ad5f36d1ca0070/coverage-7.15.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffb3c2aacea411cc7e1d27712490c11108e2de1d39019ae32915493a59a8b9ed", size = 258587, upload-time = "2026-08-06T13:49:28.692Z" }, + { url = "https://files.pythonhosted.org/packages/45/8c/fa34044f71b7cc4ecb6da9c2408770959b0591fa9b5fb6fb6bca38f94298/coverage-7.15.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a9447978a92f405d301123cfd39ff49895490efb769a758fe2734c7f631bf8ce", size = 260785, upload-time = "2026-08-06T13:49:30.472Z" }, + { url = "https://files.pythonhosted.org/packages/4f/54/d5727ce36b4524a7394ab9f5f1df378e1f23affcdab01037dc8655185cc7/coverage-7.15.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:050467a7983b8e2fe7dd41a78bb30c3e7f8c0b8cafda14b1c46f8b5e3cf2dd3c", size = 254545, upload-time = "2026-08-06T13:49:32.271Z" }, + { url = "https://files.pythonhosted.org/packages/dc/e6/6e3783e576719590194bdffb6dd6d85490801785b7c331e35a245d8cb8b5/coverage-7.15.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:d003b7a5708ddad5c206c79607a6b92abb6fc13c57d99d8a4468cc03a2941ced", size = 256682, upload-time = "2026-08-06T13:49:34.089Z" }, + { url = "https://files.pythonhosted.org/packages/dc/f2/bacdbde18b69ed2de424fcf64d9fb0a4913753d4f0eca8bae9daad69f4bd/coverage-7.15.4-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c38efe30fd74e5c19e9433f11fb1f5dc9c6522770971b7c6145bbaa413dc8800", size = 254560, upload-time = "2026-08-06T13:49:36.052Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a3/1fb927196e3477c1b48831169ab58ba08f451ba87ae311ff1de68b26a616/coverage-7.15.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:1f4f826d70f772ab8b0c052329580d7fe8b8abd191e4ce0c8f81aec6614665d3", size = 258792, upload-time = "2026-08-06T13:49:38.01Z" }, + { url = "https://files.pythonhosted.org/packages/41/58/30d4c149c69053de0edfe325614c1d28d508f62b1783e0e4a234d2e49136/coverage-7.15.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:4a4bf917c9953f57c957be31c1cd504e3bd2f34d4a352b9d391a3025336f6768", size = 253968, upload-time = "2026-08-06T13:49:39.934Z" }, + { url = "https://files.pythonhosted.org/packages/89/e4/77f639371b918aad30dda4051f95404b43578f7f2e2f87ba73e02ed1ff37/coverage-7.15.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1c9bf40ebef178a45192c75c4964760bb261b0e6ad725da5fc4c93f674f19753", size = 255893, upload-time = "2026-08-06T13:49:41.825Z" }, + { url = "https://files.pythonhosted.org/packages/5c/62/13be29b3ddab35f14c87967a4820a05106d2a3eccb4fa4ff550bf30b75e0/coverage-7.15.4-cp315-cp315-win32.whl", hash = "sha256:43619d04c3671792d2c4706ae8bf45e265dc87bbd4078189ef8b847ea1e74be2", size = 224768, upload-time = "2026-08-06T13:49:44.08Z" }, + { url = "https://files.pythonhosted.org/packages/a1/70/af0c6be0f964af6954f6b74bc109b0dbca02824696d2520fb17fe1ab06e3/coverage-7.15.4-cp315-cp315-win_amd64.whl", hash = "sha256:be619439dbcd31a2eab10b32de9fff62c26ed4bab69dc32b8363fdaaa0882809", size = 225242, upload-time = "2026-08-06T13:49:45.899Z" }, + { url = "https://files.pythonhosted.org/packages/4f/2d/f3bd3aab899fc9efc18b53133ee68f5f98574ef480649b23e12962226387/coverage-7.15.4-cp315-cp315-win_arm64.whl", hash = "sha256:def597967dafc2e8d97c9097ea453c464e0bb8ed38f193a43070f10dc623bb6d", size = 224674, upload-time = "2026-08-06T13:49:48.322Z" }, + { url = "https://files.pythonhosted.org/packages/f5/ca/f69251cd63eabc6438321aea22148754cce758a26bde07dd490e3fe7cfc5/coverage-7.15.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c7dbc748ac8a1e3e59a2b28bea47675e6e778081dbbf081bde0d75def2fcbe1d", size = 223333, upload-time = "2026-08-06T13:49:50.293Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a7/037b53b2885b0d8447064432491a4d5a1014cd9f97a594d53acd0c04541a/coverage-7.15.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:2413074a5ecbb61a01a7888fc72db0ca324d13588c5b38bc0dd8564cdcdfea26", size = 223630, upload-time = "2026-08-06T13:49:52.637Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/152b8a4779ae90da11bb24f7467df8a59f0be48a5c52acb856325ca48289/coverage-7.15.4-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4e6f6f632b7b2f714bf7a1346e8f97b650ee71f3c298aaad42a2ab60f0f07645", size = 264489, upload-time = "2026-08-06T13:49:54.52Z" }, + { url = "https://files.pythonhosted.org/packages/10/2d/84b4b9e0e1dd6528a51920ff7031f35b789382e467a28ec6a5a578cb8812/coverage-7.15.4-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8df457da2249d3c75ca2e5e835d59c725abfe92d27fdff6cd99eed85b51d5e9a", size = 267567, upload-time = "2026-08-06T13:49:56.721Z" }, + { url = "https://files.pythonhosted.org/packages/53/fc/ba01cc25299f9f8a2c8b02d3b28c53f3543d9fbfbe4e74fa2760b48f163e/coverage-7.15.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:050f66a08805acb5b8a23c6d4a517b1ecf82c08e81ed0e4bd727df065e5c6624", size = 270123, upload-time = "2026-08-06T13:49:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d0/db2647cbf40b14f8c308f94ff7bf89c06d564e59f396906edf50086ec788/coverage-7.15.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1587fb771d1ccceef708fdde1e5af8c7ed24b486b61d13a321acb7d8145390aa", size = 271107, upload-time = "2026-08-06T13:50:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/4d2d17924552c458bb4f77dd631f0e3bc92fbbdf2d2d916cd4b33bbfd5b1/coverage-7.15.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8b4f1c3a69ca580f3fbd6b2046915f536d7f586874f25c1bb23add2a3c88d50f", size = 264955, upload-time = "2026-08-06T13:50:03.023Z" }, + { url = "https://files.pythonhosted.org/packages/ee/de/dc010c7a3691f396d93bbc26bfcafa1c2a3a351cd520470f15faf5795bd5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:ffb58d7eff5b7f6ecc6fa21d6288ab7f968a212cb67d682c269c09b9eba3b66f", size = 267949, upload-time = "2026-08-06T13:50:05.557Z" }, + { url = "https://files.pythonhosted.org/packages/78/ea/dc96a11375e83c045c2f7c61fb6918277cfe9401db7c0f7b1d111a84b2e5/coverage-7.15.4-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:d9df165544774574ee004b953023d1bebada1894a80b1052a43d798b0f676e67", size = 264421, upload-time = "2026-08-06T13:50:07.612Z" }, + { url = "https://files.pythonhosted.org/packages/c8/86/b77131a0f9503ce461cd577076147d7a9040f0c5dda772686f729e2cc9cb/coverage-7.15.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:f9de0a24a4079b53e523b5c5e2c5945ec251ab486652659955187cf255a259bc", size = 269121, upload-time = "2026-08-06T13:50:09.58Z" }, + { url = "https://files.pythonhosted.org/packages/24/24/944bc35007862955e7ebf05754e645419dcf5d7526c52735cfa2715e8ebf/coverage-7.15.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:150089274bdc9f940628552cb92844e0223c987f1902ab8efe9f45a2ec758d88", size = 264565, upload-time = "2026-08-06T13:50:11.722Z" }, + { url = "https://files.pythonhosted.org/packages/c7/cc/a3bb9f93e7e740659163e2ea584f8196ddcd2c456a5dbe15f6c50105fec1/coverage-7.15.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:a58a94fed5da6997d258e8f7668c1e195fbd04a691d781b7558f1e468f9e68bc", size = 266522, upload-time = "2026-08-06T13:50:13.786Z" }, + { url = "https://files.pythonhosted.org/packages/49/dd/e0e40f3560d878d888c580698ff5ad1179f5e1c3ac949684ef66b41a3817/coverage-7.15.4-cp315-cp315t-win32.whl", hash = "sha256:ebd5a6d8466ff30836572f3ba2cae8a5e8f85029b1c6d5e2ed338dc472a5166a", size = 225068, upload-time = "2026-08-06T13:50:15.825Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7e/37732ea80eebc30e976e4cdab15c190bc42d96959a42e38ddf6f8c60468f/coverage-7.15.4-cp315-cp315t-win_amd64.whl", hash = "sha256:288bde2a2d7ab6b6c2d7252fcde8b524387f2d970bdba9658fc6f8bbcaef0f9b", size = 225895, upload-time = "2026-08-06T13:50:17.928Z" }, + { url = "https://files.pythonhosted.org/packages/c6/08/1e00f7923eaaba45fb3d51dd794125fc766304b1df264f3a9c6557bfb30e/coverage-7.15.4-cp315-cp315t-win_arm64.whl", hash = "sha256:68be5e1de60ff13c9095bbec0e5a7fa45b33b101752215b91345ea1f61c4a278", size = 225213, upload-time = "2026-08-06T13:50:19.981Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d9/e70c286c979378f061d8266e279b686ab0b0b688e1fe0af864684f23a77d/coverage-7.15.4-py3-none-any.whl", hash = "sha256:964730a1e9de9c0cf11be6a1a3c79ce419c34882842abd256086ba4698705e84", size = 214332, upload-time = "2026-08-06T13:50:22.192Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -98,6 +219,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -177,3 +312,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, ] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] From a8251a22c7d34dab1aeab987d0aba3a9b25affed Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 14 Aug 2026 18:16:52 -0700 Subject: [PATCH 08/28] fix(scripts): drop the hook-present guard from the packaging contract test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog ships no hook manifest after this change, so the guard could never be satisfied. The contract under test is that the hook root is never allow-listed, which holds regardless. Refs #2717 🧰 - Generated by Copilot --- .../tests/extension/Workflow-PackagingContracts.Tests.ps1 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 b/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 index e87baa69b..200cbed38 100644 --- a/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 +++ b/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 @@ -1029,11 +1029,10 @@ Describe 'Extension packaging configuration' -Tag 'Unit' { $catalog = Get-MarketplaceCatalog -Path $script:CatalogPath $agentIndex = Get-MarketplaceAgentIndex -Catalog $catalog -RepoRoot $script:RepositoryRoot $contributionKinds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) - $script:HookKindSeen = $false foreach ($entry in @($catalog['plugins'])) { if (-not (Test-MarketplaceEntryEligible -Entry $entry -Channel 'PreRelease')) { continue } foreach ($item in Get-MarketplaceResolvedPackageRecipe -Entry $entry -Channel 'PreRelease' -AgentIndex $agentIndex) { - if ($item.Kind -eq 'hook') { $script:HookKindSeen = $true; continue } + if ($item.Kind -eq 'hook') { continue } [void]$contributionKinds.Add([string]$item.Kind) } } @@ -1072,7 +1071,9 @@ Describe 'Extension packaging configuration' -Tag 'Unit' { } It 'Never allow-lists the plugin-only hook root' { - $script:HookKindSeen | Should -BeTrue + # The catalog ships no hook manifest today, so the hook branch above may + # never be taken. The contract under test is that the hook root is never + # allow-listed, which must hold whether or not a recipe declares a hook. $script:AllowEntries | Should -Not -Contain "!$script:HookRoot/**" } From d1107f8a4e26693705f53a71162e722d043622c5 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Tue, 18 Aug 2026 20:47:19 -0700 Subject: [PATCH 09/28] test(skills): extract shared collector config support and split pytest lanes Move the shared path constants, redaction-policy accessors, and OTTL scrub inspectors into a non-collected support module so the fuzz harness and every test module import from one place instead of from another test module. Register a slow marker and deselect it by default so the fast lane needs no container runtime and the runtime lane is selected explicitly. --- .../copilot-otel-metrics/pyproject.toml | 7 + .../tests/_config_support.py | 304 ++++++++++++++++++ .../tests/fuzz_harness.py | 6 +- 3 files changed, 314 insertions(+), 3 deletions(-) create mode 100644 .github/skills/experimental/copilot-otel-metrics/tests/_config_support.py diff --git a/.github/skills/experimental/copilot-otel-metrics/pyproject.toml b/.github/skills/experimental/copilot-otel-metrics/pyproject.toml index 960e0748b..7f759e02b 100644 --- a/.github/skills/experimental/copilot-otel-metrics/pyproject.toml +++ b/.github/skills/experimental/copilot-otel-metrics/pyproject.toml @@ -20,6 +20,13 @@ fuzz = [ testpaths = ["tests"] pythonpath = ["tests", "examples"] python_files = ["test_*.py", "fuzz_harness.py"] +# Runtime-backed carrier evidence needs Docker and a pinned image pull. The +# lane that provisions those prerequisites selects `-m slow` explicitly; every +# other caller of this project gets the container-free suite by default. +addopts = "-m \"not slow\"" +markers = [ + "slow: requires a container runtime and the digest-pinned Collector image", +] [tool.ruff] line-length = 100 diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/_config_support.py b/.github/skills/experimental/copilot-otel-metrics/tests/_config_support.py new file mode 100644 index 000000000..062f64c29 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/tests/_config_support.py @@ -0,0 +1,304 @@ +# Copyright (c) 2026 Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: MIT +"""Shared, non-collected support for the configuration and carrier test owners. + +This module holds the production-shaped helpers that more than one owner reads: +document parsing, the redaction policy accessors, the fail-closed redaction +simulator, the shipped-consumer derivation, and the OTTL scrub inspectors. It +declares no tests so that pytest never collects it and the fuzz harness can +import it without crossing a collected-test boundary. +""" + +from __future__ import annotations + +import json +import pathlib +import re +from dataclasses import dataclass +from typing import Any + +import yaml + +SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1] +EXAMPLES_DIR = SKILL_ROOT / "examples" +AZURE_DIR = EXAMPLES_DIR / "azure" +COMPOSE_PATH = EXAMPLES_DIR / "compose.yaml" +COLLECTOR_PATH = EXAMPLES_DIR / "otel-collector-local.yaml" +DASHBOARD_PATH = EXAMPLES_DIR / "dashboards" / "copilot-otel.json" +BASELINE_PATH = EXAMPLES_DIR / "baseline.py" +FLEET_COLLECTOR_PATH = AZURE_DIR / "otel-collector-config.yaml" +RELAY_DIR = AZURE_DIR / "agent-host-relay" +RELAY_COLLECTOR_PATH = RELAY_DIR / "otel-collector-config.yaml" +RELAY_COMPOSE_PATH = RELAY_DIR / "compose.yaml" + +MASK = "****" +SCRUB = "[redacted]" + +DIGEST_REFERENCE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$") + +# --- Shipped-consumer derivation ------------------------------------------- +# +# The default disposition for a carrier the Collector does not govern is to +# scrub it. The only thing that earns an exception is a shipped consumer that +# reads it, so the consumer set has to be derived from the shipped artifacts +# rather than curated by hand. A hand-curated set cannot notice a newly +# required attribute, and a scrub decision made against a stale set breaks a +# panel silently. + +# Label names Prometheus reserves or Grafana generates. Excluded by rule, not +# by listing the ones this dashboard happens to use today. +PROMETHEUS_RESERVED_LABELS = frozenset({"le", "quantile", "__name__", "job", "instance"}) + +# A metric reference is an identifier immediately followed by a label matcher +# or a range selector. A function is followed by "(", so this separates the two +# without enumerating PromQL's function set. +PROMQL_METRIC = re.compile(r"\b([a-z_][a-z0-9_]*)\s*[{\[]") +# Grafana resolves label_values(metric, label): metric first, label second. +PROMQL_LABEL_VALUES = re.compile( + r"label_values\(\s*([a-z_][a-z0-9_]*)\s*,\s*([a-z_][a-z0-9_]*)\s*\)" +) +PROMQL_GROUPING = re.compile(r"\b(?:by|without)\s*\(([^)]*)\)") +PROMQL_MATCHER = re.compile(r"([a-zA-Z_][a-zA-Z0-9_]*)\s*(?:=~|!~|=|!=)\s*\"") +# baseline.py reads label values directly rather than through a panel. Its +# reads are consumers too; leaving them out of the inventory is how a detection +# control gets scrubbed without anything noticing. +BASELINE_LABEL_READ = re.compile(r"label_values\(\s*\"([a-z_][a-z0-9_]*)\"\s*\)") + +# TraceQL. The lookbehind keeps `service.name=` and `mode_name !=` from +# reading as span-name matchers; only a bare `name` is the intrinsic. +TRACEQL_SPAN_NAME = re.compile(r"(? frozenset[str]: + """OTLP attribute keys a shipped consumer reads by their real name.""" + return self.span_attributes | self.resource_attributes + + +def _promql_labels(expr: str) -> set[str]: + labels: set[str] = set() + for clause in PROMQL_GROUPING.findall(expr): + labels.update(part.strip() for part in clause.split(",") if part.strip()) + labels.update(PROMQL_MATCHER.findall(expr)) + return labels + + +def shipped_consumers() -> ShippedConsumers: + """Derive the consumer inventory from the dashboard and the baseline helper.""" + dashboard = json.loads(DASHBOARD_PATH.read_text(encoding="utf-8")) + + metric_names: set[str] = set() + labels: set[str] = set() + span_names: list[str] = [] + span_attributes: set[str] = set() + resource_attributes: set[str] = set() + + for variable in dashboard.get("templating", {}).get("list", []): + query = variable.get("query") + if isinstance(query, str): + for metric, label in PROMQL_LABEL_VALUES.findall(query): + metric_names.add(metric) + labels.add(label) + + for panel in dashboard.get("panels", []): + if panel.get("type") == "row": + continue + source = panel.get("datasource", {}).get("type") + for target in panel.get("targets", []): + expr = target.get("expr") or target.get("query") or "" + if source == "prometheus": + metric_names.update(PROMQL_METRIC.findall(expr)) + labels.update(_promql_labels(expr)) + elif source == "tempo": + span_names.extend(TRACEQL_SPAN_NAME.findall(expr)) + for scope, key in TRACEQL_ATTRIBUTE.findall(expr): + target_set = span_attributes if scope == "span" else resource_attributes + target_set.add(key.rstrip(".")) + + # A `$name` token is a Grafana template variable, not a stored label. + labels = {label for label in labels if not label.startswith("$")} + + baseline_source = BASELINE_PATH.read_text(encoding="utf-8") + labels.update(BASELINE_LABEL_READ.findall(baseline_source)) + labels = { + label + for label in labels + if not label.startswith("$") and label not in PROMETHEUS_RESERVED_LABELS + } + trace_fields = {field for field in ("rootTraceName", "durationMs") if field in baseline_source} + + return ShippedConsumers( + metric_names=frozenset(metric_names), + prometheus_labels=frozenset(labels), + span_name_matchers=tuple(span_names), + span_attributes=frozenset(span_attributes), + resource_attributes=frozenset(resource_attributes), + trace_fields=frozenset(trace_fields), + ) + + +def prometheus_label_for(attribute_key: str) -> str: + """Render an OTLP attribute key the way Prometheus stores it as a label. + + Comparing in this direction avoids inventing an OTLP spelling from a label. + `copilot_chat_edit_source` has two plausible sources, and reconstructing one + of them would be the guess this whole change exists to stop making. + """ + return re.sub(r"[^a-zA-Z0-9_]", "_", attribute_key) + + +def load_yaml_text(text: str) -> dict[str, Any]: + """Parse YAML text into a mapping, raising ConfigError on anything else.""" + try: + document = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise ConfigError(f"document is not valid YAML: {exc}") from exc + if not isinstance(document, dict): + raise ConfigError("document root is not a mapping") + return document + + +def load_yaml_file(path: pathlib.Path) -> dict[str, Any]: + """Parse a YAML file into a mapping.""" + if not path.is_file(): + raise ConfigError(f"missing configuration file: {path.name}") + return load_yaml_text(path.read_text(encoding="utf-8")) + + +def published_ports(service: dict[str, Any]) -> list[str]: + """Return the declared host port mappings for a Compose service.""" + return [str(entry) for entry in service.get("ports", []) or []] + + +def redaction_policy(collector: dict[str, Any]) -> dict[str, Any]: + """Return the redaction processor configuration from a Collector document.""" + processors = collector.get("processors") or {} + if not isinstance(processors, dict): + raise ConfigError("processors is not a mapping") + for name, config in processors.items(): + if name == "redaction" or str(name).startswith("redaction/"): + return config or {} + raise ConfigError("no redaction processor is declared") + + +def allowed_keys(collector: dict[str, Any]) -> frozenset[str]: + """Return the allow-listed attribute keys declared by the redaction processor.""" + policy = redaction_policy(collector) + return frozenset(str(key) for key in policy.get("allowed_keys", []) or []) + + +def blocked_values(collector: dict[str, Any]) -> tuple[re.Pattern[str], ...]: + """Return the compiled value patterns the redaction processor masks.""" + policy = redaction_policy(collector) + return tuple(re.compile(str(pattern)) for pattern in policy.get("blocked_values", []) or []) + + +def simulate_redaction( + attributes: dict[str, str], + allow: frozenset[str], + blocked: tuple[re.Pattern[str], ...] = (), + allow_all_keys: bool = False, +) -> dict[str, str]: + """Model the Collector redaction processor's fail-closed key and value handling. + + Any key outside the allow-list is dropped rather than kept, so an attribute + this skill has never seen cannot reach storage. Allowed values that still + match a blocked pattern are masked rather than dropped. + """ + result: dict[str, str] = {} + for key, value in attributes.items(): + if not allow_all_keys and key not in allow: + continue + text = str(value) + result[key] = MASK if any(pattern.search(text) for pattern in blocked) else text + return result + + +def scrub_statements(collector: dict[str, Any]) -> list[str]: + """Every OTTL statement the content-scrub processor declares.""" + processors = collector.get("processors") or {} + processor = processors.get("transform/scrub", {}) or {} + statements: list[str] = [] + for key in ("trace_statements", "log_statements", "metric_statements"): + for entry in processor.get(key) or []: + if isinstance(entry, str): + statements.append(entry) + else: + statements.extend(entry.get("statements", [])) + return statements + + +def statement_target(statement: str) -> str: + """The path an OTTL statement writes to. + + Only the write target matters. A `where` clause may read `span.name` + without endangering it, and treating a read as an overreach would push the + scrub rules into contortions to avoid mentioning what they must not touch. + """ + if "(" not in statement: + return "" + inner = statement.split("(", 1)[1] + depth = 0 + for index, character in enumerate(inner): + if character in "([": + depth += 1 + elif character in ")]": + if depth == 0: + return inner[:index].strip() + depth -= 1 + elif character == "," and depth == 0: + return inner[:index].strip() + return inner.strip() + + +def overreaching_statements( + statements: list[str], consumers: ShippedConsumers +) -> list[tuple[str, str]]: + """Statements whose write target is something a shipped consumer reads.""" + protected_keys = consumers.protected_attribute_keys() + findings: list[tuple[str, str]] = [] + for statement in statements: + target = statement_target(statement) + if target in PROTECTED_INTRINSICS: + findings.append((statement, target)) + continue + for key in ATTRIBUTE_TARGET.findall(target): + if key in protected_keys: + findings.append((statement, key)) + return findings diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py b/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py index 9e627736e..022dbc16b 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py @@ -14,14 +14,14 @@ sys.path.insert(0, str(_TESTS_DIR)) sys.path.insert(0, str(_TESTS_DIR.parent / "examples")) -from _input_policy import PolicyError, check_url # noqa: E402 -from settings_upsert import SettingsError, parse_assignment, strip_jsonc # noqa: E402 -from test_local_config import ( # noqa: E402 +from _config_support import ( # noqa: E402 MASK, ConfigError, load_yaml_text, simulate_redaction, ) +from _input_policy import PolicyError, check_url # noqa: E402 +from settings_upsert import SettingsError, parse_assignment, strip_jsonc # noqa: E402 try: import atheris From cd020496bb9daae68e868193e25a82d474a48c0f Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Tue, 18 Aug 2026 20:52:46 -0700 Subject: [PATCH 10/28] feat(skills): give the fleet collector filtering parity and ship the agent-host relay Replace the fleet collector's narrow attribute delete-list with the same redaction allow-list, blocked-value patterns, and OTTL scrub statements the local collector uses, so a workstation cannot ship content the local pipeline would have removed. Add a two-file Compose-owned relay that terminates loopback OTLP/HTTP on the agent host, applies the same policy a second time, and forwards to the fleet endpoint over TLS with a bearer token and a pinned CA bundle. Runtime tests cover the trusted path plus wrong-bearer, untrusted-CA, and hostname-mismatch rejection. --- .../examples/azure/README.md | 118 +++- .../azure/agent-host-relay/compose.yaml | 67 +++ .../otel-collector-config.yaml | 166 ++++++ .../examples/azure/otel-collector-config.yaml | 119 +++- .../references/org-distribution.md | 20 +- .../tests/test_azure_templates.py | 311 ++++++++++- .../tests/test_collector_carriers.py | 519 +++++++++++++++++- 7 files changed, 1247 insertions(+), 73 deletions(-) create mode 100644 .github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml create mode 100644 .github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md b/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md index 83953e5c0..f8504b9cb 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/README.md @@ -2,7 +2,7 @@ title: Azure Capture Templates description: Collector configuration, Bicep, Terraform, and Azure CLI templates for collecting GitHub Copilot fleet telemetry into Azure author: Microsoft -ms.date: 2026-07-27 +ms.date: 2026-08-18 ms.topic: reference keywords: - opentelemetry @@ -17,14 +17,16 @@ estimated_reading_time: 4 Templates to copy, adapt, and deploy yourself. Nothing here runs automatically, and the skill will not run any of it for you. Each one creates billable Azure resources. -| File | Deploys | -|------------------------------|-------------------------------------------------------------------------| -| `otel-collector-config.yaml` | Collector pipeline receiving OTLP and exporting to Application Insights | -| `main.bicep` | Log Analytics workspace, Application Insights, Azure Monitor dashboard | -| `main.tf` and friends | The same resources through Terraform | -| `deploy.sh` | The same resources through the Azure CLI | +| File | Deploys | +|-----------------------------------------------|-------------------------------------------------------------------------| +| `otel-collector-config.yaml` | Collector pipeline receiving OTLP and exporting to Application Insights | +| `agent-host-relay/otel-collector-config.yaml` | The per-workstation relay's Collector pipeline | +| `agent-host-relay/compose.yaml` | The per-workstation relay as a restart-managed container | +| `main.bicep` | Log Analytics workspace, Application Insights, Azure Monitor dashboard | +| `main.tf` and friends | The same resources through Terraform | +| `deploy.sh` | The same resources through the Azure CLI | -Pick one deployment path. They are three routes to the same result, not three stages. +Pick one deployment path for the Azure resources. They are three routes to the same result, not three stages. The relay is not one of those routes: it is deployed to every workstation regardless of which one you pick. ## Before you deploy @@ -101,7 +103,7 @@ If TLS is terminated by an ingress controller or load balancer in front of the c What the collector configuration cannot control is the sender. Copilot's exporter decides how it validates the server certificate, and no setting here changes that. -## Agent-host telemetry needs a relay +## Every workstation runs the relay Requiring authentication on the receiver has a consequence that is easy to miss because its symptom is an absence. @@ -109,18 +111,98 @@ Managed `telemetry.headers` are applied to the extension exporter directly and a Do not fix this by exporting the token into the environment, and do not fix it by removing authentication from the receiver. The first defeats the separation that keeps the credential away from agent-spawned subprocesses; the second reopens ingestion to anything that can route to this endpoint, on a billed backend. -Run a per-workstation relay instead: a Collector bound to `127.0.0.1:4318` that accepts the agent host's unauthenticated export and adds the fleet credential on its own upstream hop. `../otel-collector-local.yaml` is the same Collector configuration, so its fail-closed attribute allow-list applies to agent-host telemetry too. +The shipped fix is `agent-host-relay/`: a Collector on each workstation that listens on loopback, applies the same fail-closed allow-list and content scrub this fleet Collector applies, and adds the fleet credential on its own upstream hop. -Two conditions decide whether this actually keeps the credential out of the editor's reach: +```text +extension ─┐ + β”œβ”€β–Ά 127.0.0.1:4318 relay ──bearer, verified TLS──▢ fleet receiver +agent host β”€β”˜ (filters both) +``` -* Launch the relay **independently** of VS Code, as a service or user daemon. A relay started from the same shell as VS Code, or as its child, can share the environment this arrangement exists to keep clean. -* Store its credential where the relay reads it and the editor does not: its configuration file or a secret store, not an exported variable. +Managed settings apply one endpoint to both emitters, so both go to the relay. Neither carries the fleet credential, and managed settings no longer set `telemetry.headers` at all. -**What the relay does not do.** Its listener is unauthenticated, so any local process that can reach loopback can inject telemetry into this backend through it. Binding to `127.0.0.1` keeps the listener off the network; it says nothing about which local process connected. The relay authenticates the relay to this endpoint. It does not authenticate the workstation and it does not authenticate the developer. +**This makes the relay load-bearing for everything.** After the managed endpoint moves to loopback, a workstation whose relay is stopped, missing, or unhealthy sends no telemetry at all. Nothing queues it elsewhere. Deploy and verify relays first; change managed settings second. -Mutual TLS would be better, and this receiver already supports the server half: adding `client_ca_file` beside `cert_file` and `key_file` makes a valid client certificate mandatory on both protocols. Whether the VS Code agent host can present one is unknown, so it is not offered here as an available option. +### What the operator supplies + +The relay ships no credential and no certificate authority. Put three values in a file outside this repository, readable only by the account that runs the relay: + +```bash +# /etc/copilot-otel/relay.env, mode 600, owned by the relay account +COPILOT_OTEL_FLEET_ENDPOINT=https://otel.example.internal:4318 +COPILOT_OTEL_INGEST_TOKEN= +COPILOT_OTEL_FLEET_CA_BUNDLE=/etc/copilot-otel/fleet-ca.pem +``` -**Verify positively.** After a reload, query the backend for a span category only the agent host emits. If extension telemetry arrives and that category does not, the split is present. The symptom is a whole category of spans missing while everything else looks healthy, not an error in a log. +```bash +chmod 600 /etc/copilot-otel/relay.env +``` + +On Windows, restrict it to the same single account: + +```powershell +icacls C:\ProgramData\copilot-otel\relay.env /inheritance:r /grant:r "$env:USERNAME:(R)" +``` + +`COPILOT_OTEL_FLEET_CA_BUNDLE` must be an absolute path to the CA bundle that signs the fleet receiver's certificate. The relay mounts it read-only and verifies against it: a receiver presenting an untrusted certificate, or one whose subject alternative names do not include the endpoint hostname, is refused rather than trusted. There is no plaintext fallback and no verification opt-out. + +### Running it + +Start the relay independently of VS Code, so it neither inherits nor shares the editor's environment and `restart: unless-stopped` brings it back after a reboot: + +```bash +docker compose --env-file /etc/copilot-otel/relay.env \ + -f /opt/copilot-otel/agent-host-relay/compose.yaml up -d +``` + +Verify it before touching managed settings: + +```bash +curl -sf http://127.0.0.1:13133/ && echo relay healthy +``` + +Rotate the token by editing the env file and recreating the container: + +```bash +docker compose --env-file /etc/copilot-otel/relay.env \ + -f /opt/copilot-otel/agent-host-relay/compose.yaml up -d --force-recreate +``` + +When decommissioning a workstation, stop the relay and delete the env file. Removing the container does not remove it: + +```bash +docker compose --env-file /etc/copilot-otel/relay.env \ + -f /opt/copilot-otel/agent-host-relay/compose.yaml down +rm -f /etc/copilot-otel/relay.env +``` + +### Rollout and rollback + +1. Deploy the relay to every workstation and confirm each one reports healthy. +2. Change managed settings to `http://127.0.0.1:4318` with `otlp-http`, and remove `telemetry.headers`. +3. Query the backend for a span category only the agent host emits. Extension telemetry arriving does not establish that agent-host telemetry is. + +If telemetry stops fleet-wide or on one workstation, point managed settings back at the fleet endpoint and restore `telemetry.headers`. That returns extension telemetry immediately and returns the agent-host gap with it, which is the state that existed before the relay. + +The relay listens on OTLP/HTTP port 4318 only. gRPC is unsupported: managed settings must select `otlp-http`, and pointing an emitter at 4317 reaches nothing. + +### Detection + +| Symptom | Likely cause | +|-------------------------------------------------------|-------------------------------------------------------------| +| No telemetry at all from one workstation | Its relay is stopped or unhealthy | +| Agent-host span categories missing, extension present | Managed settings still point at the fleet endpoint directly | +| Relay healthy, nothing arrives upstream | Certificate, hostname, or bearer rejection in the relay log | + +The relay's own log names which of the three it hit. Check it before changing anything else. + +### What the relay does not do + +Its listener is unauthenticated, so any local process that can reach loopback can inject telemetry into this backend through it. Binding to `127.0.0.1` keeps the listener off the network; it says nothing about which local process connected. The relay authenticates the relay to this endpoint. It does not authenticate the workstation and it does not authenticate the developer. + +The env file is readable by anything running as the same operating-system account, and by anything that can inspect the container. That is a smaller blast radius than the environment of every agent-spawned subprocess, which is the point, but it is not isolation. + +Mutual TLS would be better, and this receiver already supports the server half: adding `client_ca_file` beside `cert_file` and `key_file` makes a valid client certificate mandatory on both protocols. Whether the VS Code agent host can present one is unknown, so it is not offered here as an available option. ## Retention and deletion @@ -132,8 +214,8 @@ The daily cap is the only spend guardrail. Setting it to `-1` removes it. 1. Retrieve the Application Insights connection string and put it in a secret store. Do not commit it. 2. Run the collector somewhere the fleet can reach, with `APPLICATIONINSIGHTS_CONNECTION_STRING`, `COPILOT_OTEL_INGEST_TOKEN`, `COPILOT_OTEL_ENVIRONMENT`, and the TLS paths supplied from that secret store. Where it runs is your platform decision; Container Apps, AKS, and a VM behind a load balancer all work. -3. Distribute the endpoint through Copilot managed settings. See `references/org-distribution.md`. -4. Deploy the per-workstation relay described above, or accept that agent-host telemetry is dropped fleet-wide. +3. Deploy `agent-host-relay/` to every workstation and confirm each one reports healthy. Do this **before** the next step: once managed settings point at loopback, a workstation without a working relay sends nothing. +4. Distribute the loopback endpoint through Copilot managed settings. See `references/org-distribution.md`. 5. Import `../dashboards/copilot-otel-azure.json` into Azure Monitor dashboards with Grafana and set the workspace variable to your Log Analytics resource ID. The templates provision the dashboard resource empty; this import is what fills it. 6. Confirm data landed by querying Log Analytics, not by checking that the collector reports success. Query for an agent-host span category specifically; extension telemetry arriving does not establish that agent-host telemetry is. diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml new file mode 100644 index 000000000..b52bdb12a --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml @@ -0,0 +1,67 @@ +# Per-workstation agent-host relay for organization-managed Copilot telemetry. +# +# Deploy this on every workstation that reports to the fleet endpoint, and +# verify it is healthy, before switching managed settings to the loopback +# endpoint. After that switch this relay carries all of that workstation's +# Copilot telemetry: if it is stopped or missing, telemetry is not queued +# elsewhere, it is lost. +# +# Start it independently of VS Code so it does not inherit or share the +# editor's environment, and so `restart: unless-stopped` returns it after a +# reboot: +# docker compose --env-file /absolute/path/copilot-otel-relay.env \ +# -f /absolute/path/agent-host-relay/compose.yaml up -d +# +# The env file lives outside this repository, is readable only by the operator +# account, and supplies: +# COPILOT_OTEL_FLEET_ENDPOINT=https://otel.example.internal +# COPILOT_OTEL_INGEST_TOKEN= +# COPILOT_OTEL_FLEET_CA_BUNDLE=/absolute/path/fleet-ca.pem +# +# Rotate the token by editing that file and running the same command with +# --force-recreate. Delete the file when decommissioning the relay; removing +# the container does not remove it. +# +# The image is pinned by manifest digest rather than tag, matching the local +# stack, so a mutable tag cannot change this relay underneath a review. + +services: + otel-relay: + # otel/opentelemetry-collector-contrib:0.158.0 + image: otel/opentelemetry-collector-contrib@sha256:c5918f78992ee73b0d6f0e599423ac5ec52dd5d9726733114d6eca53d5a32ed5 + container_name: copilot-otel-relay + restart: unless-stopped + command: ["--config=/etc/otelcol-contrib/config.yaml"] + ports: + # Loopback only. The relay is reachable from this workstation and from + # nothing on the network; the receiver inside the container binds the + # container interface, and this mapping is what decides host exposure. + - "127.0.0.1:4318:4318" # OTLP HTTP <- both Copilot emitters export here + - "127.0.0.1:13133:13133" # health, for the post-deployment check + environment: + # Required, with no default. The :? form fails the up with this message + # rather than starting a relay that would drop everything it receives. + COPILOT_OTEL_FLEET_ENDPOINT: ${COPILOT_OTEL_FLEET_ENDPOINT:?set COPILOT_OTEL_FLEET_ENDPOINT in the relay env file} + COPILOT_OTEL_INGEST_TOKEN: ${COPILOT_OTEL_INGEST_TOKEN:?set COPILOT_OTEL_INGEST_TOKEN in the relay env file} + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro + # The operator's CA bundle, read-only. An absolute host path is required + # because a relative one would resolve against whichever directory the + # up was run from. + - ${COPILOT_OTEL_FLEET_CA_BUNDLE:?set COPILOT_OTEL_FLEET_CA_BUNDLE to the absolute path of the fleet CA bundle}:/run/copilot-otel/fleet-ca.pem:ro + networks: + - copilot-otel-relay + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + # Bounds the container so the cgroup limit and the Collector's own + # memory_limiter agree rather than contradicting each other. + mem_limit: 512m + +networks: + # Explicit and dedicated, so this relay shares no network with the local + # development stack or anything else running on the workstation. + copilot-otel-relay: + driver: bridge diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml new file mode 100644 index 000000000..8fb27fe67 --- /dev/null +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml @@ -0,0 +1,166 @@ +# OpenTelemetry Collector configuration for the per-workstation agent-host relay. +# +# Managed VS Code settings apply one telemetry endpoint to both the extension +# and the agent host, and managed `telemetry.headers` reach only the extension. +# Sending both to this relay removes that split: neither emitter carries the +# fleet credential, this relay adds it on the upstream hop, and the credential +# never enters the environment VS Code hands to agent-spawned subprocesses. +# +# Ingress is OTLP/HTTP only. Host exposure is decided by the loopback port +# mapping in compose.yaml, not here, so the receiver binds the container +# interface exactly as the local stack's Collector does. There is deliberately +# no gRPC listener: managed settings pin `otlp-http`, and a second protocol +# would be an unexercised ingress. +# +# Every runtime input arrives through the environment or a read-only mount. +# Nothing in this file is a credential, and nothing in this file is a path +# inside the repository: +# COPILOT_OTEL_FLEET_ENDPOINT https:// base URL of the fleet receiver +# COPILOT_OTEL_INGEST_TOKEN the fleet ingest token +# COPILOT_OTEL_FLEET_CA_BUNDLE absolute host path to the CA bundle that +# signs the fleet receiver's certificate, +# mounted read-only at the ca_file path below +# +# Requires the OpenTelemetry Collector Contrib distribution; the redaction +# processor is not in the core distribution. + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + # Bounds Collector memory so a traffic spike degrades rather than crashes it. + memory_limiter: + check_interval: 1s + limit_percentage: 80 + spike_limit_percentage: 20 + + # The same fail-closed policy the local stack and the fleet Collector apply. + # Filtering here as well as upstream is deliberate: content is removed on the + # workstation that produced it, before it crosses the network, so a fleet + # receiver misconfiguration cannot turn this hop into the leak. + redaction: + allow_all_keys: false + allowed_keys: + # Resource identity. Dropping these would orphan every span. + - service.name + - service.namespace + - service.version + - telemetry.sdk.language + - telemetry.sdk.name + - telemetry.sdk.version + # Model and token accounting. The token panels read these directly. + - gen_ai.provider.name + - gen_ai.operation.name + - gen_ai.request.model + - gen_ai.response.model + - gen_ai.response.finish_reasons + - gen_ai.usage.input_tokens + - gen_ai.usage.output_tokens + - gen_ai.usage.cache_read.input_tokens + # Prometheus stores this as the gen_ai_token_type label; without it, + # input and output tokens collapse into one indistinguishable series. + - gen_ai.token.type + # The discriminator baseline.py reads through the session_id label. It + # is a detection control, not a convenience. + - session.id + # Agent and tool identity. Names only; arguments and results are not + # allowed and therefore cannot reach storage. + - gen_ai.agent.name + - gen_ai.tool.name + - gen_ai.tool.type + - copilot_chat.mode_name + - copilot_chat.turn_count + - copilot_chat.copilot_usage_nano_aiu + - github.copilot.agent.type + # Both spellings collapse to the copilot_chat_edit_source label and + # normalization is not reversible, so allow-listing both keeps a wrong + # single guess from silently emptying the edit-acceptance panel. + - copilot_chat.edit_source + - copilot_chat.edit.source + # Failure triage without failure content. + - error.type + # Masked, not dropped, so a panel keeps its shape while the value is gone. + blocked_values: + - "gh[pousr]_[0-9A-Za-z]{16,}" + - "github_pat_[0-9A-Za-z_]{20,}" + - "xox[baprs]-[0-9A-Za-z-]{10,}" + - "A(?:KIA|SIA)[0-9A-Z]{16}" + - "eyJ[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]*" + - "(?i)(?:AccountKey|SharedAccessKey|SharedAccessSignature)=[0-9A-Za-z+/=%&-]{16,}" + - "sk-(?:ant-)?[0-9A-Za-z_-]{20,}" + - "AIza[0-9A-Za-z_-]{35}" + - "(?i)-----BEGIN [A-Z ]*PRIVATE KEY-----" + - "(?i)\\b(?:bearer|authorization|api[_-]?key|secret|password)\\b\\s*[:=]\\s*\\S+" + # Silent so the processor does not annotate telemetry with the names of the + # keys it removed, which would reintroduce the drift it exists to stop. + summary: silent + + # Everything the redaction processor cannot reach. redaction traverses + # attribute maps only, so span status messages, span event names, trace + # state, and every non-map log body shape would otherwise leave the + # workstation verbatim. + # + # Span names, metric names, descriptions, and units are absent because + # shipped dashboard queries and baseline.py read them; span link attributes, + # span link trace state, and metric exemplar filtered attributes are absent + # because this distribution provides no way to reach them. Both kinds of + # residual are recorded in SECURITY.md. + transform/scrub: + error_mode: ignore + trace_statements: + - set(span.status.message, "[redacted]") where span.status.message != "" + - set(span.trace_state, "") where span.trace_state != "" + - set(spanevent.name, "[redacted]") + log_statements: + - set(log.body, "[redacted]") where log.body != nil + - set(log.severity_text, "[redacted]") where log.severity_text != "" + - set(log.event_name, "[redacted]") where log.event_name != "" + + batch: + timeout: 10s + send_batch_size: 1024 + +exporters: + # The upstream hop is the only place the fleet credential exists on this + # workstation. TLS verification is explicit: `ca_file` names the operator's + # bundle and there is no `insecure` flag, so a receiver presenting an + # untrusted certificate or a mismatched hostname is refused rather than + # trusted. The bundle is mounted from a host path the operator chooses; this + # skill ships no certificate authority. + otlp_http/fleet: + endpoint: ${env:COPILOT_OTEL_FLEET_ENDPOINT} + auth: + authenticator: bearertokenauth + tls: + ca_file: /run/copilot-otel/fleet-ca.pem + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + + # The fleet ingest token. It authenticates the fleet rather than a person and + # is identical on every workstation, so holding it here narrows where it is + # readable without binding it to anyone. Any process running as the same + # operating-system user can still read the environment file that supplies it. + bearertokenauth: + scheme: Bearer + token: ${env:COPILOT_OTEL_INGEST_TOKEN} + +service: + extensions: [health_check, bearertokenauth] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, redaction, transform/scrub, batch] + exporters: [otlp_http/fleet] + metrics: + receivers: [otlp] + processors: [memory_limiter, redaction, transform/scrub, batch] + exporters: [otlp_http/fleet] + logs: + receivers: [otlp] + processors: [memory_limiter, redaction, transform/scrub, batch] + exporters: [otlp_http/fleet] diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml index 0537aed3e..13b219924 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml @@ -12,7 +12,7 @@ # export COPILOT_OTEL_TLS_KEY_FILE="/etc/otel/tls/tls.key" # # Requires the OpenTelemetry Collector Contrib distribution; the azuremonitor -# exporter is not in the core distribution. +# exporter and the redaction processor are not in the core distribution. receivers: otlp: @@ -47,28 +47,92 @@ processors: limit_percentage: 80 spike_limit_percentage: 20 - # Six of these attributes were observed carrying plaintext content on spans - # with github.copilot.chat.otel.captureContent set to false. The seventh, - # copilot_chat.reasoning_content, was observed marked [encrypted] and is - # stripped defensively. This is the only place to remove any of it before it - # reaches billable, queryable storage. Remove an action here only as a - # deliberate decision. - attributes/strip-content: - actions: - - key: copilot_chat.user_request - action: delete - - key: copilot_chat.reasoning_content - action: delete - - key: gen_ai.input.messages - action: delete - - key: gen_ai.output.messages - action: delete - - key: gen_ai.system_instructions - action: delete - - key: gen_ai.tool.call.arguments - action: delete - - key: gen_ai.tool.call.result - action: delete + # The fleet minimization control. It is the local stack's policy verbatim, + # and tests/test_azure_templates.py asserts that parity: a shared, retained + # workspace is a stricter destination than one workstation, so this + # configuration may not filter less than otel-collector-local.yaml. + # + # allow_all_keys: false makes allowed_keys authoritative. An attribute a + # future emitter release introduces is dropped rather than stored, which a + # delete-list cannot do. Adding a key here stores that attribute fleet-wide. + redaction: + allow_all_keys: false + allowed_keys: + # Resource identity. Dropping these would orphan every span. + - service.name + - service.namespace + - service.version + - telemetry.sdk.language + - telemetry.sdk.name + - telemetry.sdk.version + # Model and token accounting. The token panels read these directly. + - gen_ai.provider.name + - gen_ai.operation.name + - gen_ai.request.model + - gen_ai.response.model + - gen_ai.response.finish_reasons + - gen_ai.usage.input_tokens + - gen_ai.usage.output_tokens + - gen_ai.usage.cache_read.input_tokens + # Prometheus stores this as the gen_ai_token_type label; without it, + # input and output tokens collapse into one indistinguishable series. + - gen_ai.token.type + # The discriminator baseline.py reads through the session_id label. It + # is a detection control, not a convenience. + - session.id + # Agent and tool identity. Names only; arguments and results are not + # allowed and therefore cannot reach storage. + - gen_ai.agent.name + - gen_ai.tool.name + - gen_ai.tool.type + - copilot_chat.mode_name + - copilot_chat.turn_count + - copilot_chat.copilot_usage_nano_aiu + - github.copilot.agent.type + # Both spellings collapse to the copilot_chat_edit_source label and + # normalization is not reversible, so allow-listing both keeps a wrong + # single guess from silently emptying the edit-acceptance panel. + - copilot_chat.edit_source + - copilot_chat.edit.source + # Failure triage without failure content. + - error.type + # Masked, not dropped, so a panel keeps its shape while the value is gone. + blocked_values: + - "gh[pousr]_[0-9A-Za-z]{16,}" + - "github_pat_[0-9A-Za-z_]{20,}" + - "xox[baprs]-[0-9A-Za-z-]{10,}" + - "A(?:KIA|SIA)[0-9A-Z]{16}" + - "eyJ[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]*" + - "(?i)(?:AccountKey|SharedAccessKey|SharedAccessSignature)=[0-9A-Za-z+/=%&-]{16,}" + - "sk-(?:ant-)?[0-9A-Za-z_-]{20,}" + - "AIza[0-9A-Za-z_-]{35}" + - "(?i)-----BEGIN [A-Z ]*PRIVATE KEY-----" + - "(?i)\\b(?:bearer|authorization|api[_-]?key|secret|password)\\b\\s*[:=]\\s*\\S+" + # Silent so the processor does not annotate telemetry with the names of the + # keys it removed, which would reintroduce the drift it exists to stop. + summary: silent + + # Everything the redaction processor cannot reach. redaction traverses + # attribute maps only, so span status messages, span event names, trace + # state, and every non-map log body shape would otherwise reach the exporter + # verbatim. + # + # A carrier the allow-list does not reach is scrubbed unless a shipped + # consumer reads it. Span names, metric names, descriptions, and units are + # absent because dashboard queries and baseline.py read them; span link + # attributes, span link trace state, and metric exemplar filtered attributes + # are absent because this distribution provides no way to reach them. Both + # kinds of residual are recorded in SECURITY.md. + transform/scrub: + error_mode: ignore + trace_statements: + - set(span.status.message, "[redacted]") where span.status.message != "" + - set(span.trace_state, "") where span.trace_state != "" + - set(spanevent.name, "[redacted]") + log_statements: + - set(log.body, "[redacted]") where log.body != nil + - set(log.severity_text, "[redacted]") where log.severity_text != "" + - set(log.event_name, "[redacted]") where log.event_name != "" # Fleet dimensions for grouping. Keep these to org structure; anything that # identifies an individual lands on every span they produce. @@ -134,15 +198,18 @@ extensions: service: extensions: [health_check, bearertokenauth] pipelines: + # resource/fleet runs after filtering so operator-owned environment + # identity survives the allow-list without the allow-list having to trust + # client-supplied dimensions. traces: receivers: [otlp] - processors: [memory_limiter, attributes/strip-content, resource/fleet, batch] + processors: [memory_limiter, redaction, transform/scrub, resource/fleet, batch] exporters: [azuremonitor] metrics: receivers: [otlp] - processors: [memory_limiter, resource/fleet, batch] + processors: [memory_limiter, redaction, transform/scrub, resource/fleet, batch] exporters: [azuremonitor] logs: receivers: [otlp] - processors: [memory_limiter, attributes/strip-content, resource/fleet, batch] + processors: [memory_limiter, redaction, transform/scrub, resource/fleet, batch] exporters: [azuremonitor] diff --git a/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md b/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md index e9a196c11..8f394a83e 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/org-distribution.md @@ -20,9 +20,8 @@ Administrators mandate OTel export through the `telemetry` block in Copilot mana { "telemetry": { "enabled": true, - "endpoint": "https://collector.example.internal:4318", + "endpoint": "http://127.0.0.1:4318", "protocol": "otlp-http", - "headers": { "Authorization": "Bearer " }, "captureContent": false, "lockCaptureContent": true, "serviceName": "copilot-chat", @@ -31,9 +30,15 @@ Administrators mandate OTel export through the `telemetry` block in Copilot mana } ``` +The endpoint is the per-workstation relay, not the fleet receiver, and there is no `headers` field. One endpoint applies to both the extension and the agent host, so pointing it at the relay is what lets both authenticate: the relay adds the fleet credential on its upstream hop, and no fleet credential is distributed to workstations through managed settings. `examples/azure/agent-host-relay/` ships that relay. + +Deploy and verify the relay on every workstation **before** distributing this block. Once it applies, a workstation without a running relay sends no telemetry at all rather than sending it directly, and nothing queues it. Rolling back means restoring the fleet endpoint and its `headers` block, which also restores the agent-host gap. + +`protocol` must be `otlp-http`. The relay exposes port 4318 only; there is no gRPC listener, so `otlp-grpc` reaches nothing. + `lockCaptureContent` is worth deciding deliberately rather than copying. It removes the developer's ability to turn content capture on, which is usually what an organization wants, and it is also the setting most likely to be missed when the block is adapted. -`headers` is a managed-settings field and has no user-settings counterpart in the builds this skill has verified. An operator who goes looking for it in the settings UI will not find it; that is the surface being different, not the field being missing. Its delivery is also what produces the agent-host split below. +`headers` is a managed-settings field and has no user-settings counterpart in the builds this skill has verified. An operator who goes looking for it in the settings UI will not find it; that is the surface being different, not the field being missing. Its delivery is also what produces the agent-host split below, and why the relay topology avoids using it. `resourceAttributes` is where fleet-wide dimensions belong. Anything put here lands on every span from every developer, so keep it to team and org structure. Do not put anything that identifies an individual in it. @@ -51,18 +56,19 @@ Channel precedence enforcement begins in VS Code 1.128. Below that version the b ## Four things that decide a rollout -* **A managed value beats environment variables and user settings.** Once set, developers cannot redirect telemetry to their own collector. That is usually the point, and it is also why a wrong endpoint in a managed block is a fleet-wide outage rather than one person's problem. -* **Managed `telemetry.headers` reach the extension exporter and not the agent host.** They are applied directly rather than through environment variables, which is deliberate: it stops an auth token leaking into spawned tool subprocesses. The consequence is a silent split, where extension telemetry authenticates and agent-host telemetry does not. Against a receiver that requires authentication this is data loss, not degraded data: the rejection is a non-retryable HTTP 401 or gRPC `UNAUTHENTICATED`, so the export is dropped rather than queued. The Azure reference covers the relay that closes it, and why the two obvious alternatives are both worse. +* **A managed value beats environment variables and user settings.** Once set, developers cannot redirect telemetry to their own collector. That is usually the point, and it is also why a wrong endpoint in a managed block is a fleet-wide outage rather than one person's problem. With the relay topology, an unhealthy relay is the same kind of outage, scoped to one workstation. +* **Managed `telemetry.headers` reach the extension exporter and not the agent host.** They are applied directly rather than through environment variables, which is deliberate: it stops an auth token leaking into spawned tool subprocesses. The consequence is a silent split, where extension telemetry authenticates and agent-host telemetry does not. Against a receiver that requires authentication this is data loss, not degraded data: the rejection is a non-retryable HTTP 401 or gRPC `UNAUTHENTICATED`, so the export is dropped rather than queued. Sending both emitters to the loopback relay is what closes it, which is why the block above sets no headers at all. * **The agent host computes its telemetry configuration at start.** Changing a managed value requires a VS Code reload on every affected machine, not just a policy refresh. * **A developer can find out what applies to them.** **Developer: Policy Diagnostics** from the command palette is the per-device discriminator. It answers "is my setting being overridden" without guessing. ## What to hand the administrator -1. The `telemetry` block, with the endpoint and any resource attributes filled in from what the user supplied. +1. The `telemetry` block, with any resource attributes filled in from what the user supplied. The endpoint is the loopback relay. 2. The chosen channel and its exact path or registry key for each platform in the fleet. 3. The precedence warning: this channel wins entirely, so everything the organization intends to manage must be in this one place. 4. The reload requirement. -5. The credential question, which the Azure reference covers: whatever authenticates the collector will exist on every workstation. +5. The relay prerequisite: every workstation needs a healthy relay before this block applies, and the rollback is the previous endpoint plus its `headers` block. +6. The credential question, which the Azure reference covers: whatever authenticates the collector will exist on every workstation, now inside each relay's environment file rather than in managed settings. ## Links diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py index 0738d9812..e74c891bb 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py @@ -13,6 +13,17 @@ import pytest import yaml +from _config_support import ( + COLLECTOR_PATH as LOCAL_COLLECTOR_PATH, +) +from _config_support import ( + DIGEST_REFERENCE, + RELAY_COLLECTOR_PATH, + RELAY_COMPOSE_PATH, + load_yaml_file, + redaction_policy, + scrub_statements, +) AZURE_DIR = pathlib.Path(__file__).resolve().parents[1] / "examples" / "azure" COLLECTOR_PATH = AZURE_DIR / "otel-collector-config.yaml" @@ -21,7 +32,34 @@ VARIABLES_TF_PATH = AZURE_DIR / "variables.tf" OUTPUTS_TF_PATH = AZURE_DIR / "outputs.tf" -TEMPLATE_PATHS = [BICEP_PATH, MAIN_TF_PATH, VARIABLES_TF_PATH, OUTPUTS_TF_PATH, COLLECTOR_PATH] +TEMPLATE_PATHS = [ + BICEP_PATH, + MAIN_TF_PATH, + VARIABLES_TF_PATH, + OUTPUTS_TF_PATH, + COLLECTOR_PATH, + RELAY_COLLECTOR_PATH, + RELAY_COMPOSE_PATH, +] + +# The relay's runtime inputs, and the only values a rendered Compose document +# may carry. They are obvious non-secrets so a rendering failure cannot be +# mistaken for a leaked credential. +RELAY_SENTINELS = { + "COPILOT_OTEL_FLEET_ENDPOINT": "https://sentinel-fleet.invalid", + "COPILOT_OTEL_INGEST_TOKEN": "sentinel-token-not-a-credential", + "COPILOT_OTEL_FLEET_CA_BUNDLE": "/sentinel/absolute/path/fleet-ca.pem", +} +RELAY_CA_MOUNT = "/run/copilot-otel/fleet-ca.pem" + +# The managed-settings contract these relay files have to agree with. +ORG_DISTRIBUTION_PATH = ( + pathlib.Path(__file__).resolve().parents[1] / "references" / "org-distribution.md" +) + +# Compose interpolation this skill uses: ${NAME:?message} for a required value +# and ${NAME} for a plain one. +COMPOSE_VARIABLE = re.compile(r"\$\{(?P[A-Z0-9_]+)(?::\?(?P[^}]*))?\}") # Shapes that indicate a real credential rather than a placeholder or a # reference to one. @@ -38,6 +76,31 @@ def collector() -> dict: return yaml.safe_load(COLLECTOR_PATH.read_text(encoding="utf-8")) +@pytest.fixture(scope="module") +def local_collector() -> dict: + """The local stack policy, which is this skill's minimization baseline.""" + return load_yaml_file(LOCAL_COLLECTOR_PATH) + + +@pytest.fixture(scope="module") +def relay() -> dict: + """The shipped workstation relay Collector configuration.""" + return load_yaml_file(RELAY_COLLECTOR_PATH) + + +@pytest.fixture(scope="module") +def relay_compose() -> dict: + """The relay Compose document with its required variables rendered. + + Rendering is what makes the mount and environment assertions mean + something: an unrendered `${VAR:?...}` string satisfies any substring check + while telling nothing about what the container would actually receive. + """ + text = RELAY_COMPOSE_PATH.read_text(encoding="utf-8") + rendered = COMPOSE_VARIABLE.sub(lambda match: RELAY_SENTINELS[match["name"]], text) + return yaml.safe_load(rendered) + + @pytest.fixture(scope="module") def bicep() -> str: return BICEP_PATH.read_text(encoding="utf-8") @@ -148,6 +211,252 @@ def test_retention_is_described_as_the_deletion_boundary( assert "deletion policy" in terraform.lower() +class TestFleetPolicyParity: + """The fleet filters at least as much as the local stack, on every signal. + + The destination here is a shared, retained workspace rather than one + workstation's disposable container, so a fleet pipeline that filtered less + than the local one would make the skill's pre-storage claims false in the + place they matter most. Parity is asserted against the local document + rather than restated, so drift in either file fails. + """ + + def test_the_fleet_allow_list_matches_the_local_allow_list( + self, collector: dict, local_collector: dict + ) -> None: + assert ( + redaction_policy(collector)["allowed_keys"] + == redaction_policy(local_collector)["allowed_keys"] + ) + + def test_the_fleet_blocked_values_match_the_local_blocked_values( + self, collector: dict, local_collector: dict + ) -> None: + assert ( + redaction_policy(collector)["blocked_values"] + == redaction_policy(local_collector)["blocked_values"] + ) + + def test_the_fleet_policy_is_fail_closed_and_silent(self, collector: dict) -> None: + policy = redaction_policy(collector) + assert policy["allow_all_keys"] is False + assert policy["summary"] == "silent" + + def test_the_fleet_scrub_statements_match_the_local_scrub_statements( + self, collector: dict, local_collector: dict + ) -> None: + assert scrub_statements(collector) == scrub_statements(local_collector) + + @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) + def test_every_signal_applies_both_privacy_processors( + self, collector: dict, signal: str + ) -> None: + processors = collector["service"]["pipelines"][signal]["processors"] + assert "redaction" in processors, f"{signal} reaches storage unfiltered" + assert "transform/scrub" in processors, f"{signal} keeps ungoverned carriers" + + @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) + def test_the_privacy_processors_keep_their_local_relative_order( + self, collector: dict, local_collector: dict, signal: str + ) -> None: + """The runtime carrier probe only substitutes definitions, not ordering.""" + privacy = ("redaction", "transform/scrub") + fleet_order = [ + step + for step in collector["service"]["pipelines"][signal]["processors"] + if step in privacy + ] + local_order = [ + step + for step in local_collector["service"]["pipelines"][signal]["processors"] + if step in privacy + ] + assert fleet_order == local_order + + @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) + def test_filtering_precedes_export_and_operator_identity( + self, collector: dict, signal: str + ) -> None: + processors = collector["service"]["pipelines"][signal]["processors"] + assert processors.index("memory_limiter") < processors.index("redaction") + assert processors.index("transform/scrub") < processors.index("resource/fleet") + assert processors.index("resource/fleet") < processors.index("batch") + + def test_the_superseded_delete_list_is_gone(self, collector: dict) -> None: + """A delete-list cannot fail closed on an attribute it has never seen.""" + assert "attributes/strip-content" not in collector["processors"] + for pipeline in collector["service"]["pipelines"].values(): + assert "attributes/strip-content" not in pipeline["processors"] + + +class TestRelayTopology: + """The relay is reachable from this workstation and from nothing else.""" + + def test_the_relay_declares_only_otlp_http(self, relay: dict) -> None: + protocols = relay["receivers"]["otlp"]["protocols"] + assert set(protocols) == {"http"}, "a second ingress protocol would be unexercised" + + def test_the_relay_receiver_takes_the_container_interface(self, relay: dict) -> None: + """Host exposure is the port mapping's decision, exactly as the local stack does it.""" + assert relay["receivers"]["otlp"]["protocols"]["http"]["endpoint"] == "0.0.0.0:4318" + + def test_the_relay_receiver_requires_no_credential(self, relay: dict) -> None: + """Both emitters export without one; requiring it here would drop everything.""" + assert "auth" not in relay["receivers"]["otlp"]["protocols"]["http"] + + def test_every_published_port_is_loopback(self, relay_compose: dict) -> None: + ports = [str(entry) for entry in relay_compose["services"]["otel-relay"]["ports"]] + assert ports, "the relay publishes nothing" + for entry in ports: + assert entry.startswith("127.0.0.1:"), f"{entry} is reachable off this workstation" + assert "::" not in entry + + def test_the_only_otlp_mapping_is_http_on_4318(self, relay_compose: dict) -> None: + ports = [str(entry) for entry in relay_compose["services"]["otel-relay"]["ports"]] + assert "127.0.0.1:4318:4318" in ports + assert not [entry for entry in ports if ":4317:" in entry], "gRPC is unsupported" + + +class TestRelayUpstream: + """The upstream hop is authenticated and its certificate is verified.""" + + def test_the_endpoint_is_an_environment_reference(self, relay: dict) -> None: + exporter = relay["exporters"]["otlp_http/fleet"] + assert exporter["endpoint"] == "${env:COPILOT_OTEL_FLEET_ENDPOINT}" + + def test_the_exporter_authenticates(self, relay: dict) -> None: + assert relay["exporters"]["otlp_http/fleet"]["auth"]["authenticator"] == "bearertokenauth" + assert relay["extensions"]["bearertokenauth"]["token"].startswith("${env:") + assert "bearertokenauth" in relay["service"]["extensions"] + + def test_the_exporter_verifies_against_the_mounted_bundle(self, relay: dict) -> None: + tls = relay["exporters"]["otlp_http/fleet"]["tls"] + assert tls["ca_file"] == RELAY_CA_MOUNT + assert "insecure" not in tls, "the upstream hop must not opt out of verification" + assert "insecure_skip_verify" not in tls + + def test_no_pipeline_exports_anywhere_else(self, relay: dict) -> None: + for name, pipeline in relay["service"]["pipelines"].items(): + assert pipeline["exporters"] == ["otlp_http/fleet"], f"{name} has a second destination" + + +class TestRelayPolicyParity: + """Telemetry is minimized on the workstation that produced it.""" + + def test_the_relay_allow_list_matches_the_local_allow_list( + self, relay: dict, local_collector: dict + ) -> None: + assert ( + redaction_policy(relay)["allowed_keys"] + == redaction_policy(local_collector)["allowed_keys"] + ) + + def test_the_relay_blocked_values_match_the_local_blocked_values( + self, relay: dict, local_collector: dict + ) -> None: + assert ( + redaction_policy(relay)["blocked_values"] + == redaction_policy(local_collector)["blocked_values"] + ) + + def test_the_relay_scrub_statements_match_the_local_scrub_statements( + self, relay: dict, local_collector: dict + ) -> None: + assert scrub_statements(relay) == scrub_statements(local_collector) + + @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) + def test_every_signal_is_filtered_before_it_leaves_the_workstation( + self, relay: dict, signal: str + ) -> None: + processors = relay["service"]["pipelines"][signal]["processors"] + assert processors.index("redaction") < processors.index("batch") + assert processors.index("transform/scrub") < processors.index("batch") + + +class TestRelayRuntime: + """The relay outlives VS Code, holds no committed credential, and is bounded.""" + + def test_the_image_is_digest_pinned(self, relay_compose: dict) -> None: + assert DIGEST_REFERENCE.match(relay_compose["services"]["otel-relay"]["image"]) + + def test_the_relay_restarts_independently_of_the_editor(self, relay_compose: dict) -> None: + assert relay_compose["services"]["otel-relay"]["restart"] == "unless-stopped" + + def test_the_container_is_hardened(self, relay_compose: dict) -> None: + service = relay_compose["services"]["otel-relay"] + assert service["read_only"] is True + assert service["cap_drop"] == ["ALL"] + assert "no-new-privileges:true" in service["security_opt"] + assert service["mem_limit"] == "512m" + + @pytest.mark.parametrize("variable", sorted(RELAY_SENTINELS)) + def test_every_runtime_input_is_required(self, variable: str) -> None: + """A defaulted input would start a relay that silently discards everything.""" + text = RELAY_COMPOSE_PATH.read_text(encoding="utf-8") + matches = [match for match in COMPOSE_VARIABLE.finditer(text) if match["name"] == variable] + assert matches, f"{variable} is not read by the relay Compose document" + for match in matches: + assert match["message"], f"{variable} has no failure message and may default to empty" + + def test_the_config_and_ca_bundle_are_mounted_read_only(self, relay_compose: dict) -> None: + volumes = [str(entry) for entry in relay_compose["services"]["otel-relay"]["volumes"]] + config_mount = next(entry for entry in volumes if entry.endswith("config.yaml:ro")) + assert config_mount.startswith("./otel-collector-config.yaml:") + ca_mount = next(entry for entry in volumes if RELAY_CA_MOUNT in entry) + assert ca_mount == f"{RELAY_SENTINELS['COPILOT_OTEL_FLEET_CA_BUNDLE']}:{RELAY_CA_MOUNT}:ro" + + def test_the_ca_bundle_comes_from_outside_this_repository(self, relay_compose: dict) -> None: + """The skill ships no certificate authority, and could not ship a useful one.""" + volumes = [str(entry) for entry in relay_compose["services"]["otel-relay"]["volumes"]] + ca_mount = next(entry for entry in volumes if RELAY_CA_MOUNT in entry) + assert ca_mount.startswith("/"), "the bundle path must be absolute" + assert not ca_mount.startswith("./") + + def test_only_the_relay_service_receives_the_upstream_values(self, relay_compose: dict) -> None: + rendered = yaml.safe_dump(relay_compose) + for service, definition in relay_compose["services"].items(): + environment = definition.get("environment") or {} + if service != "otel-relay": + assert RELAY_SENTINELS["COPILOT_OTEL_INGEST_TOKEN"] not in str(environment) + assert rendered.count(RELAY_SENTINELS["COPILOT_OTEL_INGEST_TOKEN"]) == 1 + + +class TestManagedSettingsMatchTheRelay: + """The distributed managed block has to describe the relay that was shipped. + + An endpoint or protocol that drifts from the relay's single listener is not + a documentation defect; it is a fleet that stops reporting after the + managed settings change. + """ + + def test_the_managed_endpoint_is_the_relay_listener(self, relay_compose: dict) -> None: + text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8") + assert '"endpoint": "http://127.0.0.1:4318"' in text + ports = [str(entry) for entry in relay_compose["services"]["otel-relay"]["ports"]] + assert "127.0.0.1:4318:4318" in ports + + def test_the_managed_protocol_is_the_one_the_relay_serves(self, relay: dict) -> None: + text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8") + assert '"protocol": "otlp-http"' in text + assert set(relay["receivers"]["otlp"]["protocols"]) == {"http"} + + def test_managed_settings_carry_no_fleet_credential(self) -> None: + text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8") + block = text.split("```json", 1)[1].split("```", 1)[0] + assert "headers" not in block, "the managed block must not distribute the fleet token" + assert "Bearer" not in block + + def test_grpc_is_named_unsupported_rather_than_untested(self) -> None: + text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8").lower() + assert "no grpc listener" in text or "there is no grpc listener" in text + + def test_the_relay_prerequisite_precedes_the_managed_change(self) -> None: + text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8").lower() + assert "distributing this block" in text + assert "sends no telemetry at all" in text + assert "rolling back" in text or "rollback" in text + + class TestStateAndSecrets: """No template carries a credential, and state sensitivity is stated.""" diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py index ad1ea23d0..e3be70cd0 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py @@ -45,6 +45,15 @@ SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1] COLLECTOR_PATH = SKILL_ROOT / "examples" / "otel-collector-local.yaml" COMPOSE_PATH = SKILL_ROOT / "examples" / "compose.yaml" +FLEET_COLLECTOR_PATH = SKILL_ROOT / "examples" / "azure" / "otel-collector-config.yaml" + +# Which shipped document supplies the content-minimization processors under +# test. The fleet document exports to a shared, retained workspace, so its +# policy needs the same carrier-level evidence as the local one rather than a +# static claim that the two files look alike. +LOCAL_POLICY = "local" +FLEET_POLICY = "fleet" +POLICY_SOURCES = (LOCAL_POLICY, FLEET_POLICY) # The probe runs the same image the stack runs. Read from compose.yaml rather # than restated here, so a pin bump cannot leave this harness testing an image @@ -61,11 +70,17 @@ # machine and not tolerable anywhere the run's result is cited: `SECURITY.md` # and the references attribute their minimization claims to this module, so a # lane that quietly stopped starting the Collector would keep reporting green -# while those claims lost their backing. Strictness is therefore on by default -# wherever `CI` is set, and this variable overrides it in both directions. +# while those claims lost their backing. The lane that owns this evidence says +# so explicitly through this variable; no ambient signal turns strictness on, +# because a generic runner that never selected these tests should not inherit +# their runtime prerequisites. STRICT_RUNTIME_ENV = "COPILOT_OTEL_STRICT_RUNTIME" _FALSE_VALUES = frozenset({"", "0", "false", "no", "off"}) +# This module starts containers and pulls a pinned image. Selecting it is a +# deliberate act by a lane that provisions those prerequisites. +pytestmark = pytest.mark.slow + MASK = "****" SCRUB = "[redacted]" @@ -485,7 +500,15 @@ def shipped_collector_config() -> dict: return yaml.safe_load(COLLECTOR_PATH.read_text(encoding="utf-8")) -def derive_probe_config(*, remove_content_processors: bool) -> dict: +def policy_document(policy_source: str) -> dict: + """The shipped document that owns the content-minimization processors.""" + path = COLLECTOR_PATH if policy_source == LOCAL_POLICY else FLEET_COLLECTOR_PATH + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def derive_probe_config( + *, remove_content_processors: bool, policy_source: str = LOCAL_POLICY +) -> dict: """Derive a probe configuration from the shipped file. The only permitted substitution for an ordinary run is exporter wiring, so @@ -493,9 +516,20 @@ def derive_probe_config(*, remove_content_processors: bool) -> dict: this harness does not start. Receivers, processors, and processor ordering are preserved. The negative control is additionally permitted to remove every content-minimization processor, and makes no other policy change. + + A fleet run substitutes the two content-minimization processor definitions + from the fleet document and nothing else. Transport stays local because the + fleet receiver requires TLS material and a bearer credential this harness + does not hold, and because the fleet exporter targets a store it does not + start. What is under test is the policy, and the policy is what is swapped. """ config = copy.deepcopy(shipped_collector_config()) + if policy_source != LOCAL_POLICY: + source = policy_document(policy_source) + for name in CONTENT_PROCESSORS: + config["processors"][name] = copy.deepcopy(source["processors"][name]) + config["exporters"] = {"debug": {"verbosity": "detailed"}} # A ten second batch window would dominate the probe's runtime without # changing what the policy does to a record. @@ -535,6 +569,22 @@ def mount_source(self, path: pathlib.Path) -> str: """Render a host path in the form this Docker CLI can bind-mount.""" return path.as_posix() + def openssl( + self, *args: str, cwd: pathlib.Path, timeout: float = 120.0 + ) -> subprocess.CompletedProcess[str]: + """Run OpenSSL where this runtime's files live, using relative names.""" + return subprocess.run( # noqa: S603 - fixed argv, no shell + ["openssl", *args], + cwd=str(cwd), + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + def has_openssl(self) -> bool: + return shutil.which("openssl") is not None + class _WslDockerRuntime(ContainerRuntime): """Windows hosts where Docker is reachable only inside WSL.""" @@ -551,6 +601,38 @@ def mount_source(self, path: pathlib.Path) -> str: raise RuntimeError(f"wslpath failed: {translated.stderr.strip()}") return translated.stdout.strip() + def openssl( + self, *args: str, cwd: pathlib.Path, timeout: float = 120.0 + ) -> subprocess.CompletedProcess[str]: + # The certificate has to be readable by the same daemon that mounts it, + # so it is produced by the OpenSSL that lives beside that daemon. + return subprocess.run( # noqa: S603 - fixed argv, no shell + [ + "wsl.exe", + "-e", + "sh", + "-c", + 'cd "$1" && shift && exec openssl "$@"', + "sh", + self.mount_source(cwd), + *args, + ], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + def has_openssl(self) -> bool: + probe = subprocess.run( # noqa: S603 - fixed argv, no shell + ["wsl.exe", "-e", "openssl", "version"], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + return probe.returncode == 0 + def detect_runtime() -> ContainerRuntime | None: """Find a reachable Docker daemon without assuming a platform. @@ -578,15 +660,14 @@ def detect_runtime() -> ContainerRuntime | None: def strict_runtime_required(env: Mapping[str, str] | None = None) -> bool: """Whether a missing container runtime must fail this module rather than skip it. - An explicit `STRICT_RUNTIME_ENV` value decides on its own, in either - direction, so a lane without a usable runtime can opt out deliberately and - a contributor can opt in. With no explicit value the standard `CI` variable - decides, because a run whose green result is read as evidence is exactly the - run that must not skip. + Only `STRICT_RUNTIME_ENV` decides. The lane that selects these tests also + provisions Docker and pre-pulls the pinned image, so it is the only caller + positioned to promise that a green result means the probes ran. Inferring + strictness from an ambient signal would impose that promise on runners that + never selected the module. """ source = os.environ if env is None else env - explicit = source.get(STRICT_RUNTIME_ENV) - chosen = explicit if explicit is not None else source.get("CI", "") + chosen = source.get(STRICT_RUNTIME_ENV, "") return chosen.strip().lower() not in _FALSE_VALUES @@ -654,12 +735,19 @@ def _post(port: int, signal: str, payload: dict) -> None: raise RuntimeError(f"{signal} export returned HTTP {response.status}") -def run_probe(runtime: ContainerRuntime, *, remove_content_processors: bool) -> ProbeResult: +def run_probe( + runtime: ContainerRuntime, + *, + remove_content_processors: bool, + policy_source: str = LOCAL_POLICY, +) -> ProbeResult: """Start a disposable Collector, export every marker, capture the output.""" - config = derive_probe_config(remove_content_processors=remove_content_processors) + config = derive_probe_config( + remove_content_processors=remove_content_processors, policy_source=policy_source + ) http_port = _free_port() health_port = _free_port() - variant = "control" if remove_content_processors else "policy" + variant = "control" if remove_content_processors else policy_source container = f"copilot-otel-carrier-probe-{variant}-{http_port}" with tempfile.TemporaryDirectory(prefix="copilot-otel-probe-") as workdir: @@ -745,9 +833,9 @@ def runtime() -> ContainerRuntime: return resolve_runtime() -@pytest.fixture(scope="module") -def policy_output(runtime: ContainerRuntime) -> str: - return run_probe(runtime, remove_content_processors=False).output +@pytest.fixture(scope="module", params=POLICY_SOURCES) +def policy_output(request: pytest.FixtureRequest, runtime: ContainerRuntime) -> str: + return run_probe(runtime, remove_content_processors=False, policy_source=request.param).output @pytest.fixture(scope="module") @@ -828,6 +916,26 @@ def test_the_control_removes_only_the_content_processors(self) -> None: def test_the_probe_runs_the_image_the_stack_pins(self) -> None: assert "@sha256:" in pinned_collector_image() + def test_a_fleet_run_uses_the_fleet_content_processors(self) -> None: + fleet = policy_document(FLEET_POLICY) + derived = derive_probe_config(remove_content_processors=False, policy_source=FLEET_POLICY) + for name in CONTENT_PROCESSORS: + assert derived["processors"][name] == fleet["processors"][name] + + def test_a_fleet_run_substitutes_nothing_but_the_content_processors(self) -> None: + """Swapping transport as well would test the fleet's network, not its policy.""" + local = derive_probe_config(remove_content_processors=False) + fleet = derive_probe_config(remove_content_processors=False, policy_source=FLEET_POLICY) + assert local["receivers"] == fleet["receivers"] + assert local["exporters"] == fleet["exporters"] + assert local["service"] == fleet["service"] + differing = { + name + for name in local["processors"] + if local["processors"][name] != fleet["processors"][name] + } + assert differing <= set(CONTENT_PROCESSORS) + class TestStrictRuntimeMode: """A run that produces no evidence must not be able to pass quietly. @@ -836,21 +944,21 @@ class TestStrictRuntimeMode: usable container runtime, a skip would keep the suite green while those claims cite a run that never happened, which is the defect this suite exists to prevent, one level up. The rule that decides between skipping and - failing is therefore asserted rather than trusted. + failing is therefore asserted rather than trusted, including its refusal to + infer strictness from an ambient signal. """ @pytest.mark.parametrize( ("env", "expected"), [ ({}, False), - ({"CI": "true"}, True), - ({"CI": "1"}, True), - ({"CI": "false"}, False), - ({"CI": ""}, False), + ({"CI": "true"}, False), + ({"CI": "1"}, False), ({STRICT_RUNTIME_ENV: "1"}, True), ({STRICT_RUNTIME_ENV: "true", "CI": "false"}, True), ({STRICT_RUNTIME_ENV: "0", "CI": "true"}, False), ({STRICT_RUNTIME_ENV: "off", "CI": "true"}, False), + ({STRICT_RUNTIME_ENV: ""}, False), ], ) def test_strictness_follows_the_environment(self, env: dict[str, str], expected: bool) -> None: @@ -994,3 +1102,372 @@ def test_the_shipped_configuration_logs_no_deprecation_warning( ) finally: runtime.docker("rm", "-f", container, timeout=120) + + +# --- Agent-host relay ------------------------------------------------------ +# +# The relay is the only carrier for organization-managed agent-host telemetry, +# and everything that makes it safe is transport: it must reach the fleet +# receiver over TLS it verified against the operator's bundle, carrying the +# fleet credential, and it must refuse to forward when any of that is wrong. +# None of that is observable in a configuration file, so the shipped relay is +# run against a disposable receiver that holds the other half of the handshake. + +RELAY_COLLECTOR_PATH = ( + SKILL_ROOT / "examples" / "azure" / "agent-host-relay" / ("otel-collector-config.yaml") +) +RELAY_SAN_HOST = "fleet-receiver" +RELAY_WRONG_HOST = "other-receiver-name" +RELAY_TOKEN = "relay-runtime-fleet-token" +RELAY_CA_MOUNT = "/run/copilot-otel/fleet-ca.pem" +RELAY_MARKER_ATTRIBUTE = "session.id" + +NO_OPENSSL_REASON = ( + "OpenSSL is unavailable beside the container runtime, so no ephemeral CA " + "and server certificate could be issued. The relay's TLS and credential " + "behaviour was not verified by this run." +) + + +@dataclass(frozen=True) +class RelayCase: + """One end-to-end relay attempt and whether forwarding may succeed.""" + + name: str + token: str + ca_file: str + endpoint_host: str + forwards: bool + + +RELAY_CASES = ( + RelayCase("trusted", RELAY_TOKEN, "ca.pem", RELAY_SAN_HOST, forwards=True), + RelayCase("wrong-bearer", "not-the-fleet-token", "ca.pem", RELAY_SAN_HOST, forwards=False), + RelayCase("untrusted-ca", RELAY_TOKEN, "other-ca.pem", RELAY_SAN_HOST, forwards=False), + RelayCase("hostname-mismatch", RELAY_TOKEN, "ca.pem", RELAY_WRONG_HOST, forwards=False), +) + + +def resolve_openssl(runtime: ContainerRuntime) -> None: + """Stop this module the way strictness requires when OpenSSL is missing.""" + if runtime.has_openssl(): + return + if strict_runtime_required(): + pytest.fail( + f"{NO_OPENSSL_REASON} Strict mode is active, so this is a failure and " + f"not a skip. Set {STRICT_RUNTIME_ENV}=0 to allow the skip.", + pytrace=False, + ) + pytest.skip(NO_OPENSSL_REASON) + + +def _issue_relay_material(runtime: ContainerRuntime, workdir: pathlib.Path) -> None: + """Issue a throwaway CA, a SAN-matched server certificate, and a rival CA. + + The rival CA is what makes the untrusted-certificate case a real rejection + rather than an absence: the receiver still presents a valid certificate, + signed by an authority the relay was not given. + """ + # Only RELAY_SAN_HOST is named, so the mismatched-hostname case rests on + # the certificate rather than on the receiver refusing the connection. + (workdir / "san.cnf").write_text(f"subjectAltName=DNS:{RELAY_SAN_HOST}\n", encoding="utf-8") + commands = [ + # The authority the relay is told to trust. + ( + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "ca.key", + "-out", + "ca.pem", + "-days", + "1", + "-subj", + "/CN=copilot-otel-relay-test-ca", + "-addext", + "basicConstraints=critical,CA:TRUE", + ), + # An authority the relay is never told about. + ( + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "other-ca.key", + "-out", + "other-ca.pem", + "-days", + "1", + "-subj", + "/CN=copilot-otel-rival-ca", + "-addext", + "basicConstraints=critical,CA:TRUE", + ), + ( + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "server.key", + "-out", + "server.csr", + "-subj", + f"/CN={RELAY_SAN_HOST}", + ), + ( + "x509", + "-req", + "-in", + "server.csr", + "-CA", + "ca.pem", + "-CAkey", + "ca.key", + "-CAcreateserial", + "-out", + "server.crt", + "-days", + "1", + "-extfile", + "san.cnf", + ), + ] + for command in commands: + result = runtime.openssl(*command, cwd=workdir) + if result.returncode != 0: + raise RuntimeError(f"openssl {command[0]} failed: {result.stderr.strip()}") + + +def _fleet_receiver_config() -> dict: + """A disposable stand-in for the fleet receiver: TLS, bearer auth, and a log.""" + return { + "receivers": { + "otlp": { + "protocols": { + "http": { + "endpoint": "0.0.0.0:4318", + "auth": {"authenticator": "bearertokenauth"}, + "tls": { + "cert_file": "/etc/otel/server.crt", + "key_file": "/etc/otel/server.key", + }, + } + } + } + }, + "exporters": {"debug": {"verbosity": "detailed"}}, + "extensions": { + "health_check": {"endpoint": "0.0.0.0:13133"}, + "bearertokenauth": {"scheme": "Bearer", "token": RELAY_TOKEN}, + }, + "service": { + "extensions": ["health_check", "bearertokenauth"], + "pipelines": {"traces": {"receivers": ["otlp"], "exporters": ["debug"]}}, + }, + } + + +def _agent_host_payload(marker: str) -> dict: + """A trace only the agent host would produce, carrying a unique marker.""" + return { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "copilot-agent-host"}} + ] + }, + "scopeSpans": [ + { + "scope": {"name": "copilot-otel-relay-test"}, + "spans": [ + { + "traceId": "d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5", + "spanId": "a1b2c3d4e5f60718", + "name": "invoke_agent", + "kind": 1, + "startTimeUnixNano": "1700000000000000000", + "endTimeUnixNano": "1700000000100000000", + "attributes": [ + { + "key": RELAY_MARKER_ATTRIBUTE, + "value": {"stringValue": marker}, + } + ], + } + ], + } + ], + } + ] + } + + +def _container_logs(runtime: ContainerRuntime, container: str) -> str: + logs = runtime.docker("logs", container, timeout=60) + return logs.stdout + logs.stderr + + +def run_relay_case(runtime: ContainerRuntime, case: RelayCase) -> tuple[str, str]: + """Run the shipped relay against a disposable fleet receiver. + + Returns the receiver's output and the relay's own output. The relay + configuration is used exactly as shipped, including its ten second batch + window, so nothing about the transport under test is a test-only variant. + """ + marker = f"RELAYMARKER{case.name.upper().replace('-', '')}{_free_port()}" + network = f"copilot-otel-relay-net-{_free_port()}" + receiver_container = f"copilot-otel-relay-receiver-{_free_port()}" + relay_container = f"copilot-otel-relay-{_free_port()}" + relay_http_port = _free_port() + receiver_health_port = _free_port() + relay_health_port = _free_port() + + with tempfile.TemporaryDirectory(prefix="copilot-otel-relay-") as raw_workdir: + workdir = pathlib.Path(raw_workdir) + _issue_relay_material(runtime, workdir) + receiver_config = workdir / "receiver.yaml" + receiver_config.write_text( + yaml.safe_dump(_fleet_receiver_config(), sort_keys=False), encoding="utf-8" + ) + + created = runtime.docker("network", "create", network) + if created.returncode != 0: + raise RuntimeError(f"relay test network failed: {created.stderr.strip()}") + try: + started = runtime.docker( + "run", + "-d", + "--name", + receiver_container, + "--network", + network, + "--network-alias", + RELAY_SAN_HOST, + "--network-alias", + RELAY_WRONG_HOST, + "-p", + f"127.0.0.1:{receiver_health_port}:13133", + "-v", + f"{runtime.mount_source(receiver_config)}:/etc/otelcol-contrib/config.yaml:ro", + "-v", + f"{runtime.mount_source(workdir / 'server.crt')}:/etc/otel/server.crt:ro", + "-v", + f"{runtime.mount_source(workdir / 'server.key')}:/etc/otel/server.key:ro", + pinned_collector_image(), + "--config=/etc/otelcol-contrib/config.yaml", + ) + if started.returncode != 0: + raise RuntimeError(f"fleet receiver failed to start: {started.stderr.strip()}") + + started = runtime.docker( + "run", + "-d", + "--name", + relay_container, + "--network", + network, + "-e", + f"COPILOT_OTEL_FLEET_ENDPOINT=https://{case.endpoint_host}:4318", + "-e", + f"COPILOT_OTEL_INGEST_TOKEN={case.token}", + "-p", + f"127.0.0.1:{relay_http_port}:4318", + "-p", + f"127.0.0.1:{relay_health_port}:13133", + "-v", + f"{runtime.mount_source(RELAY_COLLECTOR_PATH)}:/etc/otelcol-contrib/config.yaml:ro", + "-v", + f"{runtime.mount_source(workdir / case.ca_file)}:{RELAY_CA_MOUNT}:ro", + pinned_collector_image(), + "--config=/etc/otelcol-contrib/config.yaml", + ) + if started.returncode != 0: + raise RuntimeError(f"relay failed to start: {started.stderr.strip()}") + + for name, port in ( + (receiver_container, receiver_health_port), + (relay_container, relay_health_port), + ): + if not _wait_for_health(port, timeout=60): + raise RuntimeError( + f"{name} never became healthy; output:\n{_container_logs(runtime, name)}" + ) + + _post(relay_http_port, "traces", _agent_host_payload(marker)) + + # The shipped relay batches for ten seconds, so no verdict is + # available until that window has passed at least once. + deadline = time.monotonic() + 45 + receiver_output = "" + while time.monotonic() < deadline: + receiver_output = _container_logs(runtime, receiver_container) + if marker in receiver_output: + break + time.sleep(1) + relay_output = _container_logs(runtime, relay_container) + return receiver_output.replace(marker, "AGENTHOSTMARKER"), relay_output + finally: + for name in (relay_container, receiver_container): + runtime.docker("rm", "-f", name, timeout=120) + runtime.docker("network", "rm", network, timeout=120) + + +@pytest.fixture(scope="module") +def relay_results(runtime: ContainerRuntime) -> dict[str, tuple[str, str]]: + """Every relay case, run once, because each one starts two containers.""" + resolve_openssl(runtime) + return {case.name: run_relay_case(runtime, case) for case in RELAY_CASES} + + +class TestAgentHostRelay: + """The shipped relay forwards only over verified TLS with the fleet credential. + + A static check can confirm the relay declares a CA file and an + authenticator. It cannot confirm that the Collector refuses a receiver + signed by another authority, or one whose certificate does not name the + host the relay dialled. Those are the failures that would otherwise be + found by a fleet already sending telemetry somewhere it should not. + """ + + def test_a_trusted_receiver_with_the_right_credential_receives_the_span( + self, relay_results: dict[str, tuple[str, str]] + ) -> None: + receiver_output, _ = relay_results["trusted"] + assert "AGENTHOSTMARKER" in receiver_output, ( + "the relay did not forward agent-host telemetry over verified TLS" + ) + + def test_the_forwarded_span_keeps_its_agent_host_identity( + self, relay_results: dict[str, tuple[str, str]] + ) -> None: + receiver_output, _ = relay_results["trusted"] + assert "copilot-agent-host" in receiver_output + + @pytest.mark.parametrize("case", [entry.name for entry in RELAY_CASES if not entry.forwards]) + def test_a_rejected_transport_forwards_nothing( + self, relay_results: dict[str, tuple[str, str]], case: str + ) -> None: + receiver_output, _ = relay_results[case] + assert "AGENTHOSTMARKER" not in receiver_output, ( + f"the {case} case reached the fleet receiver anyway" + ) + + def test_a_wrong_bearer_is_refused_by_the_receiver( + self, relay_results: dict[str, tuple[str, str]] + ) -> None: + _, relay_output = relay_results["wrong-bearer"] + assert "401" in relay_output or "unauthorized" in relay_output.lower() + + @pytest.mark.parametrize("case", ["untrusted-ca", "hostname-mismatch"]) + def test_a_certificate_failure_is_reported_as_one( + self, relay_results: dict[str, tuple[str, str]], case: str + ) -> None: + _, relay_output = relay_results[case] + assert "certificate" in relay_output.lower() or "tls" in relay_output.lower() From d4f212d2f8bf4bbed28e730223a86cc58f0b2a63 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Tue, 18 Aug 2026 21:20:24 -0700 Subject: [PATCH 11/28] fix(skills): tighten helper endpoint policy, settings writes, and container bounds Resolve endpoint hosts before deciding loopback so a name that answers with both a loopback and a routable address is refused instead of accepted, and require a globally routable address for an opted-in remote endpoint. Write settings through a staged temporary file and an atomic replace, preserve a UTF-8 BOM, handle empty and comments-only files, evaluate policy against the merged result rather than the incoming fragment, drop userinfo from endpoint summaries, and retain a bounded number of backups in creation order. Bound the local Collector container with a memory limit, a read-only root, dropped capabilities, and no-new-privileges, and widen the blocked-value patterns to cover temporary AWS keys, unsigned JWTs, Azure account and shared-access keys, and common model-provider key shapes. --- .../copilot-otel-metrics/examples/README.md | 4 +- .../examples/_input_policy.py | 65 ++- .../examples/compose.yaml | 10 + .../examples/otel-collector-local.yaml | 21 +- .../examples/settings_upsert.py | 319 +++++++++++++-- .../examples/validate_dashboard.py | 70 +++- .../references/local-setup.md | 18 +- .../references/local-stack.md | 2 +- .../tests/test_helpers.py | 166 +++++++- .../tests/test_local_config.py | 384 +++++------------- .../tests/test_settings_upsert.py | 326 ++++++++++++++- 11 files changed, 1005 insertions(+), 380 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/README.md b/.github/skills/experimental/copilot-otel-metrics/examples/README.md index 9e2d18361..519f58d2e 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/README.md +++ b/.github/skills/experimental/copilot-otel-metrics/examples/README.md @@ -2,7 +2,7 @@ title: Copilot OTel Metrics Examples description: Reference stack definition, dashboard, and helper scripts for capturing GitHub Copilot OpenTelemetry output locally author: Microsoft -ms.date: 2026-08-13 +ms.date: 2026-08-18 ms.topic: reference keywords: - opentelemetry @@ -93,7 +93,7 @@ To move to a newer image, resolve the multi-architecture manifest digest for the `baseline.py` writes its snapshot to `~/.cache/copilot-otel/pre-enable-baseline.json`. Set `COPILOT_OTEL_BASELINE` to choose a different path. Capture before enabling export, diff afterwards. -`validate_dashboard.py` imports with `overwrite: true`, so it replaces any dashboard sharing the uid. It refuses a Grafana that is not on loopback unless you set `COPILOT_OTEL_ALLOW_REMOTE=1`. Pass a dashboard path to check a generated dashboard instead of the bundled one, and override the target with `COPILOT_OTEL_GRAFANA`, `COPILOT_OTEL_GRAFANA_USER`, and `COPILOT_OTEL_GRAFANA_PASSWORD`. +`validate_dashboard.py` imports with `overwrite: true`, so it replaces any dashboard sharing the uid. It refuses a Grafana that is not on loopback unless you set `COPILOT_OTEL_ALLOW_REMOTE=1`. Pass a dashboard path to check a generated dashboard instead of the bundled one; the path is resolved against the directory you run in, so a dashboard produced anywhere on this machine can be checked. Override the target with `COPILOT_OTEL_GRAFANA`, `COPILOT_OTEL_GRAFANA_USER`, and `COPILOT_OTEL_GRAFANA_PASSWORD`. `inspect_metrics.py` is the answer to "is this metric name still correct". Run it before trusting any metric name copied from documentation, including the tables in the skill itself. diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py index 0e280e628..83872a3ae 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py @@ -20,6 +20,7 @@ import ipaddress import os import pathlib +import socket import urllib.parse import urllib.request @@ -28,9 +29,11 @@ "PolicyError", "check_url", "contain_path", + "is_loopback_host", "open_url", "origin_of", "require_credentials", + "resolve_addresses", ] @@ -43,7 +46,6 @@ class PolicyError(ValueError): DEFAULT_ALLOWED_PORTS = frozenset({3000, 3200, 4317, 4318, 9090}) _ALLOWED_SCHEMES = frozenset({"http", "https"}) -_LOOPBACK_NAMES = frozenset({"localhost", "ip6-localhost", "ip6-loopback"}) # Headers that identify the caller to the origin they were addressed to. They # are meaningless to a different origin and dangerous there. @@ -70,14 +72,44 @@ def origin_of(url: str) -> tuple[str, str, int]: return scheme, host, port if port is not None else _DEFAULT_PORTS.get(scheme, -1) -def _is_loopback(host: str) -> bool: - """Whether a hostname or address refers to this machine.""" - if host.lower() in _LOOPBACK_NAMES: - return True +def resolve_addresses(host: str) -> tuple[ipaddress.IPv4Address | ipaddress.IPv6Address, ...]: + """Every address this host currently resolves to. + + An address literal resolves to itself. A name is resolved, because a name + says nothing about where a connection goes: `localhost` can be redefined in + a hosts file, and a name that looks local can answer with a routable + address. Resolution failure is refused rather than treated as either + answer. + """ + literal = host.strip("[]") try: - return ipaddress.ip_address(host.strip("[]")).is_loopback + return (ipaddress.ip_address(literal),) except ValueError: - return False + pass + + try: + infos = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM) + except (socket.gaierror, UnicodeError, ValueError) as exc: + raise PolicyError(f"refusing '{host}': it does not resolve ({exc})") from exc + + addresses = {ipaddress.ip_address(info[4][0].split("%", 1)[0]) for info in infos} + if not addresses: + raise PolicyError(f"refusing '{host}': it resolves to no address") + return tuple(sorted(addresses, key=str)) + + +def is_loopback_host(host: str) -> bool: + """Whether every address this host resolves to is on this machine. + + Every address, not any: a name answering with one loopback address and one + routable address would otherwise be treated as local while a connection + could still leave the machine. + + This narrows where a request may go. It does not prevent DNS rebinding, + because the transport resolves the name again when it connects and may get + a different answer than this check saw. + """ + return all(address.is_loopback for address in resolve_addresses(host)) def check_url( @@ -116,8 +148,9 @@ def check_url( raise PolicyError("refusing a URL with no host") resolved_port = port if port is not None else (443 if parsed.scheme == "https" else 80) + addresses = resolve_addresses(host) - if _is_loopback(host): + if all(address.is_loopback for address in addresses): if resolved_port not in allowed_ports: raise PolicyError( f"refusing local port {resolved_port}: expected one of " @@ -125,6 +158,12 @@ def check_url( ) return parsed + if any(address.is_loopback for address in addresses): + raise PolicyError( + f"refusing '{host}': it resolves to both loopback and non-loopback addresses, " + "so where a connection lands is not decidable here" + ) + if not allow_remote: raise PolicyError( f"refusing non-loopback host '{host}'. " @@ -132,6 +171,16 @@ def check_url( ) if parsed.scheme != "https": raise PolicyError(f"refusing plaintext http to remote host '{host}': use https") + + # An opted-in remote target must be genuinely remote. Private, link-local, + # and metadata-service ranges are the addresses a redirect would choose to + # reach something inside the network that never expected this request. + internal = [address for address in addresses if not address.is_global] + if internal: + raise PolicyError( + f"refusing remote host '{host}': it resolves to non-global " + f"address(es) {', '.join(str(address) for address in internal)}" + ) return parsed diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml index 4e13e5cb1..6a54f0291 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml @@ -45,6 +45,16 @@ services: - ./otel-collector-local.yaml:/etc/otelcol-contrib/config.yaml:ro networks: - copilot-otel + # memory_limiter resolves its percentages against the container's cgroup + # limit when one exists and against total host memory when one does not. + # Without this line its stated bound is 80% of the workstation's RAM, in + # front of an ingress any local process can reach. + mem_limit: 512m + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true depends_on: lgtm: condition: service_healthy diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml index 7c1779db0..b4f042921 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml @@ -88,12 +88,22 @@ processors: # Failure triage without failure content. - error.type # Masked, not dropped, so a panel keeps its shape while the value is gone. + # These are recognizable shapes, not secret detection: a credential with no + # distinctive prefix passes, and that limit is recorded in SECURITY.md. blocked_values: - "gh[pousr]_[0-9A-Za-z]{16,}" - "github_pat_[0-9A-Za-z_]{20,}" - "xox[baprs]-[0-9A-Za-z-]{10,}" - - "AKIA[0-9A-Z]{16}" - - "eyJ[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]{10,}" + # Temporary credentials use the ASIA prefix; AKIA alone missed them. + - "A(?:KIA|SIA)[0-9A-Z]{16}" + # An unsigned token carries the same payload, so the third segment may + # legitimately be empty. + - "eyJ[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]{10,}\\.[0-9A-Za-z_-]*" + # Azure connection-string secrets. + - "(?i)(?:AccountKey|SharedAccessKey|SharedAccessSignature)=[0-9A-Za-z+/=%&-]{16,}" + # Provider API keys in the shapes most likely to be pasted into a chat. + - "sk-(?:ant-)?[0-9A-Za-z_-]{20,}" + - "AIza[0-9A-Za-z_-]{35}" - "(?i)-----BEGIN [A-Z ]*PRIVATE KEY-----" - "(?i)\\b(?:bearer|authorization|api[_-]?key|secret|password)\\b\\s*[:=]\\s*\\S+" # Silent so the processor does not annotate telemetry with the names of the @@ -122,6 +132,13 @@ processors: # this distribution provides no way to reach them. OTTL has no `spanlink` # context and refuses to index `links`, and assigning to `datapoint.exemplars` # parses but does nothing. Both are recorded as gaps. + # `error_mode: ignore` is deliberate and load-bearing. A statement that + # errors on one record shape is skipped for that record, which is why a + # map-valued log body keeps the allow-listed subset rather than being + # replaced wholesale. It also makes this processor fail-open per statement, + # unlike the allow-list above, which is fail-closed. The digest pins and the + # runtime carrier harness are what bound that; the residual is registered in + # SECURITY.md rather than left implicit. transform/scrub: error_mode: ignore trace_statements: diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py index 045e53377..ad8705092 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py @@ -23,12 +23,29 @@ import argparse import datetime import json +import logging +import os import pathlib import shutil import sys import urllib.parse -from _input_policy import DEFAULT_ALLOWED_PORTS, PolicyError, check_url, contain_path +from _input_policy import DEFAULT_ALLOWED_PORTS, PolicyError, check_url + +LOGGER = logging.getLogger("settings_upsert") + +# Exit codes. 2 separates "you asked for something that cannot be done" from a +# crash, and 130 is the shell's convention for an interrupt. +EXIT_OK = 0 +EXIT_REFUSED = 2 +EXIT_INTERRUPTED = 130 + +# Backups accumulate on every apply. Five is enough to walk back a bad session +# and few enough that a settings directory does not fill with copies of a file +# that may name an output path. +BACKUP_RETENTION = 5 + +UTF8_BOM = "\ufeff" # Provenance for the schema below. This is the authoritative manifest that was # read, not a recollection of one. @@ -95,15 +112,75 @@ def parse_assignment(raw: str) -> tuple[str, object]: return key, text -def check_policy(values: dict[str, object]) -> None: - """Apply the local-mode rules that the schema alone cannot express.""" +def repository_root() -> pathlib.Path: + """The checkout this script is part of, or the skill directory if none.""" + here = pathlib.Path(__file__).resolve() + for candidate in (here, *here.parents): + if (candidate / ".git").exists(): + return candidate + return here.parents[1] + + +def check_outfile(outfile: str) -> None: + """Refuse an output path that would write telemetry somewhere unsafe. + + A relative path resolves against whatever directory VS Code happened to + start in, which is not a location anyone chose. A path inside this checkout + puts captured spans where a commit would pick them up. + + Absoluteness is judged under both path flavours, because a settings file is + routinely authored on one platform for another and `/var/log/spans.jsonl` + is a deliberate absolute path even when this script runs on Windows. + """ + expanded = os.path.expanduser(outfile) + # Judged on the supplied text: constructing a `pathlib.Path` first would + # rewrite `/var/log/spans.jsonl` into a rootless Windows path and report a + # deliberate POSIX absolute path as relative. + if not ( + pathlib.PurePosixPath(expanded).is_absolute() + or pathlib.PureWindowsPath(expanded).is_absolute() + ): + raise SettingsError( + f"outfile must be an absolute path, got '{outfile}'. A relative path " + "resolves against the editor's working directory." + ) + + candidate = pathlib.Path(expanded) + if not candidate.is_absolute(): + # A path absolute only under the other platform's rules cannot name a + # location inside this checkout. + return + root = repository_root() + resolved = candidate.resolve() + if resolved == root or root in resolved.parents: + raise SettingsError( + f"refusing an outfile inside this repository: {resolved}. Captured " + "telemetry would be a commit away from being published." + ) + + +def check_policy( + values: dict[str, object], + *, + existing: dict[str, object] | None = None, + allow_remote_endpoint: bool = False, +) -> None: + """Apply the rules the schema alone cannot express. + + Cross-setting rules are evaluated against the settings that would result, + not against this invocation's arguments. Setting `exporterType` alone + otherwise passes while producing a file that contradicts the `outfile` + already written beside it. + """ + merged = {**(existing or {}), **values} + if values.get("github.copilot.chat.otel.captureContent") is True: raise SettingsError( "refusing to enable captureContent. It puts prompt text, responses, system " "instructions, and tool arguments onto spans. Set it by hand, deliberately." ) - exporter = values.get("github.copilot.chat.otel.exporterType") + exporter = merged.get("github.copilot.chat.otel.exporterType") if exporter is not None and exporter not in EXPORTER_TYPES: raise SettingsError( f"exporterType must be one of {', '.join(sorted(EXPORTER_TYPES))}, got '{exporter}'" @@ -112,18 +189,24 @@ def check_policy(values: dict[str, object]) -> None: endpoint = values.get("github.copilot.chat.otel.otlpEndpoint") if endpoint: try: - check_url(str(endpoint), allow_remote=False, allowed_ports=DEFAULT_ALLOWED_PORTS) + check_url( + str(endpoint), + allow_remote=allow_remote_endpoint, + allowed_ports=DEFAULT_ALLOWED_PORTS, + ) except PolicyError as exc: raise SettingsError(f"otlpEndpoint rejected: {exc}") from exc - outfile = values.get("github.copilot.chat.otel.outfile") - if outfile and exporter is not None and exporter.startswith("otlp-"): - raise SettingsError( - "outfile forces the file exporter, which contradicts the requested " - f"exporterType '{exporter}'. Set one or the other." - ) + outfile = merged.get("github.copilot.chat.otel.outfile") + if outfile: + check_outfile(str(outfile)) + if exporter is not None and str(exporter).startswith("otlp-"): + raise SettingsError( + "outfile forces the file exporter, which contradicts the resulting " + f"exporterType '{exporter}'. Set one or the other." + ) - size = values.get("github.copilot.chat.otel.maxAttributeSizeChars") + size = merged.get("github.copilot.chat.otel.maxAttributeSizeChars") if size is not None and int(size) < 0: raise SettingsError("maxAttributeSizeChars cannot be negative") @@ -289,16 +372,23 @@ def summarize(key: str, value: object) -> object: def summarize_endpoint(value: str) -> str: - """Reduce a URL to scheme and authority. - - An OTLP endpoint's path and query are operator-supplied and can carry a - tenant identifier or a token. Neither is needed to answer the question the - record is kept for, which is where telemetry was pointed. + """Reduce a URL to scheme, host, and port. + + The authority is not safe to keep whole: `https://user:token@host:4318` + puts a credential in the netloc, and an audit record that copies it turns a + record of what changed into a second place the credential lives. An OTLP + endpoint's path and query are operator-supplied and can carry a tenant + identifier or a token for the same reason. Host and port answer the only + question the record is kept for, which is where telemetry was pointed. """ parsed = urllib.parse.urlparse(value) - if not parsed.scheme or not parsed.netloc: + try: + host, port = parsed.hostname, parsed.port + except ValueError: return f"" - return f"{parsed.scheme}://{parsed.netloc}" + if not parsed.scheme or not host: + return f"" + return f"{parsed.scheme}://{host}:{port}" if port else f"{parsed.scheme}://{host}" def write_audit(audit_path: pathlib.Path, record: dict[str, object]) -> None: @@ -315,36 +405,119 @@ def backup_path_for( A fixed `.bak` name makes the second run destroy the only copy of the original, which is the opposite of what a backup is for. The timestamp is - to the second and a counter resolves same-second collisions, so repeated - runs accumulate rather than overwrite. + to the second, so repeated runs accumulate rather than overwrite. - Retention is the operator's: this tool never deletes a backup it wrote. + The counter is always present and zero-padded. An optional suffix would + sort `...Z-1.bak` before `...Z.bak`, and retention reads that order to + decide what to delete, so the newest backups would be the ones pruned. """ stamp = (now or datetime.datetime.now(datetime.UTC)).strftime("%Y%m%dT%H%M%SZ") - candidate = settings_path.with_name(f"{settings_path.name}.{stamp}.bak") - collision = 1 + collision = 0 + candidate = settings_path.with_name(f"{settings_path.name}.{stamp}-{collision:03d}.bak") while candidate.exists(): - candidate = settings_path.with_name(f"{settings_path.name}.{stamp}-{collision}.bak") collision += 1 + candidate = settings_path.with_name(f"{settings_path.name}.{stamp}-{collision:03d}.bak") return candidate +def existing_backups(settings_path: pathlib.Path) -> list[pathlib.Path]: + """Backups this tool wrote for one settings file, oldest first. + + Sorted by name rather than by modification time, because the name carries + the stamp this tool assigned and a copy operation does not preserve + ordering the way that stamp does. + """ + return sorted(settings_path.parent.glob(f"{settings_path.name}.*.bak"), key=lambda p: p.name) + + +def prune_backups( + settings_path: pathlib.Path, *, retain: int = BACKUP_RETENTION +) -> list[pathlib.Path]: + """Delete all but the newest `retain` backups and return what was removed. + + Unbounded retention was the earlier behaviour, which left a growing set of + copies of a file whose contents include an output path and an endpoint. + Removals are returned so the caller can report them rather than deleting + silently. + """ + backups = existing_backups(settings_path) + removed: list[pathlib.Path] = [] + for stale in backups[: max(len(backups) - retain, 0)]: + try: + stale.unlink() + except OSError as exc: + LOGGER.warning("could not remove old backup %s: %s", stale, exc) + continue + removed.append(stale) + return removed + + +def read_settings(settings_path: pathlib.Path) -> tuple[str, str]: + """Return the document's byte-order mark and its text without it. + + VS Code writes `settings.json` with a BOM on some installs. Decoding with + `utf-8-sig` and writing back without it silently rewrites a byte the editor + put there; keeping the BOM as a separate value means the write puts back + exactly what it found. + + A missing, empty, whitespace-only, or comments-only file is an editable + empty object. All four are documents a person can reasonably have, and + refusing them meant the tool could not perform the first edit on a settings + file that had never held a setting. + """ + if not settings_path.is_file(): + return "", "{}\n" + + try: + raw = settings_path.read_text(encoding="utf-8-sig") + except (OSError, UnicodeDecodeError) as exc: + raise SettingsError(f"cannot read {settings_path}: {exc}") from exc + + bom = UTF8_BOM if settings_path.read_bytes().startswith(b"\xef\xbb\xbf") else "" + if not strip_jsonc(raw).strip(): + # Comments are preserved: they are the only content this document has, + # and discarding them would be the unrelated change this tool refuses + # to make everywhere else. + return bom, f"{raw.rstrip()}\n{{}}\n" if raw.strip() else "{}\n" + return bom, raw + + +def write_atomically(settings_path: pathlib.Path, text: str) -> None: + """Stage beside the target, validate the staged bytes, then replace. + + Writing in place truncates first, so an interrupted write leaves a settings + file that is neither the old document nor the new one. Staging in the same + directory keeps the replace on one filesystem, which is what makes it + atomic. + """ + staged = settings_path.with_name(f"{settings_path.name}.{os.getpid()}.staged") + try: + staged.write_text(text, encoding="utf-8") + json.loads(strip_jsonc(staged.read_text(encoding="utf-8-sig")) or "{}") + os.replace(staged, settings_path) + except BaseException: + staged.unlink(missing_ok=True) + raise + + def apply_changes( settings_path: pathlib.Path, values: dict[str, object], *, apply: bool, audit_path: pathlib.Path | None = None, + allow_remote_endpoint: bool = False, ) -> str: - """Validate, then optionally write, restoring the backup if the result is invalid.""" - check_policy(values) + """Validate, then optionally write, leaving the original intact on any failure.""" + bom, original = read_settings(settings_path) - original = settings_path.read_text(encoding="utf-8") if settings_path.is_file() else "{}\n" try: before = json.loads(strip_jsonc(original) or "{}") except json.JSONDecodeError as exc: raise SettingsError(f"existing settings file is not valid JSONC: {exc}") from exc + check_policy(values, existing=before, allow_remote_endpoint=allow_remote_endpoint) + updated = upsert(original, values) try: @@ -366,13 +539,10 @@ def apply_changes( backup_path = backup_path_for(settings_path) if settings_path.is_file(): shutil.copy2(settings_path, backup_path) - try: - settings_path.write_text(updated, encoding="utf-8") - json.loads(strip_jsonc(settings_path.read_text(encoding="utf-8")) or "{}") - except Exception: - if backup_path.is_file(): - shutil.copy2(backup_path, settings_path) - raise + write_atomically(settings_path, bom + updated) + + for removed in prune_backups(settings_path): + LOGGER.info("removed old backup %s", removed) if audit_path is not None: write_audit( @@ -394,13 +564,62 @@ def apply_changes( return updated +def resolve_audit_path(argument: str, settings_path: pathlib.Path) -> pathlib.Path: + """Resolve the audit path against the caller's directory, refusing aliases. + + The audit path used to be contained to the settings directory, which + silently relocated an operator's chosen path into the one directory where + an appended JSON line can destroy the file being edited. Resolving from the + current directory keeps the operator's choice, and the aliases that would + corrupt settings or a backup are refused outright instead. + """ + path = pathlib.Path(argument).expanduser() + if not path.is_absolute(): + path = pathlib.Path.cwd() / path + path = path.resolve() + + if path == settings_path.resolve(): + raise SettingsError("refusing an audit path that is the settings file itself") + if path.name.endswith(".bak") and path.parent == settings_path.resolve().parent: + raise SettingsError(f"refusing an audit path that is a settings backup: {path}") + if path.is_dir(): + raise SettingsError(f"audit path is a directory: {path}") + return path + + +def silence_broken_pipe() -> None: + """Point stdout at the null device after the reader has gone away. + + Without this the interpreter reports the same broken pipe again while + flushing during shutdown, turning a pager quit into a second error message + and a non-zero status. + """ + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + + +def configure_logging(verbose: bool = False) -> None: + """Send diagnostics to stderr so stdout carries only the document.""" + logging.basicConfig( + stream=sys.stderr, + level=logging.DEBUG if verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--settings", required=True, help="path to the settings.json to edit") parser.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") parser.add_argument("--apply", action="store_true", help="write; omit for a dry run") parser.add_argument("--audit", help="append an audit record to this path") + parser.add_argument( + "--allow-remote-endpoint", + action="store_true", + help="permit an https endpoint outside this machine; loopback is the default", + ) + parser.add_argument("--verbose", action="store_true", help="log at debug level") args = parser.parse_args(argv) + configure_logging(args.verbose) try: values: dict[str, object] = {} @@ -413,17 +632,31 @@ def main(argv: list[str] | None = None) -> int: raise SettingsError("nothing to do: pass at least one --set key=value") settings_path = pathlib.Path(args.settings).expanduser() - audit_path = contain_path(args.audit, settings_path.parent) if args.audit else None - updated = apply_changes(settings_path, values, apply=args.apply, audit_path=audit_path) + audit_path = resolve_audit_path(args.audit, settings_path) if args.audit else None + updated = apply_changes( + settings_path, + values, + apply=args.apply, + audit_path=audit_path, + allow_remote_endpoint=args.allow_remote_endpoint, + ) except (SettingsError, PolicyError) as exc: print(str(exc), file=sys.stderr) - return 1 + return EXIT_REFUSED + except KeyboardInterrupt: + return EXIT_INTERRUPTED - if not args.apply: - print(updated) - print("\n(dry run; pass --apply to write)", file=sys.stderr) - return 0 + try: + if not args.apply: + print(updated) + print("\n(dry run; pass --apply to write)", file=sys.stderr) + sys.stdout.flush() + except BrokenPipeError: + # A closed reader is the pager quitting, not a failure of the edit. + silence_broken_pipe() + return EXIT_OK + return EXIT_OK if __name__ == "__main__": - raise SystemExit(main()) + sys.exit(main()) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py b/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py index 9c61836b0..44e183600 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/validate_dashboard.py @@ -9,6 +9,12 @@ validate_dashboard.py [dashboard.json] +The dashboard argument is resolved against the current directory, so a +generated dashboard anywhere on this machine can be checked. Confining it to +the installed skill directory made the documented workflow impossible: a +dashboard worth validating is usually one that was just produced, not one that +ships here. + Handles Prometheus and Tempo panels only. There is no Azure Monitor path, so this cannot validate the Azure dashboard. @@ -43,7 +49,13 @@ import urllib.request from collections.abc import Callable -from _input_policy import PolicyError, check_url, contain_path, open_url, require_credentials +from _input_policy import ( + PolicyError, + check_url, + is_loopback_host, + open_url, + require_credentials, +) EXAMPLES_DIR = pathlib.Path(__file__).resolve().parent DEFAULT_GRAFANA = "http://localhost:3000" @@ -200,6 +212,43 @@ def validate_panels( return ok_count, empty_count +def load_dashboard(argument: str | None) -> dict: + """Read a caller-selected dashboard, or the bundled one, with reportable errors. + + Every failure here is a mistyped path or a malformed file, which is a + message the caller can act on. A traceback would report the same thing as + an internal defect. + """ + path = ( + pathlib.Path(argument).expanduser() + if argument + else EXAMPLES_DIR / "dashboards" / "copilot-otel.json" + ) + if not path.is_absolute(): + path = pathlib.Path.cwd() / path + + # Checked before reading because the two platforms disagree about which + # error a directory read raises, and the caller needs one message. + if path.is_dir(): + raise PolicyError(f"{path} is a directory, not a dashboard file") + + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as exc: + raise PolicyError(f"no dashboard at {path}") from exc + except (OSError, UnicodeDecodeError) as exc: + raise PolicyError(f"cannot read {path}: {exc}") from exc + + try: + dashboard = json.loads(text) + except json.JSONDecodeError as exc: + raise PolicyError(f"{path} is not valid JSON: {exc}") from exc + + if not isinstance(dashboard, dict) or not isinstance(dashboard.get("panels"), list): + raise PolicyError(f"{path} is not a Grafana dashboard: no panels array") + return dashboard + + def main(argv: list[str] | None = None) -> int: args = sys.argv[1:] if argv is None else argv grafana_url = os.environ.get("COPILOT_OTEL_GRAFANA", DEFAULT_GRAFANA) @@ -220,20 +269,19 @@ def main(argv: list[str] | None = None) -> int: user, password = require_credentials( "COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD" ) - # The import uses overwrite semantics, so a dashboard path outside this - # directory is refused rather than read. - dash_path = ( - contain_path(args[0], EXAMPLES_DIR) - if args - else EXAMPLES_DIR / "dashboards" / "copilot-otel.json" + dash = load_dashboard(args[0] if args else None) + # The import uses overwrite semantics, so a target that is not this + # machine is worth saying out loud. The same resolved-address rule the + # policy uses decides it; a hostname prefix comparison would call a + # name that merely starts with "localhost" local. + grafana_is_local = is_loopback_host( + check_url(grafana_url, allow_remote=allow_remote).hostname ) except PolicyError as exc: print(str(exc), file=sys.stderr) return 1 - if not check_url(grafana_url, allow_remote=allow_remote).hostname.startswith( - ("localhost", "127.", "::1") - ): + if not grafana_is_local: print( f"warning: importing into {grafana_url} overwrites any dashboard sharing the same uid.", file=sys.stderr, @@ -249,8 +297,6 @@ def store(url: str) -> dict: """Prometheus and Tempo. No credential is in scope for this closure.""" return fetch(url, allow_remote=allow_remote) - dash = json.loads(dash_path.read_text()) - # Silently falling through to Tempo for an unknown datasource would produce # empty results indistinguishable from a genuinely empty store, which is the # exact failure this tool exists to detect. diff --git a/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md b/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md index 8a26c5afe..9625a6445 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/local-setup.md @@ -80,7 +80,7 @@ Stop and ask if the file cannot be identified with confidence. Do not write to a Follow every step. Steps 1, 4, and 5 are what make this reversible and visible. -`examples/settings_upsert.py` implements this contract rather than describing it. Prefer it over an ad-hoc edit: it refuses any key outside the verified schema, rejects a type mismatch, an unsafe endpoint, an `outfile` and `otlp-*` combination, and any attempt to enable `captureContent`; it splices only the target value span, backs the file up, restores the backup if the result does not parse, and refuses to write if unrelated settings would change. Run it without `--apply` to get the diff for step 4. +`examples/settings_upsert.py` implements this contract rather than describing it. Prefer it over an ad-hoc edit: it refuses any key outside the verified schema, a type mismatch, an unsafe endpoint, an `outfile` and `otlp-*` combination in the settings that would result, a relative or in-repository `outfile`, and any attempt to enable `captureContent`. It splices only the target value span, preserves a byte-order mark and comments, backs the file up, stages the new document beside the target and replaces it atomically so an interrupted write cannot truncate it, and refuses to write if unrelated settings would change. Run it without `--apply` to get the diff for step 4. ```bash python3 settings_upsert.py --settings \ @@ -89,20 +89,28 @@ python3 settings_upsert.py --settings \ --set github.copilot.chat.otel.otlpEndpoint=http://localhost:4318 ``` +An endpoint outside this machine needs `--allow-remote-endpoint`, and is still required to be HTTPS and to resolve to a globally routable address. The default is loopback because the local stack is what this reference sets up. + +`--audit ` appends one summarized record per applied change. The path is yours and is resolved from the directory you run in; the tool refuses only the aliases that would destroy what it is protecting, meaning the settings file itself or one of its backups. + +It keeps the five newest backups and removes older ones, reporting each removal. Each backup is a full copy of a file that can name an output path and an endpoint, so they are not left to accumulate. + +Exit codes: `0` for success, `2` for a refused request, `130` for an interrupt. + Its schema is frozen against one build and records which one. Re-verify against the installed extension before trusting it on a newer version. The steps below remain the contract, whether the tool performs them or a hand edit does. -1. **Back up.** Copy the file alongside itself as `settings.json.bak-otel-`, for example `settings.json.bak-otel-20260727T142530Z`. Tell the user the backup path and that restoring it is a file copy in the other direction. That path sits beside `settings.json`, outside the workspace; name it when asking for the write rather than treating it as a separate approval. -2. **Read the file as text.** If it does not exist or is empty, treat the content as `{}`. It is JSONC: it may contain `//` and `/* */` comments and trailing commas, all of which are legal and all of which the user wrote deliberately. +1. **Back up.** Copy the file alongside itself with a UTC timestamp, as `settings.json.-000.bak`, for example `settings.json.20260727T142530Z-000.bak`. Tell the user the backup path and that restoring it is a file copy in the other direction. That path sits beside `settings.json`, outside the workspace; name it when asking for the write rather than treating it as a separate approval. +2. **Read the file as text.** If it does not exist, is empty, or holds only comments, treat the content as `{}` and keep any comments that are there. It is JSONC: it may contain `//` and `/* */` comments and trailing commas, all of which are legal and all of which the user wrote deliberately. Preserve a byte-order mark if the file has one; VS Code wrote it. 3. **Compute the change per key. Do not write yet.** Work out the exact edit without applying it, so step 4 has something to show. * For a key already present, the change replaces only its value span, leaving the key, its indentation, and any adjacent comment untouched. * For a key not present, it is collected and inserted immediately before the document's final `}`, matching the file's existing indentation and adding the separating comma to the previous last entry when one is needed. * Match keys at the document's top level only. `"github.copilot.chat.otel.enabled"` appearing inside a `[...]` override block or a nested object is a different key and is not a target. * **Never reserialize.** Reserializing the document destroys every comment and every formatting choice in a file the user owns. 4. **Show the exact diff and stop.** Present the computed change as a unified diff against the file on disk, and name the file path above it. Ask for approval. Do not fold this into a broader "shall I proceed" question that bundles other work. -5. **Apply the upsert**, exactly as shown in the approved diff and no more. -6. **Validate after writing.** Re-read the file and parse it with comments and trailing commas tolerated. If it does not parse, restore the backup immediately, tell the user it was restored, and hand them the manual block instead. +5. **Apply the upsert**, exactly as shown in the approved diff and no more. Write it by staging a file beside the target and replacing the target with it, so an interrupted write leaves either the old document or the new one and never a truncated one. +6. **Validate before replacing.** Parse the staged content with comments and trailing commas tolerated. If it does not parse, discard the staged file, leave the original in place, and hand the user the manual block instead. 7. **Reload.** These settings are read at window startup. Have the user run **Developer: Reload Window**. A skipped reload is the single most common cause of an empty dashboard. Every write gets its own diff and its own approval. An approval earlier in the session does not carry forward to the next key or the next value. diff --git a/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md b/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md index 1f2ba0b63..079f17f43 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md @@ -121,7 +121,7 @@ These are for the user to run. Present the command; let them decide. Everything | `settings_upsert.py` | Edits a settings file in place after a backup, with an optional audit record | Performing the assisted settings write | | `_input_policy.py` | Not run directly. Shared endpoint, redirect, path, and credential policy | Never offered; it is imported by the helpers above | -The configurable inputs of `baseline.py` and `validate_dashboard.py` go through `_input_policy.py` before any request or write. It refuses non-http schemes, credentials in the authority, unrelated local ports, and non-loopback hosts without an explicit opt-in, re-applies the same rules to every redirect target, and keeps written paths inside their root. `verify.py` and `inspect_metrics.py` take no configurable endpoint and are left as they are. +All five helpers import `_input_policy.py`, so every request and every write crosses it. It refuses non-http schemes, credentials in the authority, unrelated local ports, and hosts that do not resolve entirely to loopback without an explicit opt-in; it re-applies the same rules to every redirect target and keeps written paths inside their root. `verify.py` and `inspect_metrics.py` take no configurable endpoint, so they exercise the transport half only; `baseline.py`, `validate_dashboard.py`, and `settings_upsert.py` additionally pass configurable endpoints or paths through it. Names are classified by the addresses they resolve to rather than by how they are spelled, which is what stops a hosts-file entry making a routable host look local. `validate_dashboard.py` needs two constraints stated when offered. It handles Prometheus and Tempo panels only: it has no Azure Monitor path, so pointing it at the Azure dashboard imports the dashboard and then runs empty queries that prove nothing. And it imports with `overwrite: true`, so it replaces any dashboard sharing the uid; it refuses a non-loopback Grafana unless `COPILOT_OTEL_ALLOW_REMOTE=1` is set. It accepts a dashboard path argument so a generated local dashboard can be checked instead of the bundled one. The endpoint comes from `COPILOT_OTEL_GRAFANA` and defaults to the local stack. `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD` are required with no default: they are the same pair the compose file requires, so a running stack always has them, and the helper exits with a configuration error naming both rather than sending a request that fails as an auth error. diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py index eca8992a5..dc17e9b96 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py @@ -10,6 +10,7 @@ from __future__ import annotations import io +import ipaddress import json import os import pathlib @@ -17,6 +18,7 @@ import sys import urllib.request +import _input_policy import inspect_metrics import pytest import validate_dashboard @@ -26,6 +28,7 @@ PolicyError, check_url, contain_path, + is_loopback_host, open_url, origin_of, require_credentials, @@ -33,6 +36,37 @@ EXAMPLES_DIR = pathlib.Path(__file__).resolve().parents[1] / "examples" +# Address literals, so these tests exercise the policy rather than whatever +# DNS answers on the machine running them. 93.184.216.34 is globally routable +# and 10.0.0.7 is not; no test opens a connection to either. +GLOBAL_ADDRESS = "93.184.216.34" +PRIVATE_ADDRESS = "10.0.0.7" + + +@pytest.fixture +def resolves(monkeypatch: pytest.MonkeyPatch): + """Pin what each name resolves to, so the rule under test is the policy. + + Resolution is the thing this policy now depends on, which makes real DNS a + dependency of the test rather than a subject of it. An unroutable name and + a name that answers with a routable address are both failures worth + asserting, and neither is reliably reproducible against a real resolver. + """ + + def apply(mapping: dict[str, list[str]]) -> None: + def fake(host: str) -> tuple: + try: + return (ipaddress.ip_address(host.strip("[]")),) + except ValueError: + pass + if host not in mapping: + raise PolicyError(f"refusing '{host}': it does not resolve") + return tuple(ipaddress.ip_address(value) for value in mapping[host]) + + monkeypatch.setattr(_input_policy, "resolve_addresses", fake) + + return apply + class TestScheme: """Only http and https are addressable.""" @@ -93,19 +127,56 @@ def test_an_unrelated_local_port_is_refused(self, port: int) -> None: class TestRemoteOptIn: - """A remote target needs an explicit opt-in and TLS.""" + """A remote target needs an explicit opt-in, TLS, and a globally routable address.""" - def test_a_remote_host_is_refused_without_opt_in(self) -> None: + def test_a_remote_host_is_refused_without_opt_in(self, resolves) -> None: + resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) with pytest.raises(PolicyError, match="non-loopback"): check_url("https://grafana.example.com/") - def test_a_remote_host_is_allowed_with_opt_in(self) -> None: + def test_a_remote_host_is_allowed_with_opt_in(self, resolves) -> None: + resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) assert check_url("https://grafana.example.com/", allow_remote=True).scheme == "https" - def test_plaintext_remote_is_refused_even_with_opt_in(self) -> None: + def test_plaintext_remote_is_refused_even_with_opt_in(self, resolves) -> None: + resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) with pytest.raises(PolicyError, match="plaintext"): check_url("http://grafana.example.com/", allow_remote=True) + def test_an_opted_in_remote_name_resolving_inside_the_network_is_refused( + self, resolves + ) -> None: + """The opt-in permits a remote target, not a route back into the network.""" + resolves({"grafana.example.com": [PRIVATE_ADDRESS]}) + with pytest.raises(PolicyError, match="non-global"): + check_url("https://grafana.example.com/", allow_remote=True) + + +class TestHostResolution: + """A name is classified by where it resolves, not by how it is spelled.""" + + def test_a_local_looking_name_that_resolves_off_machine_is_not_local(self, resolves) -> None: + resolves({"localhost": [GLOBAL_ADDRESS]}) + assert is_loopback_host("localhost") is False + with pytest.raises(PolicyError, match="non-loopback"): + check_url("http://localhost:3000/") + + def test_a_name_resolving_to_both_is_refused(self, resolves) -> None: + """Treating a mixed answer as local would permit a connection off machine.""" + resolves({"split.example": ["127.0.0.1", GLOBAL_ADDRESS]}) + assert is_loopback_host("split.example") is False + with pytest.raises(PolicyError, match="loopback and non-loopback"): + check_url("http://split.example:3000/") + + def test_a_name_that_does_not_resolve_is_refused(self, resolves) -> None: + resolves({}) + with pytest.raises(PolicyError, match="does not resolve"): + check_url("http://nowhere.invalid:3000/") + + @pytest.mark.parametrize("literal", ["127.0.0.1", "[::1]"]) + def test_an_address_literal_needs_no_resolver(self, literal: str) -> None: + assert is_loopback_host(literal) is True + class TestRedirects: """Policy is re-applied to redirect targets, not only the original URL.""" @@ -118,7 +189,7 @@ def test_a_redirect_off_loopback_is_refused(self) -> None: request = urllib.request.Request("http://localhost:3000/api") with pytest.raises(PolicyError, match="non-loopback"): self._handler().redirect_request( - request, None, 302, "Found", {}, "http://evil.example.com/" + request, None, 302, "Found", {}, f"http://{GLOBAL_ADDRESS}/" ) def test_a_redirect_to_a_file_url_is_refused(self) -> None: @@ -254,28 +325,30 @@ def test_baseline_refuses_a_traversal_snapshot_path(self, tmp_path: pathlib.Path assert result.returncode != 0 assert "refusing a path outside" in (result.stderr + result.stdout) - def test_validate_dashboard_refuses_a_dashboard_outside_the_examples_directory( + def test_validate_dashboard_reports_a_malformed_dashboard_without_a_traceback( self, tmp_path: pathlib.Path ) -> None: - outside = tmp_path / "someone-elses-dashboard.json" - outside.write_text("{}", encoding="utf-8") + broken = tmp_path / "generated-dashboard.json" + broken.write_text("{not json", encoding="utf-8") result = run_helper( "validate_dashboard.py", - [str(outside)], + [str(broken)], { "COPILOT_OTEL_GRAFANA_USER": "operator", "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", }, ) - assert result.returncode != 0 - assert "refusing a path outside" in (result.stderr + result.stdout) + assert result.returncode == 1 + combined = result.stderr + result.stdout + assert "not valid JSON" in combined + assert "Traceback" not in combined def test_validate_dashboard_refuses_a_non_loopback_endpoint_without_opt_in(self) -> None: result = run_helper( "validate_dashboard.py", [], { - "COPILOT_OTEL_GRAFANA": "https://grafana.example.com", + "COPILOT_OTEL_GRAFANA": f"https://{GLOBAL_ADDRESS}:3000", "COPILOT_OTEL_GRAFANA_USER": "operator", "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", }, @@ -289,7 +362,7 @@ def test_validate_dashboard_refuses_an_unguarded_query_endpoint(self) -> None: "validate_dashboard.py", [], { - "COPILOT_OTEL_PROMETHEUS": "http://evil.example.com:9090", + "COPILOT_OTEL_PROMETHEUS": f"http://{GLOBAL_ADDRESS}:9090", "COPILOT_OTEL_GRAFANA_USER": "operator", "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", }, @@ -309,6 +382,59 @@ def test_validate_dashboard_reports_missing_credentials_as_configuration(self) - assert "COPILOT_OTEL_GRAFANA_PASSWORD" in combined +class TestDashboardSelection: + """A generated dashboard is the normal subject, not an intrusion. + + Confining the argument to the installed skill directory made the workflow + the README documents impossible: the dashboard worth checking is usually + one that was just produced somewhere else. What replaces the containment + check is a controlled failure for every way a selected file can be wrong. + """ + + def _dashboard(self) -> dict: + return {"panels": [{"title": "one", "type": "row"}]} + + def test_a_generated_dashboard_outside_the_skill_is_accepted( + self, tmp_path: pathlib.Path + ) -> None: + generated = tmp_path / "generated.json" + generated.write_text(json.dumps(self._dashboard()), encoding="utf-8") + assert validate_dashboard.load_dashboard(str(generated)) == self._dashboard() + + def test_a_relative_path_resolves_against_the_current_directory( + self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + (tmp_path / "generated.json").write_text(json.dumps(self._dashboard()), encoding="utf-8") + monkeypatch.chdir(tmp_path) + assert validate_dashboard.load_dashboard("generated.json") == self._dashboard() + + def test_the_bundled_dashboard_is_the_default(self) -> None: + assert validate_dashboard.load_dashboard(None)["panels"] + + def test_a_missing_file_is_reported_as_a_missing_file(self, tmp_path: pathlib.Path) -> None: + with pytest.raises(PolicyError, match="no dashboard at"): + validate_dashboard.load_dashboard(str(tmp_path / "absent.json")) + + def test_a_directory_is_reported_as_a_directory(self, tmp_path: pathlib.Path) -> None: + with pytest.raises(PolicyError, match="not a dashboard file"): + validate_dashboard.load_dashboard(str(tmp_path)) + + def test_invalid_json_is_reported_as_invalid_json(self, tmp_path: pathlib.Path) -> None: + broken = tmp_path / "broken.json" + broken.write_text("{not json", encoding="utf-8") + with pytest.raises(PolicyError, match="not valid JSON"): + validate_dashboard.load_dashboard(str(broken)) + + @pytest.mark.parametrize("content", ["[]", '{"title": "no panels"}', '{"panels": {}}']) + def test_json_that_is_not_a_dashboard_is_reported_as_such( + self, tmp_path: pathlib.Path, content: str + ) -> None: + candidate = tmp_path / "not-a-dashboard.json" + candidate.write_text(content, encoding="utf-8") + with pytest.raises(PolicyError, match="no panels array"): + validate_dashboard.load_dashboard(str(candidate)) + + def _json_response(payload: dict) -> io.BytesIO: """A minimal stand-in for what open_url returns.""" return io.BytesIO(json.dumps(payload).encode("utf-8")) @@ -414,8 +540,9 @@ def test_a_same_origin_redirect_keeps_the_credential(self) -> None: redirected = self._redirect("http://localhost:3000/api", "http://localhost:3000/other") assert redirected.headers["Authorization"] == "Basic issued-for-the-original-origin" - def test_an_explicit_default_port_is_the_same_origin(self) -> None: + def test_an_explicit_default_port_is_the_same_origin(self, resolves) -> None: """`https://h/x` and `https://h:443/y` are one origin, not two.""" + resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) redirected = self._redirect( "https://grafana.example.com/api", "https://grafana.example.com:443/other", @@ -423,7 +550,13 @@ def test_an_explicit_default_port_is_the_same_origin(self) -> None: ) assert redirected.headers["Authorization"] == "Basic issued-for-the-original-origin" - def test_a_different_remote_host_drops_the_credential(self) -> None: + def test_a_different_remote_host_drops_the_credential(self, resolves) -> None: + resolves( + { + "grafana.example.com": [GLOBAL_ADDRESS], + "someone-else.example.com": [GLOBAL_ADDRESS], + } + ) redirected = self._redirect( "https://grafana.example.com/api", "https://someone-else.example.com/api", @@ -431,7 +564,8 @@ def test_a_different_remote_host_drops_the_credential(self) -> None: ) assert "Authorization" not in redirected.headers - def test_a_different_remote_port_drops_the_credential(self) -> None: + def test_a_different_remote_port_drops_the_credential(self, resolves) -> None: + resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) redirected = self._redirect( "https://grafana.example.com/api", "https://grafana.example.com:9090/api", diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py index 4248fc5f2..aaf23b9b6 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py @@ -5,238 +5,36 @@ These tests parse the committed Compose and Collector documents and simulate the declared attribute policy. They never start a container and never assert that a running Collector behaves as configured; they assert only that the configuration -this skill ships declares the intended policy. +this skill ships declares the intended policy. Runtime behaviour is the subject +of `test_collector_carriers.py`, which is marked `slow` and does start one. """ from __future__ import annotations -import json -import pathlib import re -from dataclasses import dataclass from typing import Any import pytest -import yaml - -SKILL_ROOT = pathlib.Path(__file__).resolve().parents[1] -EXAMPLES_DIR = SKILL_ROOT / "examples" -COMPOSE_PATH = EXAMPLES_DIR / "compose.yaml" -COLLECTOR_PATH = EXAMPLES_DIR / "otel-collector-local.yaml" -DASHBOARD_PATH = EXAMPLES_DIR / "dashboards" / "copilot-otel.json" -BASELINE_PATH = EXAMPLES_DIR / "baseline.py" - -MASK = "****" -SCRUB = "[redacted]" - -DIGEST_REFERENCE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$") - -# --- Shipped-consumer derivation ------------------------------------------- -# -# The default disposition for a carrier the Collector does not govern is to -# scrub it. The only thing that earns an exception is a shipped consumer that -# reads it, so the consumer set has to be derived from the shipped artifacts -# rather than curated by hand. A hand-curated set cannot notice a newly -# required attribute, and a scrub decision made against a stale set breaks a -# panel silently. - -# Label names Prometheus reserves or Grafana generates. Excluded by rule, not -# by listing the ones this dashboard happens to use today. -PROMETHEUS_RESERVED_LABELS = frozenset({"le", "quantile", "__name__", "job", "instance"}) - -# A metric reference is an identifier immediately followed by a label matcher -# or a range selector. A function is followed by "(", so this separates the two -# without enumerating PromQL's function set. -PROMQL_METRIC = re.compile(r"\b([a-z_][a-z0-9_]*)\s*[{\[]") -# Grafana resolves label_values(metric, label): metric first, label second. -PROMQL_LABEL_VALUES = re.compile( - r"label_values\(\s*([a-z_][a-z0-9_]*)\s*,\s*([a-z_][a-z0-9_]*)\s*\)" +from _config_support import ( + COLLECTOR_PATH, + COMPOSE_PATH, + DIGEST_REFERENCE, + EXAMPLES_DIR, + MASK, + OBSERVED_CONTENT_ATTRIBUTES, + ConfigError, + allowed_keys, + blocked_values, + load_yaml_file, + load_yaml_text, + overreaching_statements, + prometheus_label_for, + published_ports, + redaction_policy, + scrub_statements, + shipped_consumers, + simulate_redaction, ) -PROMQL_GROUPING = re.compile(r"\b(?:by|without)\s*\(([^)]*)\)") -PROMQL_MATCHER = re.compile(r"([a-zA-Z_][a-zA-Z0-9_]*)\s*(?:=~|!~|=|!=)\s*\"") -# baseline.py reads label values directly rather than through a panel. Its -# reads are consumers too; leaving them out of the inventory is how a detection -# control gets scrubbed without anything noticing. -BASELINE_LABEL_READ = re.compile(r"label_values\(\s*\"([a-z_][a-z0-9_]*)\"\s*\)") - -# TraceQL. The lookbehind keeps `service.name=` and `mode_name !=` from -# reading as span-name matchers; only a bare `name` is the intrinsic. -TRACEQL_SPAN_NAME = re.compile(r"(? frozenset[str]: - """OTLP attribute keys a shipped consumer reads by their real name.""" - return self.span_attributes | self.resource_attributes - - -def _promql_labels(expr: str) -> set[str]: - labels: set[str] = set() - for clause in PROMQL_GROUPING.findall(expr): - labels.update(part.strip() for part in clause.split(",") if part.strip()) - labels.update(PROMQL_MATCHER.findall(expr)) - return labels - - -def shipped_consumers() -> ShippedConsumers: - """Derive the consumer inventory from the dashboard and the baseline helper.""" - dashboard = json.loads(DASHBOARD_PATH.read_text(encoding="utf-8")) - - metric_names: set[str] = set() - labels: set[str] = set() - span_names: list[str] = [] - span_attributes: set[str] = set() - resource_attributes: set[str] = set() - - for variable in dashboard.get("templating", {}).get("list", []): - query = variable.get("query") - if isinstance(query, str): - for metric, label in PROMQL_LABEL_VALUES.findall(query): - metric_names.add(metric) - labels.add(label) - - for panel in dashboard.get("panels", []): - if panel.get("type") == "row": - continue - source = panel.get("datasource", {}).get("type") - for target in panel.get("targets", []): - expr = target.get("expr") or target.get("query") or "" - if source == "prometheus": - metric_names.update(PROMQL_METRIC.findall(expr)) - labels.update(_promql_labels(expr)) - elif source == "tempo": - span_names.extend(TRACEQL_SPAN_NAME.findall(expr)) - for scope, key in TRACEQL_ATTRIBUTE.findall(expr): - target_set = span_attributes if scope == "span" else resource_attributes - target_set.add(key.rstrip(".")) - - # A `$name` token is a Grafana template variable, not a stored label. - labels = {label for label in labels if not label.startswith("$")} - - baseline_source = BASELINE_PATH.read_text(encoding="utf-8") - labels.update(BASELINE_LABEL_READ.findall(baseline_source)) - labels = { - label - for label in labels - if not label.startswith("$") and label not in PROMETHEUS_RESERVED_LABELS - } - trace_fields = {field for field in ("rootTraceName", "durationMs") if field in baseline_source} - - return ShippedConsumers( - metric_names=frozenset(metric_names), - prometheus_labels=frozenset(labels), - span_name_matchers=tuple(span_names), - span_attributes=frozenset(span_attributes), - resource_attributes=frozenset(resource_attributes), - trace_fields=frozenset(trace_fields), - ) - - -def prometheus_label_for(attribute_key: str) -> str: - """Render an OTLP attribute key the way Prometheus stores it as a label. - - Comparing in this direction avoids inventing an OTLP spelling from a label. - `copilot_chat_edit_source` has two plausible sources, and reconstructing one - of them would be the guess this whole change exists to stop making. - """ - return re.sub(r"[^a-zA-Z0-9_]", "_", attribute_key) - - -# Attributes observed carrying plaintext content on spans even with content -# capture left at its default. None may appear in the allow-list. -OBSERVED_CONTENT_ATTRIBUTES = frozenset( - { - "copilot_chat.user_request", - "copilot_chat.reasoning_content", - "gen_ai.input.messages", - "gen_ai.output.messages", - "gen_ai.system_instructions", - "gen_ai.tool.call.arguments", - "gen_ai.tool.call.result", - } -) - - -class ConfigError(ValueError): - """Raised when a configuration document cannot yield the requested value.""" - - -def load_yaml_text(text: str) -> dict[str, Any]: - """Parse YAML text into a mapping, raising ConfigError on anything else.""" - try: - document = yaml.safe_load(text) - except yaml.YAMLError as exc: - raise ConfigError(f"document is not valid YAML: {exc}") from exc - if not isinstance(document, dict): - raise ConfigError("document root is not a mapping") - return document - - -def load_yaml_file(path: pathlib.Path) -> dict[str, Any]: - """Parse a YAML file into a mapping.""" - if not path.is_file(): - raise ConfigError(f"missing configuration file: {path.name}") - return load_yaml_text(path.read_text(encoding="utf-8")) - - -def published_ports(service: dict[str, Any]) -> list[str]: - """Return the declared host port mappings for a Compose service.""" - return [str(entry) for entry in service.get("ports", []) or []] - - -def redaction_policy(collector: dict[str, Any]) -> dict[str, Any]: - """Return the redaction processor configuration from a Collector document.""" - processors = collector.get("processors") or {} - if not isinstance(processors, dict): - raise ConfigError("processors is not a mapping") - for name, config in processors.items(): - if name == "redaction" or str(name).startswith("redaction/"): - return config or {} - raise ConfigError("no redaction processor is declared") - - -def allowed_keys(collector: dict[str, Any]) -> frozenset[str]: - """Return the allow-listed attribute keys declared by the redaction processor.""" - policy = redaction_policy(collector) - return frozenset(str(key) for key in policy.get("allowed_keys", []) or []) - - -def blocked_values(collector: dict[str, Any]) -> tuple[re.Pattern[str], ...]: - """Return the compiled value patterns the redaction processor masks.""" - policy = redaction_policy(collector) - return tuple(re.compile(str(pattern)) for pattern in policy.get("blocked_values", []) or []) - - -def simulate_redaction( - attributes: dict[str, str], - allow: frozenset[str], - blocked: tuple[re.Pattern[str], ...] = (), - allow_all_keys: bool = False, -) -> dict[str, str]: - """Model the Collector redaction processor's fail-closed key and value handling. - - Any key outside the allow-list is dropped rather than kept, so an attribute - this skill has never seen cannot reach storage. Allowed values that still - match a blocked pattern are masked rather than dropped. - """ - result: dict[str, str] = {} - for key, value in attributes.items(): - if not allow_all_keys and key not in allow: - continue - text = str(value) - result[key] = MASK if any(pattern.search(text) for pattern in blocked) else text - return result @pytest.fixture(scope="module") @@ -448,73 +246,91 @@ def test_the_dashboard_helper_reads_the_same_credential_pair(self) -> None: assert "require_credentials(" in text, "credentials bypass the shared policy" -class TestImagePinning: - """Both images are pinned by immutable digest.""" +class TestContainerBounds: + """The Collector's declared memory bound has to resolve against something. - @pytest.mark.parametrize("service", ["otel-collector", "lgtm"]) - def test_the_image_is_digest_pinned(self, compose: dict[str, Any], service: str) -> None: - assert DIGEST_REFERENCE.match(compose["services"][service]["image"]) + `memory_limiter` uses percentages, and those resolve against the cgroup + limit when one exists and against total host memory when one does not. + Without a declared limit, the configuration's stated bound is 80% of the + developer's machine, in front of an ingress any local process can reach. + """ + + def test_the_collector_declares_a_cgroup_memory_limit(self, compose: dict[str, Any]) -> None: + assert compose["services"]["otel-collector"]["mem_limit"] == "512m" + + def test_the_collector_drops_every_capability(self, compose: dict[str, Any]) -> None: + assert compose["services"]["otel-collector"]["cap_drop"] == ["ALL"] + + def test_the_collector_cannot_gain_privileges(self, compose: dict[str, Any]) -> None: + assert "no-new-privileges:true" in compose["services"]["otel-collector"]["security_opt"] + + def test_the_collector_root_filesystem_is_read_only(self, compose: dict[str, Any]) -> None: + assert compose["services"]["otel-collector"]["read_only"] is True + + +class TestCredentialShapeCoverage: + """The value patterns are masking of recognizable shapes, not secret detection.""" + + @pytest.mark.parametrize( + "value", + [ + "ghp_" + "a" * 36, + "ASIA" + "B" * 16, + "AKIA" + "C" * 16, + "eyJhbGciOiJub25l.eyJzdWIiOiIxMjM0NTY3.", + "AccountKey=" + "d" * 24, + "sk-ant-" + "e" * 24, + "AIza" + "f" * 35, + ], + ids=[ + "github-token", + "aws-temporary-key-id", + "aws-long-term-key-id", + "unsigned-jwt", + "azure-account-key", + "anthropic-api-key", + "google-api-key", + ], + ) + def test_a_recognizable_credential_shape_is_masked( + self, collector: dict[str, Any], value: str + ) -> None: + kept = simulate_redaction( + {"gen_ai.request.model": value}, + allowed_keys(collector), + blocked_values(collector), + ) + assert kept == {"gen_ai.request.model": MASK} + + def test_an_ordinary_value_is_not_masked(self, collector: dict[str, Any]) -> None: + """Without this, a pattern broad enough to mask everything would pass.""" + kept = simulate_redaction( + {"gen_ai.request.model": "gpt-5"}, + allowed_keys(collector), + blocked_values(collector), + ) + assert kept == {"gen_ai.request.model": "gpt-5"} -def scrub_statements(collector: dict[str, Any]) -> list[str]: - """Every OTTL statement the content-scrub processor declares.""" - processor = collector["processors"].get("transform/scrub", {}) - statements: list[str] = [] - for key in ("trace_statements", "log_statements", "metric_statements"): - for entry in processor.get(key) or []: - if isinstance(entry, str): - statements.append(entry) - else: - statements.extend(entry.get("statements", [])) - return statements +class TestScrubFailureDirection: + """The scrub is fail-open per statement, and that is recorded where it lives.""" + def test_the_scrub_error_mode_is_documented_as_deliberate( + self, collector: dict[str, Any] + ) -> None: + assert collector["processors"]["transform/scrub"]["error_mode"] == "ignore" + text = COLLECTOR_PATH.read_text(encoding="utf-8") + assert "fail-open per statement" in text, ( + "the asymmetry with the fail-closed allow-list is not recorded" + ) -def statement_target(statement: str) -> str: - """The path an OTTL statement writes to. - Only the write target matters. A `where` clause may read `span.name` - without endangering it, and treating a read as an overreach would push the - scrub rules into contortions to avoid mentioning what they must not touch. - """ - if "(" not in statement: - return "" - inner = statement.split("(", 1)[1] - depth = 0 - for index, character in enumerate(inner): - if character in "([": - depth += 1 - elif character in ")]": - if depth == 0: - return inner[:index].strip() - depth -= 1 - elif character == "," and depth == 0: - return inner[:index].strip() - return inner.strip() - - -# Intrinsics a shipped consumer reads directly. `span.name` is here because -# three TraceQL queries match on it; the metric intrinsics are here because -# every Prometheus query selects by metric name. -PROTECTED_INTRINSICS = frozenset({"span.name", "metric.name", "metric.description", "metric.unit"}) - -ATTRIBUTE_TARGET = re.compile(r"attributes\[\"([^\"]+)\"\]") - - -def overreaching_statements( - statements: list[str], consumers: ShippedConsumers -) -> list[tuple[str, str]]: - """Statements whose write target is something a shipped consumer reads.""" - protected_keys = consumers.protected_attribute_keys() - findings: list[tuple[str, str]] = [] - for statement in statements: - target = statement_target(statement) - if target in PROTECTED_INTRINSICS: - findings.append((statement, target)) - continue - for key in ATTRIBUTE_TARGET.findall(target): - if key in protected_keys: - findings.append((statement, key)) - return findings +class TestImagePinning: + """Both images are pinned by immutable digest.""" + + @pytest.mark.parametrize("service", ["otel-collector", "lgtm"]) + def test_the_image_is_digest_pinned(self, compose: dict[str, Any], service: str) -> None: + assert DIGEST_REFERENCE.match(compose["services"][service]["image"]) class TestScrubStaysInsideTheInventory: diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py index a5b3e72ed..4a16bcef5 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py @@ -13,6 +13,10 @@ import pytest from settings_upsert import ( + BACKUP_RETENTION, + EXIT_INTERRUPTED, + EXIT_OK, + EXIT_REFUSED, SCHEMA, SCHEMA_SOURCE, SettingsError, @@ -21,6 +25,9 @@ check_policy, main, parse_assignment, + prune_backups, + read_settings, + resolve_audit_path, strip_jsonc, summarize, top_level_spans, @@ -120,18 +127,85 @@ def test_an_unsafe_endpoint_is_refused(self, endpoint: str) -> None: def test_the_local_collector_endpoint_is_allowed(self) -> None: check_policy({ENDPOINT: "http://localhost:4318"}) - def test_outfile_with_an_otlp_exporter_is_refused(self) -> None: + def test_outfile_with_an_otlp_exporter_is_refused(self, tmp_path: pathlib.Path) -> None: + outfile = str(tmp_path / "spans.jsonl") with pytest.raises(SettingsError, match="outfile"): - check_policy({OUTFILE: "/tmp/spans.jsonl", EXPORTER: "otlp-http"}) + check_policy({OUTFILE: outfile, EXPORTER: "otlp-http"}) - def test_outfile_with_the_file_exporter_is_allowed(self) -> None: - check_policy({OUTFILE: "/tmp/spans.jsonl", EXPORTER: "file"}) + def test_outfile_with_the_file_exporter_is_allowed(self, tmp_path: pathlib.Path) -> None: + check_policy({OUTFILE: str(tmp_path / "spans.jsonl"), EXPORTER: "file"}) def test_a_negative_truncation_limit_is_refused(self) -> None: with pytest.raises(SettingsError, match="negative"): check_policy({MAXSIZE: -1}) +class TestMergedStatePolicy: + """Cross-setting rules are judged on the file that would result. + + Checking only this invocation's arguments let a contradiction be assembled + across two runs: set `outfile` once, set `exporterType` later, and each run + passes while the resulting file says both. + """ + + def test_an_exporter_that_contradicts_an_existing_outfile_is_refused( + self, tmp_path: pathlib.Path + ) -> None: + existing = {OUTFILE: str(tmp_path / "spans.jsonl")} + with pytest.raises(SettingsError, match="outfile"): + check_policy({EXPORTER: "otlp-http"}, existing=existing) + + def test_an_outfile_that_contradicts_an_existing_exporter_is_refused( + self, tmp_path: pathlib.Path + ) -> None: + with pytest.raises(SettingsError, match="outfile"): + check_policy({OUTFILE: str(tmp_path / "spans.jsonl")}, existing={EXPORTER: "otlp-http"}) + + def test_replacing_the_conflicting_setting_in_the_same_run_is_allowed( + self, tmp_path: pathlib.Path + ) -> None: + check_policy( + {OUTFILE: str(tmp_path / "spans.jsonl"), EXPORTER: "file"}, + existing={EXPORTER: "otlp-http"}, + ) + + def test_an_existing_unknown_exporter_is_reported(self) -> None: + with pytest.raises(SettingsError, match="exporterType"): + check_policy({ENABLED: True}, existing={EXPORTER: "otlp-quic"}) + + +class TestOutfilePolicy: + """Captured spans may not land somewhere nobody chose.""" + + def test_a_relative_outfile_is_refused(self) -> None: + with pytest.raises(SettingsError, match="absolute"): + check_policy({OUTFILE: "spans.jsonl", EXPORTER: "file"}) + + def test_an_outfile_inside_this_repository_is_refused(self) -> None: + inside = str(pathlib.Path(__file__).resolve().parent / "spans.jsonl") + with pytest.raises(SettingsError, match="inside this repository"): + check_policy({OUTFILE: inside, EXPORTER: "file"}) + + def test_a_foreign_platform_absolute_path_is_accepted(self) -> None: + """A settings file is routinely authored on one platform for another.""" + check_policy({OUTFILE: "/var/log/copilot/spans.jsonl", EXPORTER: "file"}) + + +class TestRemoteEndpointOptIn: + """A remote endpoint is a deliberate choice, not a default.""" + + def test_a_remote_endpoint_is_refused_by_default(self) -> None: + with pytest.raises(SettingsError, match="otlpEndpoint"): + check_policy({ENDPOINT: "https://93.184.216.34:4318"}) + + def test_a_remote_endpoint_is_allowed_with_the_opt_in(self) -> None: + check_policy({ENDPOINT: "https://93.184.216.34:4318"}, allow_remote_endpoint=True) + + def test_the_opt_in_does_not_permit_plaintext(self) -> None: + with pytest.raises(SettingsError, match="otlpEndpoint"): + check_policy({ENDPOINT: "http://93.184.216.34:4318"}, allow_remote_endpoint=True) + + class TestJsoncHandling: """Comments survive, offsets line up, and only the target value moves.""" @@ -198,14 +272,70 @@ def test_a_duplicate_key_is_refused(self, settings_file: pathlib.Path) -> None: "--apply", ] ) - assert code == 1 + assert code == EXIT_REFUSED assert settings_file.read_text(encoding="utf-8") == EXISTING def test_no_assignment_is_refused(self, settings_file: pathlib.Path) -> None: - assert main(["--settings", str(settings_file)]) == 1 + assert main(["--settings", str(settings_file)]) == EXIT_REFUSED assert settings_file.read_text(encoding="utf-8") == EXISTING +class TestExitConventions: + """The exit code distinguishes a refusal from a crash and from an interrupt.""" + + def test_a_successful_dry_run_exits_zero(self, settings_file: pathlib.Path) -> None: + assert main(["--settings", str(settings_file), "--set", f"{ENABLED}=true"]) == EXIT_OK + + def test_a_refusal_exits_two(self, settings_file: pathlib.Path) -> None: + code = main(["--settings", str(settings_file), "--set", f"{CAPTURE}=true", "--apply"]) + assert code == EXIT_REFUSED + + def test_an_interrupt_exits_one_hundred_thirty( + self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + def interrupt(*args: object, **kwargs: object) -> str: + raise KeyboardInterrupt + + monkeypatch.setattr("settings_upsert.apply_changes", interrupt) + code = main(["--settings", str(settings_file), "--set", f"{ENABLED}=true"]) + assert code == EXIT_INTERRUPTED + + def test_the_entry_point_passes_the_code_to_sys_exit(self) -> None: + """`raise SystemExit(main())` and `sys.exit(main())` differ to a reader.""" + source = ( + pathlib.Path(__file__).resolve().parents[1] / "examples" / "settings_upsert.py" + ).read_text(encoding="utf-8") + assert "sys.exit(main())" in source + + def test_diagnostics_use_a_module_logger(self) -> None: + source = ( + pathlib.Path(__file__).resolve().parents[1] / "examples" / "settings_upsert.py" + ).read_text(encoding="utf-8") + assert 'LOGGER = logging.getLogger("settings_upsert")' in source + assert "def configure_logging(" in source + + def test_a_closed_reader_is_not_an_error( + self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A quit pager is the reader leaving, not the edit failing. + + The real redirect replaces this process's stdout descriptor, which + would take the test runner's capture with it, so the handler's effect + is recorded rather than performed. + """ + silenced: list[str] = [] + + def broken(*args: object, **kwargs: object) -> None: + raise BrokenPipeError + + monkeypatch.setattr("builtins.print", broken) + monkeypatch.setattr( + "settings_upsert.silence_broken_pipe", lambda: silenced.append("silenced") + ) + assert main(["--settings", str(settings_file), "--set", f"{ENABLED}=true"]) == EXIT_OK + assert silenced == ["silenced"] + + class TestApply: """Dry runs write nothing; applied edits are backed up and reversible.""" @@ -269,7 +399,7 @@ def test_a_sensitive_value_is_summarized_not_copied(self) -> None: def test_an_ordinary_value_is_recorded_directly(self) -> None: assert summarize(ENABLED, True) is True - def test_an_endpoint_keeps_its_authority_and_loses_its_path(self) -> None: + def test_an_endpoint_keeps_its_host_and_loses_everything_else(self) -> None: """A path or query on an OTLP endpoint is operator-supplied. It is where a tenant identifier or a token would sit, and it is not @@ -280,6 +410,13 @@ def test_an_endpoint_keeps_its_authority_and_loses_its_path(self) -> None: assert "token" not in str(summarized) assert "v1/traces" not in str(summarized) + def test_endpoint_userinfo_is_not_copied_into_the_record(self) -> None: + """The netloc carries the credential; the host and port do not.""" + summarized = summarize(ENDPOINT, "https://someone:s3cret@ingest.example.com:4318/v1") + assert summarized == "https://ingest.example.com:4318" + assert "s3cret" not in str(summarized) + assert "someone" not in str(summarized) + def test_a_loopback_endpoint_is_still_recognizable(self) -> None: assert summarize(ENDPOINT, "http://localhost:4318/v1/traces") == "http://localhost:4318" @@ -291,6 +428,136 @@ def test_an_absent_value_records_as_absent(self) -> None: assert summarize(ENDPOINT, None) is None +class TestDocumentShapes: + """A settings file a person can reasonably have is editable. + + Refusing an empty or comments-only document meant the tool could not make + the first edit to a settings file that had never held a setting, which is + exactly when it is most useful. + """ + + @pytest.mark.parametrize("content", ["", " \n\n", "// nothing set yet\n"]) + def test_an_effectively_empty_document_is_editable( + self, tmp_path: pathlib.Path, content: str + ) -> None: + target = tmp_path / "settings.json" + target.write_text(content, encoding="utf-8") + apply_changes(target, {ENABLED: True}, apply=True) + assert json.loads(strip_jsonc(target.read_text(encoding="utf-8-sig")))[ENABLED] is True + + def test_a_comments_only_document_keeps_its_comments(self, tmp_path: pathlib.Path) -> None: + target = tmp_path / "settings.json" + target.write_text("// a note the operator left\n", encoding="utf-8") + apply_changes(target, {ENABLED: True}, apply=True) + assert "a note the operator left" in target.read_text(encoding="utf-8") + + def test_a_byte_order_mark_is_preserved(self, tmp_path: pathlib.Path) -> None: + """VS Code wrote it; removing it is an unrelated change to the file.""" + target = tmp_path / "settings.json" + target.write_bytes(b"\xef\xbb\xbf" + EXISTING.encode("utf-8")) + apply_changes(target, {ENABLED: True}, apply=True) + assert target.read_bytes().startswith(b"\xef\xbb\xbf") + assert json.loads(strip_jsonc(target.read_text(encoding="utf-8-sig")))[ENABLED] is True + + def test_a_document_without_a_mark_does_not_gain_one(self, settings_file: pathlib.Path) -> None: + apply_changes(settings_file, {ENABLED: True}, apply=True) + assert not settings_file.read_bytes().startswith(b"\xef\xbb\xbf") + + def test_read_settings_reports_the_mark_separately(self, tmp_path: pathlib.Path) -> None: + target = tmp_path / "settings.json" + target.write_bytes(b"\xef\xbb\xbf{}\n") + bom, text = read_settings(target) + assert bom == "\ufeff" + assert not text.startswith("\ufeff") + + +class TestAtomicWrite: + """An interrupted write may not leave a truncated settings file. + + Writing in place truncates before it writes, so a failure at the wrong + moment destroys the document. Staging beside the target and replacing means + the file is either the old one or the new one. + """ + + def test_a_failed_write_leaves_the_original_intact( + self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + def fail(*args: object, **kwargs: object) -> None: + raise OSError("disk full") + + monkeypatch.setattr("settings_upsert.os.replace", fail) + with pytest.raises(OSError, match="disk full"): + apply_changes(settings_file, {ENABLED: True}, apply=True) + assert settings_file.read_text(encoding="utf-8") == EXISTING + + def test_a_failed_write_leaves_no_staged_file_behind( + self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + def fail(*args: object, **kwargs: object) -> None: + raise OSError("disk full") + + monkeypatch.setattr("settings_upsert.os.replace", fail) + with pytest.raises(OSError): + apply_changes(settings_file, {ENABLED: True}, apply=True) + assert list(settings_file.parent.glob("*.staged")) == [] + + def test_a_new_settings_file_is_not_created_by_a_failed_write( + self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + target = tmp_path / "settings.json" + + def fail(*args: object, **kwargs: object) -> None: + raise OSError("disk full") + + monkeypatch.setattr("settings_upsert.os.replace", fail) + with pytest.raises(OSError): + apply_changes(target, {ENABLED: True}, apply=True) + assert not target.exists() + + +class TestAuditPathAliases: + """An audit path may not be an alias for the file being protected. + + Containing the path to the settings directory was worse than useless: it + silently moved the operator's chosen path into the one directory where an + appended JSON line destroys the settings file or its backup. + """ + + def test_the_settings_file_itself_is_refused(self, settings_file: pathlib.Path) -> None: + with pytest.raises(SettingsError, match="settings file itself"): + resolve_audit_path(str(settings_file), settings_file) + + def test_a_backup_beside_the_settings_file_is_refused( + self, settings_file: pathlib.Path + ) -> None: + backup = backup_path_for(settings_file) + with pytest.raises(SettingsError, match="settings backup"): + resolve_audit_path(str(backup), settings_file) + + def test_a_directory_is_refused( + self, settings_file: pathlib.Path, tmp_path: pathlib.Path + ) -> None: + with pytest.raises(SettingsError, match="directory"): + resolve_audit_path(str(tmp_path), settings_file) + + def test_an_operator_chosen_path_elsewhere_is_kept( + self, settings_file: pathlib.Path, tmp_path: pathlib.Path + ) -> None: + chosen = tmp_path.parent / "audit-elsewhere.jsonl" + assert resolve_audit_path(str(chosen), settings_file) == chosen.resolve() + + def test_a_relative_path_resolves_against_the_caller( + self, settings_file: pathlib.Path, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + elsewhere = tmp_path / "cwd" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + assert ( + resolve_audit_path("audit.jsonl", settings_file) + == (elsewhere / "audit.jsonl").resolve() + ) + + class TestBackupRetention: """A backup that overwrites the previous backup is not a backup. @@ -321,12 +588,57 @@ def test_a_same_second_collision_gets_its_own_name(self, tmp_path: pathlib.Path) assert second != first assert not second.exists() + def test_a_later_backup_sorts_after_an_earlier_one(self, tmp_path: pathlib.Path) -> None: + """Retention reads name order to decide what to delete. + + An optional collision suffix put `...Z-1.bak` before `...Z.bak`, which + would have made pruning remove the newest backups and keep the oldest. + """ + settings = tmp_path / "settings.json" + settings.write_text("{}\n", encoding="utf-8") + names = [] + for _ in range(3): + candidate = backup_path_for(settings) + candidate.write_text("taken", encoding="utf-8") + names.append(candidate.name) + assert names == sorted(names) + def test_the_backup_sits_beside_the_settings_file(self, tmp_path: pathlib.Path) -> None: settings = tmp_path / "settings.json" assert backup_path_for(settings).parent == tmp_path assert backup_path_for(settings).name.startswith("settings.json.") assert backup_path_for(settings).name.endswith(".bak") + def test_only_the_newest_backups_are_kept(self, settings_file: pathlib.Path) -> None: + """Each backup is a full copy of a file naming an endpoint and an output path. + + Unbounded retention left a growing pile of them beside the settings + file, which is a disclosure surface that grows with ordinary use. + """ + for index in range(BACKUP_RETENTION + 3): + apply_changes(settings_file, {MAXSIZE: index + 1}, apply=True) + backups = sorted(settings_file.parent.glob(f"{settings_file.name}.*.bak")) + assert len(backups) == BACKUP_RETENTION + + def test_pruning_reports_what_it_removed(self, tmp_path: pathlib.Path) -> None: + settings = tmp_path / "settings.json" + settings.write_text("{}\n", encoding="utf-8") + created = [] + for index in range(BACKUP_RETENTION + 2): + backup = settings.with_name(f"{settings.name}.2026010{index}T000000Z-000.bak") + backup.write_text("{}\n", encoding="utf-8") + created.append(backup) + removed = prune_backups(settings) + assert removed == created[:2] + assert all(not path.exists() for path in removed) + + def test_pruning_keeps_the_newest_backup(self, settings_file: pathlib.Path) -> None: + for index in range(BACKUP_RETENTION + 2): + apply_changes(settings_file, {MAXSIZE: index + 1}, apply=True) + backups = sorted(settings_file.parent.glob(f"{settings_file.name}.*.bak")) + newest = json.loads(strip_jsonc(backups[-1].read_text(encoding="utf-8-sig"))) + assert newest[MAXSIZE] == BACKUP_RETENTION + 1 + class TestTrailingCommas: """A trailing comma is accepted; a comma inside a string is untouchable. From 9688c1a3e87ff96adb853dd7971531cdaf6a4cb2 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Tue, 18 Aug 2026 21:29:04 -0700 Subject: [PATCH 12/28] ci(workflows): add a strict runtime pytest lane for copilot-otel-metrics Parameterize the reusable pytest workflow with a changed-paths pattern, a marker expression, a strict-runtime toggle, an artifact suffix, and an optional Compose service to pre-pull, so a skill can select a container-backed subset without a second workflow. Pre-pull the digest-pinned Collector image with bounded retries and assert the pin before the tests run, then select the slow marker in a dedicated pull-request job so runtime TLS and authentication evidence is produced in CI rather than only on a developer machine. --- .github/workflows/pr-validation.yml | 17 ++++++ .github/workflows/pytest-tests.yml | 88 ++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index b012e84e3..55f19fcd7 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -196,6 +196,23 @@ jobs: matrix: directory: ${{ fromJson(needs.discover-python-projects.outputs.directories) }} + copilot-otel-runtime-tests: + name: Copilot OTEL Runtime Tests + uses: ./.github/workflows/pytest-tests.yml + permissions: + contents: read + id-token: write + with: + working-directory: .github/skills/experimental/copilot-otel-metrics + soft-fail: false + changed-files-only: true + changed-paths-pattern: '^\.github/skills/experimental/copilot-otel-metrics/(examples/.*\.(py|yaml)|examples/azure/agent-host-relay/.*\.yaml|tests/.*\.py|pyproject\.toml)$|^\.github/workflows/(pytest-tests|pr-validation)\.yml$' + marker-expression: slow + strict-runtime: true + artifact-suffix: -runtime + pre-pull-compose-file: examples/compose.yaml + pre-pull-compose-service: otel-collector + node-tests: name: "Node Tests (${{ matrix.directory }})" needs: discover-node-projects diff --git a/.github/workflows/pytest-tests.yml b/.github/workflows/pytest-tests.yml index b36cbb35f..34e05303d 100644 --- a/.github/workflows/pytest-tests.yml +++ b/.github/workflows/pytest-tests.yml @@ -19,6 +19,36 @@ on: required: false type: boolean default: false + changed-paths-pattern: + description: 'Extended regex of changed paths that trigger this lane; overrides the default Python-file check' + required: false + type: string + default: '' + marker-expression: + description: 'Pytest -m expression that overrides the project default marker selection' + required: false + type: string + default: '' + strict-runtime: + description: 'Require runtime-backed tests to run instead of skipping when a runtime is unavailable' + required: false + type: boolean + default: false + artifact-suffix: + description: 'Suffix that keeps this lane coverage artifact and Codecov name distinct' + required: false + type: string + default: '' + pre-pull-compose-file: + description: 'Compose document, relative to working-directory, holding the digest-pinned runtime image' + required: false + type: string + default: '' + pre-pull-compose-service: + description: 'Compose service whose digest-pinned image is pulled before pytest' + required: false + type: string + default: '' permissions: contents: read @@ -42,9 +72,10 @@ jobs: shell: bash env: INPUT_WORKING_DIRECTORY: ${{ inputs.working-directory }} + INPUT_ARTIFACT_SUFFIX: ${{ inputs.artifact-suffix }} run: | set -euo pipefail - echo "name=pytest-results-$(echo "$INPUT_WORKING_DIRECTORY" | tr '/' '-')" >> "$GITHUB_OUTPUT" + echo "name=pytest-results-$(echo "$INPUT_WORKING_DIRECTORY" | tr '/' '-')${INPUT_ARTIFACT_SUFFIX}" >> "$GITHUB_OUTPUT" - name: Check for Python files id: check-python @@ -52,16 +83,23 @@ jobs: shell: bash env: INPUT_WORKING_DIRECTORY: ${{ inputs.working-directory }} + INPUT_CHANGED_PATHS_PATTERN: ${{ inputs.changed-paths-pattern }} BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} run: | set -euo pipefail - # Check if any Python files changed in the working directory - if git diff --name-only "$BASE_SHA" HEAD | grep -q "^${INPUT_WORKING_DIRECTORY}/.*\.py$"; then + # A lane that owns non-Python inputs supplies its own pattern; the + # default keeps the generic Python-file behaviour for every caller. + if [ -n "$INPUT_CHANGED_PATHS_PATTERN" ]; then + pattern="$INPUT_CHANGED_PATHS_PATTERN" + else + pattern="^${INPUT_WORKING_DIRECTORY}/.*\.py$" + fi + if git diff --name-only "$BASE_SHA" HEAD | grep -Eq "$pattern"; then echo "has_changes=true" >> "$GITHUB_OUTPUT" - echo "Python files changed in $INPUT_WORKING_DIRECTORY" + echo "Tracked files changed for pattern $pattern" else echo "has_changes=false" >> "$GITHUB_OUTPUT" - echo "No Python files changed in $INPUT_WORKING_DIRECTORY" + echo "No tracked files changed for pattern $pattern" fi - name: Install uv @@ -105,15 +143,51 @@ jobs: SEARCH_ROOT: ${{ inputs.working-directory }} run: ./scripts/ci/install-skill-node-deps.sh + - name: Pre-pull the digest-pinned runtime image + if: inputs.pre-pull-compose-service != '' && (!inputs.changed-files-only || steps.check-python.outputs.has_changes == 'true') + working-directory: ${{ inputs.working-directory }} + shell: bash + timeout-minutes: 10 + env: + COMPOSE_FILE_PATH: ${{ inputs.pre-pull-compose-file }} + COMPOSE_SERVICE: ${{ inputs.pre-pull-compose-service }} + run: | + set -euo pipefail + # Pulling here keeps registry unavailability distinguishable from a + # test failure, and reads the pin from the shipped Compose document + # so this lane can never exercise an image the stack no longer uses. + image="$(uv run python -c 'import os, yaml; document = yaml.safe_load(open(os.environ["COMPOSE_FILE_PATH"], encoding="utf-8")); print(document["services"][os.environ["COMPOSE_SERVICE"]]["image"])')" + case "$image" in + *@sha256:*) ;; + *) echo "::error::runtime image '$image' is not digest-pinned"; exit 1 ;; + esac + for attempt in 1 2 3; do + if docker pull "$image"; then + echo "Pulled $image on attempt $attempt" + exit 0 + fi + echo "::warning::docker pull attempt $attempt failed for $image" + sleep $((attempt * 10)) + done + echo "::error::exhausted 3 pull attempts for $image" + exit 1 + - name: Run pytest with coverage if: "!inputs.changed-files-only || steps.check-python.outputs.has_changes == 'true'" working-directory: ${{ inputs.working-directory }} shell: bash + env: + INPUT_MARKER_EXPRESSION: ${{ inputs.marker-expression }} + COPILOT_OTEL_STRICT_RUNTIME: ${{ inputs.strict-runtime && '1' || '0' }} run: | set -euo pipefail if [ -f "pyproject.toml" ]; then echo "Running pytest with coverage" - uv run pytest --cov --cov-report=xml:coverage.xml --cov-report=term-missing -v + if [ -n "$INPUT_MARKER_EXPRESSION" ]; then + uv run pytest -m "$INPUT_MARKER_EXPRESSION" --cov --cov-report=xml:coverage.xml --cov-report=term-missing -v + else + uv run pytest --cov --cov-report=xml:coverage.xml --cov-report=term-missing -v + fi else echo "No pyproject.toml found - skipping tests" fi @@ -137,5 +211,5 @@ jobs: fail_ci_if_error: false verbose: true flags: pytest - name: pytest-coverage-${{ inputs.working-directory }} + name: pytest-coverage-${{ inputs.working-directory }}${{ inputs.artifact-suffix }} continue-on-error: true From 591d7665076ef967b1da87c59e185c2a4e602f83 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Tue, 18 Aug 2026 21:30:53 -0700 Subject: [PATCH 13/28] docs(skills): reconcile the security model, transparency note, and skill docs Rewrite the skill security model so the trust-boundary diagram, adversary entries, image inventory, and executive summary describe the shipped relay, double filtering, atomic settings write, and the availability cost of a relay outage, and record the new denial-of-service and infrastructure gaps. Correct the documented artifact inventory, helper counts, hook event list, and transparency note now that the telemetry hook is retired and the skill is opt-in, and reconcile the repository security model with the skill it describes. --- .../copilot-otel-metrics/SECURITY.md | 129 +++++++++--------- .../copilot-otel-metrics/SKILL.md | 7 +- TRANSPARENCY-NOTE.md | 4 +- docs/contributing/hooks.md | 4 +- docs/customization/README.md | 33 ++--- docs/customization/copilot-otel-metrics.md | 13 +- docs/security/security-model.md | 32 ++--- 7 files changed, 114 insertions(+), 108 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/SECURITY.md b/.github/skills/experimental/copilot-otel-metrics/SECURITY.md index 2eb9aecbd..1ebdfd332 100644 --- a/.github/skills/experimental/copilot-otel-metrics/SECURITY.md +++ b/.github/skills/experimental/copilot-otel-metrics/SECURITY.md @@ -2,7 +2,7 @@ title: Copilot OTel Metrics Skill Security Model description: STRIDE threat model for the copilot-otel-metrics skill organized by assets, adversaries, and trust buckets (editor OTLP ingest, telemetry at rest, reference helper scripts, container image supply chain, editor-global configuration mutation, host process control, cloud control-plane artifact generation) with in-design mitigations and acknowledged enterprise readiness gaps author: microsoft/hve-core -ms.date: 2026-08-14 +ms.date: 2026-08-18 ms.topic: reference estimated_reading_time: 18 keywords: @@ -15,7 +15,7 @@ keywords: # Copilot OTel Metrics Skill Security Model -This document records the STRIDE threat model for the copilot-otel-metrics skill. The shipped runtime is `examples/compose.yaml` (the local stack definition), `examples/otel-collector-local.yaml` (the local filtering pipeline), two Grafana dashboards under `examples/dashboards/`, five reference helper scripts and one shared policy module under `examples/`, and the Azure templates under `examples/azure/` (collector configuration, Bicep, Terraform, and an Azure CLI script). The model is organized by trust bucket: editor OTLP ingest (B1), telemetry at rest and its query surfaces (B2), reference helper scripts to local service APIs (B3), container image supply chain (B4), editor-global configuration mutation (B5), host process control (B6), and cloud control-plane artifact generation (B7). Each bucket enumerates all six STRIDE categories. Assets and adversaries are enumerated first. Acknowledged enterprise readiness gaps are listed at the end. +This document records the STRIDE threat model for the copilot-otel-metrics skill. The shipped runtime is `examples/compose.yaml` (the local stack definition), `examples/otel-collector-local.yaml` (the local filtering pipeline), two Grafana dashboards under `examples/dashboards/`, five reference helper scripts and one shared policy module under `examples/`, and the Azure templates under `examples/azure/` (fleet collector configuration, the per-workstation relay under `agent-host-relay/`, Bicep, Terraform, and an Azure CLI script). The model is organized by trust bucket: editor OTLP ingest (B1), telemetry at rest and its query surfaces (B2), reference helper scripts to local service APIs (B3), container image supply chain (B4), editor-global configuration mutation (B5), host process control (B6), and cloud control-plane artifact generation (B7). Each bucket enumerates all six STRIDE categories. Assets and adversaries are enumerated first. Acknowledged enterprise readiness gaps are listed at the end. The skill is an assistant rather than a reference pack, and that changes the model materially. It may write the user's global `settings.json` after presenting a diff and obtaining explicit approval, and it generates files intended for the user to execute. It never starts a service and never provisions infrastructure: `docker compose`, `az deployment`, and `terraform apply` are handed to the user, not run. Buckets B5 through B7 exist because generating and writing are themselves exposures, independent of who runs the result. @@ -25,7 +25,7 @@ The skill is an assistant rather than a reference pack, and that changes the mod The copilot-otel-metrics skill helps an operator turn on GitHub Copilot Chat's OTLP export and stand up somewhere for the data to land, locally or in Azure. Its highest-risk property is **not the code, it is the payload**. Spans emitted by the extension were directly observed carrying full prompt text, tool call arguments and results, and system instructions, on a configuration where content capture was left at its default. Anyone who follows this skill therefore accumulates a durable corpus of prompt content, in a Docker volume locally or in a billed Log Analytics workspace for an organization. -That exposure is **not harmless and is not accepted silently**. Both capture paths filter before storage. The local path routes every export through an OpenTelemetry Collector whose attribute policy is a fail-closed allow-list, so an attribute this skill has never seen is dropped rather than stored; the organization path deletes the seven observed content attributes before they reach billable storage. The direction matters: a delete-list removes the content attributes someone already knows about and passes through the one a future extension release adds. What filtering cannot change is what the extension emits, so content still leaves the editor and still crosses the loopback hop in plaintext. The skill states the same position everywhere it appears: the endpoint is sensitive regardless of the `captureContent` setting, and the user owns the decision. +That exposure is **not harmless and is not accepted silently**. Both capture paths filter before storage, and they filter the same way. The local path routes every export through an OpenTelemetry Collector whose attribute policy is a fail-closed allow-list, so an attribute this skill has never seen is dropped rather than stored. The fleet path applies that same allow-list and the same content scrub to traces, metrics, and logs. The direction matters: the delete-list this configuration previously used removed the content attributes someone already knew about and passed through the one a future extension release adds, and it did not run on the metrics pipeline at all. Under the organization topology telemetry is filtered twice, once by the workstation relay before it leaves the machine and again by the fleet Collector before shared storage. What filtering cannot change is what the extension emits, so content still leaves the editor and still crosses the loopback hop in plaintext. The skill states the same position everywhere it appears: the endpoint is sensitive regardless of the `captureContent` setting, and the user owns the decision. **How far the local filtering reaches is measured, not asserted.** `tests/test_collector_carriers.py` starts the pinned Collector with the shipped configuration, sends payloads placing a distinct marker in each of 28 OTLP carriers, and records what survives. Every claim about local minimization in this document is that module's output. Its method matters as much as its result: a paired run with the content processors removed establishes that the instrument would have rendered each marker had it survived, so a carrier is never called governed merely because its marker was absent. The map is asserted, so a configuration or image change that opens a carrier fails the suite. @@ -33,7 +33,7 @@ That measurement corrected this document. The allow-list governs attributes at e The design bounds the local case by construction: a filtering Collector is the only OTLP listener published on the host, its attribute policy is a fail-closed allow-list rather than a delete-list, every published port binds to `127.0.0.1`, both images are pinned by manifest digest, Grafana requires an operator-supplied credential with anonymous access disabled, the documented settings block omits `captureContent`, and the skill supplies a verification command so a reader checks rather than assumes. Residual risk concentrates in four places the skill cannot close: the extension's own span content behavior before the Collector sees it, the carriers the Collector governs incompletely or not at all, the absence of authentication on the Prometheus and Tempo query APIs, and the lack of encryption on the persistent volume. -The assistant behaviors add three bounded exposures. The settings write mutates a user-owned JSONC file, which is contained by an executable that enforces a verified schema, splices only the target value span, backs up before writing without overwriting an earlier backup, restores on a parse failure, and refuses to write when unrelated settings would change. Generated compose and IaC files are inert until the user runs them, and the agent is prohibited from running them. Organization capture places a shared write-side credential on every workstation with no per-user binding and no documented in-place rotation, which is the single largest new exposure and is the reason the Azure path leads with that fact rather than with the architecture. +The assistant behaviors add three bounded exposures. The settings write mutates a user-owned JSONC file, which is contained by an executable that enforces a verified schema, splices only the target value span, backs up before writing without overwriting an earlier backup, stages the result and swaps it in atomically so a failed write leaves the original rather than a truncated file, and refuses to write when unrelated settings would change. Generated compose and IaC files are inert until the user runs them, and the agent is prohibited from running them. Organization capture places a shared write-side credential on every workstation with no per-user binding and no documented in-place rotation, which is the single largest new exposure and is the reason the Azure path leads with that fact rather than with the architecture. Holding that credential in the relay's own runtime environment keeps it out of the editor's environment and out of every agent-spawned subprocess, at the cost of making relay health a prerequisite for all of that workstation's telemetry. ### Security Posture Overview @@ -44,7 +44,7 @@ The assistant behaviors add three bounded exposures. The settings write mutates | Credentials | Grafana admin credentials supplied by the operator through required environment variables, with no default and no committed value; an Application Insights connection string and a fleet ingest token for the Azure path, both operator-supplied and never written by the skill | | Network egress | None locally after the image pull. The Azure path sends telemetry to a collector the operator runs, which forwards to Application Insights over TLS | | Agent execution | Writes the user's global `settings.json` after an approved diff; writes generated artifacts to disk. Never starts a service and never provisions infrastructure | -| Open residual gaps | 25 registered, 10 open (highest: InfoDisc-High, prompt content present in spans before the Collector filters it) | +| Open residual gaps | 27 registered, 13 open (highest: InfoDisc-Med, the carriers the filter does not reach and the shared fleet ingest credential). G-INF-1 is the highest-severity entry overall and is partially mitigated rather than open | ## Contents @@ -66,7 +66,7 @@ The assistant behaviors add three bounded exposures. The settings write mutates ### Components -1. `examples/compose.yaml` β€” declares a Collector container publishing the only loopback-bound OTLP ports and one `grafana/otel-lgtm` container with no host OTLP mapping, joined by an explicit bridge network, mounting an external named volume at `/data`, requiring operator-supplied Grafana credentials with anonymous access disabled, and passing delta-to-cumulative conversion plus 120-day retention to Prometheus. Both images are digest-pinned. +1. `examples/compose.yaml` β€” declares a Collector container publishing the only loopback-bound OTLP ports and one `grafana/otel-lgtm` container with no host OTLP mapping, joined by an explicit bridge network, mounting an external named volume at `/data`, requiring operator-supplied Grafana credentials with anonymous access disabled, and passing delta-to-cumulative conversion plus 120-day retention to Prometheus. Both images are digest-pinned, and the Collector service declares a memory limit, a read-only root filesystem, all capabilities dropped, and no-new-privileges. 2. `examples/otel-collector-local.yaml` β€” the local Collector pipeline. Applies a fail-closed `redaction` allow-list across the trace, metric, and log pipelines, masks recognizable credential shapes in surviving values, scrubs the content carriers the allow-list cannot reach, and exports only to the LGTM service over the internal network. What each of its controls actually reaches is recorded by `tests/test_collector_carriers.py` rather than inferred from the configuration. 3. `examples/dashboards/copilot-otel.json` β€” Grafana dashboard referencing the pre-provisioned `prometheus` and `tempo` datasource uids. Contains PromQL and TraceQL queries only. 4. `examples/_input_policy.py` β€” shared, not run directly. Centralizes endpoint scheme, authority, port, remote opt-in, redirect revalidation, path containment, and required-credential checks for the helpers that take configurable input. @@ -74,12 +74,13 @@ The assistant behaviors add three bounded exposures. The settings write mutates 6. `examples/baseline.py` β€” read-mostly. Snapshots Prometheus label values and Tempo trace names, writes one JSON file contained to the user cache root, and diffs a later store state against it. 7. `examples/inspect_metrics.py` β€” read-only. Enumerates `copilot_chat` and `gen_ai` series with labels and current values. Takes no configurable endpoint. 8. `examples/validate_dashboard.py` β€” imports a dashboard through the Grafana API with `overwrite: true`, then replays each panel query against Prometheus or Tempo. Every configurable endpoint and the dashboard path pass the shared policy first, and Grafana credentials are required with no default. -9. `examples/settings_upsert.py` β€” the settings mutation executable. Enforces a schema verified against a named extension build, splices only target value spans, backs up, restores on failure, and appends a redacted audit record. +9. `examples/settings_upsert.py` β€” the settings mutation executable. Enforces a schema verified against a named extension build, splices only target value spans, backs up while retaining the five newest backups, stages the result and replaces the target atomically so a failed write leaves the original intact, and appends a redacted audit record whose endpoint summary carries no userinfo. 10. `examples/dashboards/copilot-otel-azure.json` β€” Grafana dashboard for the Azure path. Contains KQL queries against Log Analytics only. -11. `examples/azure/otel-collector-config.yaml` β€” fleet OpenTelemetry Collector pipeline. Requires active bearer authentication and TLS material from the environment, deletes the seven observed plaintext content attributes, labels the deployment environment, and exports to Application Insights using a connection string read from the environment. -12. `examples/azure/main.bicep`, `main.tf`, `variables.tf`, `outputs.tf`, `versions.tf`, `deploy.sh` β€” templates that create a per-environment Log Analytics workspace, Application Insights component, Azure Monitor dashboard, and an optional workspace-scoped `Monitoring Reader` role assignment. Inert until an operator deploys them. -13. `tests/` β€” static and behavioral tests over the local configuration, the shared policy, the settings executable, and the Azure templates, plus a fuzz harness. They parse committed files and temporary fixtures; they start no container and contact no cloud. -14. `SKILL.md` and `references/` β€” the instructional surface. It authorizes exactly two write behaviors: a diff-approved per-key upsert into the user's global `settings.json`, and writing generated artifacts to disk. +11. `examples/azure/otel-collector-config.yaml` β€” fleet OpenTelemetry Collector pipeline. Requires active bearer authentication and TLS material from the environment, applies the local stack's fail-closed allow-list and content scrub to traces, metrics, and logs, labels the deployment environment after filtering, and exports to Application Insights using a connection string read from the environment. +12. `examples/azure/agent-host-relay/` β€” the per-workstation relay: an OTLP/HTTP loopback receiver, the same filtering policy applied to every signal, and an exporter that authenticates with the fleet ingest token and verifies the fleet receiver's certificate against an operator-supplied CA bundle mounted read-only. Its Compose service is digest-pinned, restart-managed independently of VS Code, memory-limited, read-only, capability-dropped, and reads every runtime input from an operator-owned environment file outside this repository. +13. `examples/azure/main.bicep`, `main.tf`, `variables.tf`, `outputs.tf`, `versions.tf`, `deploy.sh` β€” templates that create a per-environment Log Analytics workspace, Application Insights component, Azure Monitor dashboard, and an optional workspace-scoped `Monitoring Reader` role assignment. Inert until an operator deploys them. +14. `tests/` β€” static and behavioral tests over the local configuration, the shared policy, the settings executable, and the Azure templates, plus a fuzz harness and one runtime module. Most parse committed files and temporary fixtures. `tests/test_collector_carriers.py` is marked `slow` and does start disposable containers: the pinned Collector for the carrier map, and a relay plus an ephemeral TLS receiver for the forwarding evidence. Nothing in the suite contacts a cloud or uses a real credential. +15. `SKILL.md` and `references/` β€” the instructional surface. It authorizes exactly two write behaviors: a diff-approved per-key upsert into the user's global `settings.json`, and writing generated artifacts to disk. ### Data Flow @@ -163,12 +164,12 @@ flowchart TD β”‚ β”‚ prompt-bearing spans β”‚ queries β”‚ β”‚ ─────────┼─────────────────────────────────┼─────────────── β”‚ β”‚ β”‚ 127.0.0.1 ONLY (no off-host listener) β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ OTel Collector β€” the only published OTLP listener β”‚ β”‚ β”‚ β”‚ fail-closed allow-list; unknown attributes dropped β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ internal network, no host mapping β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ TRUST BOUNDARY: otel-lgtm container β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ β”‚ β”‚ OTLP recvβ”‚ β”‚Prometheusβ”‚ β”‚ Tempo β”‚ β”‚ Grafana β”‚ β”‚ β”‚ @@ -224,16 +225,16 @@ flowchart TD ## Adversaries -| Id | Adversary | In-scope mitigations | -|-------|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ADV-a | Off-host network attacker | Every published port binds `127.0.0.1`, so no listener is reachable off the host. Default Grafana credentials are never exposed to a network. | -| ADV-b | Malicious or compromised process on the same host | Not mitigated. Loopback services are unauthenticated and any local process can read or write them (G-SPF-1). | -| ADV-c | Another user on a shared workstation | Bounded by Docker socket access and filesystem permissions on the volume. Not otherwise mitigated (G-INF-2). | -| ADV-d | Upstream image or registry compromise | Tag pinning limits drift; no digest verification or signature check is performed (G-SUP-1). | -| ADV-e | Operator error redirecting the exporter off-host | Documentation states the endpoint carries prompt content and must be treated as sensitive regardless of the capture setting. | -| ADV-f | The agent itself, writing the wrong thing | Backup before write, per-key upsert that never reserializes, exact diff, explicit approval, post-write parse with automatic restore (G-TAM-2). | -| ADV-g | A fleet member misusing the shared ingest credential | Not mitigated. The credential is write-side only and identical on every workstation, with no per-user binding and no in-place rotation (G-INF-3, G-DOS-2). A workstation relay narrows where the credential is readable without binding it to a person (G-INF-7). | -| ADV-h | An operator deploying generated templates carelessly | Every template requires named inputs with no defaults for subscription, region, and naming; the role assignment is opt-in; a daily ingestion cap is set by default (G-EOP-3). | +| Id | Adversary | In-scope mitigations | +|-------|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ADV-a | Off-host network attacker | Every published port binds `127.0.0.1`, so no listener is reachable off the host. Grafana has no default credential: anonymous access is disabled and the admin credential is operator-supplied, so an unset variable fails the `up` rather than starting on a guessable one. | +| ADV-b | Malicious or compromised process on the same host | Not mitigated. Loopback services are unauthenticated and any local process can read or write them (G-SPF-1). | +| ADV-c | Another user on a shared workstation | Bounded by Docker socket access and filesystem permissions on the volume. Not otherwise mitigated (G-INF-2). | +| ADV-d | Upstream image or registry compromise | Both images are pinned by multi-architecture manifest digest, which removes substitution under a tag. Signature and provenance verification remain a documented operator step (G-SUP-1). | +| ADV-e | Operator error redirecting the exporter off-host | Documentation states the endpoint carries prompt content and must be treated as sensitive regardless of the capture setting. | +| ADV-f | The agent itself, writing the wrong thing | Backup before write, per-key upsert that never reserializes, exact diff, explicit approval, and a staged file validated then swapped in atomically so a failed write leaves the original (G-TAM-2). | +| ADV-g | A fleet member misusing the shared ingest credential | Not mitigated. The credential is write-side only and identical on every workstation, with no per-user binding and no in-place rotation (G-INF-3, G-DOS-2). The shipped workstation relay narrows where the credential is readable without binding it to a person (G-INF-7). | +| ADV-h | An operator deploying generated templates carelessly | Every template requires named inputs with no defaults for subscription, region, and naming; the role assignment is opt-in; a daily ingestion cap is set by default (G-EOP-3). | ## Bucket B1: Editor OTLP ingest @@ -330,7 +331,7 @@ Grafana's administrator role is the highest privilege in this bucket. It was pre | Threat | Likelihood | Impact | Residual Risk | Status | |---------------------------------------------------|------------|--------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Prompt content readable at rest by any local user | Low | High | Medium | Collector drops or scrubs content in every carrier it reaches; span names, metric metadata, links, and exemplars survive, and the volume remains unencrypted (G-INF-2, G-INF-8, G-INF-9) | -| Default Grafana credentials accepted | High | Medium | Low | Loopback-only publishing (G-SPF-1, G-EOP-1) | +| Grafana admin reachable without a credential | Low | Medium | Low | Anonymous access disabled and the admin credential is operator-supplied with no default; loopback-only publishing (G-SPF-1, G-EOP-1) | | Dashboard overwritten by the validation helper | Low | Low | Low | Non-loopback targets refused by default (G-TAM-1) | | Unbounded trace growth exhausts disk | Low | Medium | Low | Accepted; operator removes the volume to reclaim | @@ -374,19 +375,19 @@ Not applicable. The helpers run with the invoking user's privileges, spawn no su ## Bucket B4: Container image supply chain -Covers acquisition of the `grafana/otel-lgtm` image. +Covers acquisition of the three images this skill runs: `grafana/otel-lgtm` and `otel/opentelemetry-collector-contrib` in the local stack, and the same Collector image in the workstation relay. ### Spoofing -The image is referenced as `grafana/otel-lgtm:0.29.2` from the default public registry. A registry or namespace compromise able to repoint that tag would be accepted without challenge, because no digest pin and no signature verification are performed. Tracked as G-SUP-1. +Every image is referenced by multi-architecture manifest digest, with the human-readable tag retained in a comment beside it. A registry or namespace compromise able to repoint a tag is therefore rejected, because the digest no longer resolves. What is not performed is signature or provenance verification before the container runs, so a compromise of the publisher's own signing path would still be accepted. Tracked as G-SUP-1. ### Tampering -Tags are mutable. A republished `0.29.2` would be pulled on any host that has not already cached the layers, and nothing in the skill would detect the substitution. Digest pinning is the standard control and is not applied here. Tracked as G-SUP-1. +A republished tag cannot reach a host that pulls by digest, which is what removes the substitution vector a mutable tag leaves open. The residual is narrower: an image whose digest this repository records could itself have been built from compromised inputs, and nothing here verifies the publisher's build provenance. Tracked as G-SUP-1. ### Repudiation -Docker records the resolved image digest locally after a pull, so the specific image actually running is identifiable on that host after the fact. The skill does not capture or compare that digest, so drift between two hosts pulling the same tag at different times goes unnoticed. +Docker records the resolved image digest locally after a pull, so the specific image actually running is identifiable on that host after the fact. Because the reference is the digest, two hosts pulling at different times run the same image by construction rather than by coincidence. ### Information Disclosure @@ -394,19 +395,19 @@ Not applicable. The pull requests a public image by name and reveals nothing bey ### Denial of Service -Registry unavailability blocks the first run only. Once layers are cached, `docker compose up -d` succeeds without network access, and `restart: unless-stopped` returns the stack after a host reboot. +Registry unavailability blocks the first run only. Once layers are cached, `docker compose up -d` succeeds without network access, and `restart: unless-stopped` returns the stack after a host reboot. For the relay this matters more than it does locally, because a relay that cannot start takes that workstation's whole telemetry path with it (G-DOS-3). ### Elevation of Privilege -The container runs under the Docker daemon with whatever default privileges the image declares, and the compose definition adds no capabilities, no privileged flag, and no host mounts other than the single named data volume. A compromised image would nonetheless hold whatever authority the daemon grants, which on a typical developer workstation is root-equivalent. That is inherent to running any container and is bounded by the tag pin alone. Tracked as G-SUP-1 and G-EOP-2. +The containers run under the Docker daemon with whatever privileges the image declares. The local Collector and the relay drop all capabilities, set `no-new-privileges`, mount a read-only root filesystem, declare a memory limit, and mount no host paths beyond their read-only configuration, the named data volume, and the relay's read-only CA bundle. A compromised image would nonetheless hold whatever authority the daemon grants, which on a typical developer workstation is root-equivalent. That is inherent to running any container and is bounded here by digest pinning and by those container constraints. Tracked as G-SUP-1 and G-EOP-2. ### Risk Rating -| Threat | Likelihood | Impact | Residual Risk | Status | -|------------------------------------------|------------|--------|---------------|-------------------------------------------------------------------------------------------| -| Malicious image substitution under a tag | Low | High | Low | Both images digest-pinned; publisher verification is a separate documented step (G-SUP-1) | -| Compromised image gains daemon authority | Low | High | Medium | No added capabilities or host mounts (G-EOP-2) | -| Registry outage blocks first run | Low | Low | Low | Accepted; cached layers make later runs offline | +| Threat | Likelihood | Impact | Residual Risk | Status | +|------------------------------------------|------------|--------|---------------|------------------------------------------------------------------------------------------------------------------------------------| +| Malicious image substitution under a tag | Low | High | Low | Both images digest-pinned; publisher verification is a separate documented step (G-SUP-1) | +| Compromised image gains daemon authority | Low | High | Medium | All capabilities dropped, no-new-privileges, read-only root, memory limit, no host mounts beyond read-only configuration (G-EOP-2) | +| Registry outage blocks first run | Low | Low | Low | Accepted; cached layers make later runs offline | ## Bucket B5: Editor-global configuration mutation @@ -528,33 +529,35 @@ Deployment creates billable Azure resources and, when a principal is supplied, a ## Enterprise Readiness Gaps -| Id | Severity | Gap | Status | -|----------|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| G-INF-1 | InfoDisc-High | Spans carry full prompt text, tool call arguments and results, and system instructions on a configuration where content capture was left at its documented default. The skill cannot change extension behavior. | Partially mitigated. Content still leaves the editor and crosses the loopback hop in plaintext, which remains outside the skill's control. Both the local and organization paths now filter before storage: local uses a fail-closed allow-list at the Collector, so an attribute added by a future extension release is dropped rather than stored, and the Azure path deletes the seven observed content attributes before export. | -| G-INF-2 | InfoDisc-Med | The `copilot-otel-data` volume stores whatever survives filtering, unencrypted, and survives `docker compose down` by design. | Open on encryption, reduced on content. The allow-list keeps observed and unknown content attributes out of the volume, so what remains is usage and cost data rather than prompt text. Volume encryption is a Docker and host-platform property the skill does not control. Traces inherit the image's 336h Tempo expiry rather than persisting indefinitely; metrics retain for 120 days by configuration. Teardown documentation states both variants explicitly. | -| G-INF-3 | InfoDisc-Med | Organization capture distributes one static write-side credential to every workstation, with no per-user binding and no documented in-place rotation. Anything holding it can inject telemetry indistinguishable from a real developer's. | Open and now active rather than optional. The receiver requires the token instead of leaving authentication commented out, so the gap is a bounded shared-credential weakness rather than an unauthenticated endpoint. Disclosed before any Azure artifact is generated, with the write-side-only blast radius stated plainly. | -| G-INF-4 | InfoDisc-Low | The assisted settings write reads the whole global `settings.json`, so unrelated values in that file enter model context. | Accepted. Reading the whole document is required to preserve it; only the changed lines are echoed in the presented diff. | -| G-SPF-1 | Spoofing-Med | The Prometheus and Tempo query APIs are unauthenticated, and the local OTLP ingress accepts spans from any local process. | Reduced. Grafana now requires an operator-supplied credential with anonymous access disabled. Prometheus and Tempo remain unauthenticated behind `127.0.0.1` binding, and injected spans are still accepted, though the allow-list bounds what an injector can store. `baseline.py` provides after-the-fact detection of injected series. | -| G-SUP-1 | SupplyChain-Med | Image identity was previously pinned by mutable tag, and no signature or provenance verification is performed before the container runs. | Partially closed. Both images are now pinned by multi-architecture manifest digest, which removes the substitution-under-a-tag vector. Signature and provenance verification remains a documented operator step rather than an enforced one; the update procedure states that digest pinning, publisher verification, and vulnerability review are three separate checks. | -| G-EOP-1 | EoP-Low | Grafana administrator access was previously reachable from any local process using published default credentials. | Closed. Anonymous Admin access is disabled and the admin credential is operator-supplied with no default. The residual observation stands that the same local actor can read the volume directly, so this was never a host-user boundary crossing. | -| G-EOP-2 | EoP-Med | A compromised stack image would execute with Docker daemon authority, which is root-equivalent on a typical developer workstation. | Bounded by digest pinning, by adding no capabilities, and by mounting no host paths beyond the named volume and the read-only Collector configuration. | -| G-TAM-1 | Tampering-Low | `validate_dashboard.py` imports with `overwrite: true`, so it would replace an unrelated dashboard sharing the same uid. | Constrained. All configurable endpoints pass a shared policy that refuses non-http schemes, credentials in the authority, unrelated local ports, and non-loopback hosts without an explicit opt-in, and re-applies the same rules to every redirect target. The dashboard path is contained to the examples directory. | -| G-TAM-2 | Tampering-Med | The assisted settings write mutates a user-owned JSONC file that may hold comments, formatting, and unrelated configuration the skill did not create. | Mitigated structurally and now executably. `examples/settings_upsert.py` enforces the contract the prose described: schema-checked keys and types, value-span splicing that never reserializes, a refusal to write when unrelated settings would change or the result would not parse, backup before write, restore on failure, and a redacted audit record. Concurrent VS Code writes remain unpreventable. | -| G-REP-1 | Repudiation-Low | Settings Sync conflict behavior for a write made outside the running VS Code instance was never confirmed. | Open. Disclosed to the user as a caveat before the write rather than asserted in either direction. | -| G-DOS-1 | DoS-Low | Delta-to-cumulative conversion state is held in memory and resets on container restart, producing a bounded gap in converted series. | Accepted. Without the flag the failure mode is worse, because a dropped delta metric can fail an entire batched write. | -| G-DOS-2 | DoS-Low | The shared fleet credential permits unbounded ingestion against a billed backend, so the practical denial of service is financial. | Mitigated by a 5 GB daily cap defaulted in every generated template, and by naming `captureContent` as the dominant volume multiplier wherever cost is discussed. | -| G-EOP-3 | EoP-Med | Generated infrastructure templates provision billable Azure resources and can create a `Monitoring Reader` role assignment when deployed. | Bounded. The agent never holds Azure credentials and never deploys; the role assignment is opt-in through an empty-by-default parameter and is scoped to a single workspace rather than to the resource group or subscription; required inputs have no defaults. | -| G-TLS-2 | Spoofing-Med | The generated fleet receiver now terminates TLS from operator-supplied material, but how Copilot's exporter validates that certificate is exporter behavior the skill does not control. | Open by ownership. The receiver requires TLS material or an explicitly recorded terminating ingress that owns certificate lifecycle and cipher policy. No claim is made about exporter-side validation or custom CA support, because none was verified. | -| G-INF-5 | InfoDisc-Med | A single shared Log Analytics workspace gives every authorized reader visibility across all environments, and a resource attribute cannot restrict that after the fact. | Mitigated by default. One collector endpoint, ingest token, Application Insights component, and workspace per environment is now the shipped topology; environment is a required input with no default in Bicep, Terraform, and the CLI script. Reader assignments are scoped to a single workspace. Attributes are documented as grouping keys, never as isolation controls. This is environment separation, not customer multi-tenancy, and is stated as such. | -| G-INF-6 | InfoDisc-Low | Retention is the only deletion mechanism. Neither the local stack nor the Azure templates purge a specific person's data on request. | Open and operator-owned. Retention is documented as the deletion boundary in both templates and their guidance, and an erasure obligation is presented as an operator procedure against the workspace rather than a template capability. No purge automation is provided. | -| G-SUP-2 | SupplyChain-Low | Terraform state for the fleet templates contains the Application Insights connection string, a fleet-wide write credential, and `sensitive = true` does not remove it from state. | Documented rather than solved. The configuration, its README, and the Azure reference all state that sensitivity hides CLI display only, and recommend an encrypted remote backend with restricted access and treating the state file as a secret. | -| G-EOP-4 | EoP-Med | The prohibition on the agent running `docker compose`, `az deployment`, or `terraform apply` is advisory prose in `SKILL.md` rather than an enforced control. | Open. Stated in the constraints and the stop rules and exercised by the behavior gate. A hook would make it enforced. | -| G-TLS-1 | InfoDisc-Low | OTLP ingest and all service queries use plaintext HTTP with no transport security. | Acceptable on loopback. Material the moment `otlpEndpoint` targets a remote collector, which the skill states explicitly. | -| G-INF-7 | InfoDisc-Med | The per-workstation relay that restores fleet agent-host telemetry listens without authentication, so any local process able to reach loopback can inject spans that the relay then forwards under the fleet credential. | Open, and the least-bad of three options: the alternatives are delivering the ingest token through an environment agent-spawned subprocesses inherit, or removing receiver authentication entirely. Loopback binding is necessary and not sufficient for provenance. The relay authenticates the relay, not the workstation and not the developer. mTLS is the preferred replacement and is blocked on unestablished agent-host client-certificate support. | -| G-INF-8 | InfoDisc-Med | Span names, metric names, metric descriptions, and metric units reach storage unfiltered. A span name is emitter-composed and can carry request text. | Open by choice, not by oversight. Three dashboard TraceQL queries match on span name, `baseline.py` reads Tempo `rootTraceName` as its injection discriminator, and every shipped Prometheus query selects by metric name; scrubbing any of them breaks shipped functionality. Recorded by `tests/test_collector_carriers.py`, which fails if a scrub rule is later added that targets them. | -| G-INF-9 | InfoDisc-Med | Span link attributes, span link trace state, and metric exemplar filtered attributes reach storage unfiltered. | Open by capability. Verified against the pinned Collector: the `redaction` processor does not traverse links or exemplars, OTTL has no `spanlink` context and refuses to index `links`, and assigning to `datapoint.exemplars` parses without effect. No processor in this distribution reaches them. Whether Copilot populates links or exemplars in practice is unmeasured, so the exposure is recorded unbounded rather than narrowed by an assumption; settling it needs a live capture. | -| G-INF-10 | InfoDisc-Low | The carrier map is established against the OTLP/HTTP receiver. The shipped configuration also publishes OTLP/gRPC, whose content handling is unprobed. | Open. Both receivers feed the same processor chain, so divergence is unlikely rather than excluded. Closing it means extending the harness to the gRPC path. | -| G-INF-11 | InfoDisc-Low | Instrumentation scope name and version and the resource and scope schema URLs reach storage unfiltered. They normally carry library identity, but the receiver is unauthenticated and no processor filters them, so a local process can place arbitrary text there. | Open by low expected content, the weakest of this document's three justifications for an open carrier. Recorded as a gap rather than dismissed by rationale, because "identity, not content" is the same argument the harness disproved for span links. Closing it means extending `transform/scrub` to these fields, which is possible for the schema URLs and would need a consumer check for scope name. | +| Id | Severity | Gap | Status | +|----------|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| G-INF-1 | InfoDisc-High | Spans carry full prompt text, tool call arguments and results, and system instructions on a configuration where content capture was left at its documented default. The skill cannot change extension behavior. | Partially mitigated. Content still leaves the editor and crosses the loopback hop in plaintext, which remains outside the skill's control. Both the local and organization paths now filter before storage: local uses a fail-closed allow-list at the Collector, so an attribute added by a future extension release is dropped rather than stored, and the Azure path applies the same fail-closed allow-list and content scrub to traces, metrics, and logs before export. | +| G-INF-2 | InfoDisc-Med | The `copilot-otel-data` volume stores whatever survives filtering, unencrypted, and survives `docker compose down` by design. | Open on encryption, reduced on content. The allow-list keeps observed and unknown content attributes out of the volume, so what remains is usage and cost data rather than prompt text. Volume encryption is a Docker and host-platform property the skill does not control. Traces inherit the image's 336h Tempo expiry rather than persisting indefinitely; metrics retain for 120 days by configuration. Teardown documentation states both variants explicitly. | +| G-INF-3 | InfoDisc-Med | Organization capture distributes one static write-side credential to every workstation, with no per-user binding and no documented in-place rotation. Anything holding it can inject telemetry indistinguishable from a real developer's. | Open and now active rather than optional. The receiver requires the token instead of leaving authentication commented out, so the gap is a bounded shared-credential weakness rather than an unauthenticated endpoint. Disclosed before any Azure artifact is generated, with the write-side-only blast radius stated plainly. | +| G-INF-4 | InfoDisc-Low | The assisted settings write reads the whole global `settings.json`, so unrelated values in that file enter model context. | Accepted. Reading the whole document is required to preserve it; only the changed lines are echoed in the presented diff. | +| G-SPF-1 | Spoofing-Med | The Prometheus and Tempo query APIs are unauthenticated, and the local OTLP ingress accepts spans from any local process. | Reduced. Grafana now requires an operator-supplied credential with anonymous access disabled. Prometheus and Tempo remain unauthenticated behind `127.0.0.1` binding, and injected spans are still accepted, though the allow-list bounds what an injector can store. `baseline.py` provides after-the-fact detection of injected series. | +| G-SUP-1 | SupplyChain-Med | Image identity was previously pinned by mutable tag, and no signature or provenance verification is performed before the container runs. | Partially closed. Both images are now pinned by multi-architecture manifest digest, which removes the substitution-under-a-tag vector. Signature and provenance verification remains a documented operator step rather than an enforced one; the update procedure states that digest pinning, publisher verification, and vulnerability review are three separate checks. | +| G-EOP-1 | EoP-Low | Grafana administrator access was previously reachable from any local process using published default credentials. | Closed. Anonymous Admin access is disabled and the admin credential is operator-supplied with no default. The residual observation stands that the same local actor can read the volume directly, so this was never a host-user boundary crossing. | +| G-EOP-2 | EoP-Med | A compromised stack image would execute with Docker daemon authority, which is root-equivalent on a typical developer workstation. | Bounded by digest pinning, by adding no capabilities, and by mounting no host paths beyond the named volume and the read-only Collector configuration. | +| G-TAM-1 | Tampering-Low | `validate_dashboard.py` imports with `overwrite: true`, so it would replace an unrelated dashboard sharing the same uid. | Constrained. All configurable endpoints pass a shared policy that refuses non-http schemes, credentials in the authority, unrelated local ports, and non-loopback hosts without an explicit opt-in, and re-applies the same rules to every redirect target. The dashboard path is contained to the examples directory. | +| G-TAM-2 | Tampering-Med | The assisted settings write mutates a user-owned JSONC file that may hold comments, formatting, and unrelated configuration the skill did not create. | Mitigated structurally and now executably. `examples/settings_upsert.py` enforces the contract the prose described: schema-checked keys and types, value-span splicing that never reserializes, a refusal to write when unrelated settings would change or the result would not parse, backup before write, restore on failure, and a redacted audit record. Concurrent VS Code writes remain unpreventable. | +| G-REP-1 | Repudiation-Low | Settings Sync conflict behavior for a write made outside the running VS Code instance was never confirmed. | Open. Disclosed to the user as a caveat before the write rather than asserted in either direction. | +| G-DOS-1 | DoS-Low | Delta-to-cumulative conversion state is held in memory and resets on container restart, producing a bounded gap in converted series. | Accepted. Without the flag the failure mode is worse, because a dropped delta metric can fail an entire batched write. | +| G-DOS-2 | DoS-Low | The shared fleet credential permits unbounded ingestion against a billed backend, so the practical denial of service is financial. | Mitigated by a 5 GB daily cap defaulted in every generated template, and by naming `captureContent` as the dominant volume multiplier wherever cost is discussed. | +| G-EOP-3 | EoP-Med | Generated infrastructure templates provision billable Azure resources and can create a `Monitoring Reader` role assignment when deployed. | Bounded. The agent never holds Azure credentials and never deploys; the role assignment is opt-in through an empty-by-default parameter and is scoped to a single workspace rather than to the resource group or subscription; required inputs have no defaults. | +| G-TLS-2 | Spoofing-Med | The generated fleet receiver now terminates TLS from operator-supplied material, but how Copilot's exporter validates that certificate is exporter behavior the skill does not control. | Open by ownership. The receiver requires TLS material or an explicitly recorded terminating ingress that owns certificate lifecycle and cipher policy. No claim is made about exporter-side validation or custom CA support, because none was verified. | +| G-INF-5 | InfoDisc-Med | A single shared Log Analytics workspace gives every authorized reader visibility across all environments, and a resource attribute cannot restrict that after the fact. | Mitigated by default. One collector endpoint, ingest token, Application Insights component, and workspace per environment is now the shipped topology; environment is a required input with no default in Bicep, Terraform, and the CLI script. Reader assignments are scoped to a single workspace. Attributes are documented as grouping keys, never as isolation controls. This is environment separation, not customer multi-tenancy, and is stated as such. | +| G-INF-6 | InfoDisc-Low | Retention is the only deletion mechanism. Neither the local stack nor the Azure templates purge a specific person's data on request. | Open and operator-owned. Retention is documented as the deletion boundary in both templates and their guidance, and an erasure obligation is presented as an operator procedure against the workspace rather than a template capability. No purge automation is provided. | +| G-SUP-2 | SupplyChain-Low | Terraform state for the fleet templates contains the Application Insights connection string, a fleet-wide write credential, and `sensitive = true` does not remove it from state. | Documented rather than solved. The configuration, its README, and the Azure reference all state that sensitivity hides CLI display only, and recommend an encrypted remote backend with restricted access and treating the state file as a secret. | +| G-EOP-4 | EoP-Med | The prohibition on the agent running `docker compose`, `az deployment`, or `terraform apply` is advisory prose in `SKILL.md` rather than an enforced control. | Open. Stated in the constraints and the stop rules and exercised by the behavior gate. A hook would make it enforced. | +| G-TLS-1 | InfoDisc-Low | OTLP ingest and all service queries use plaintext HTTP with no transport security. | Acceptable on loopback. Material the moment `otlpEndpoint` targets a remote collector, which the skill states explicitly. | +| G-INF-7 | InfoDisc-Med | The shipped per-workstation relay listens without authentication, so any local process able to reach loopback can inject spans that the relay then forwards under the fleet credential. Its runtime environment file is readable by anything running as the same operating-system account or able to inspect the container. | Open, and the least-bad of three options: the alternatives are delivering the ingest token through an environment agent-spawned subprocesses inherit, or removing receiver authentication entirely. Loopback binding is necessary and not sufficient for provenance. The relay authenticates the relay, not the workstation and not the developer. The relay verifies the fleet receiver's certificate against an operator-supplied CA bundle and refuses an untrusted certificate or a mismatched hostname, which is proven by `tests/test_collector_carriers.py`. mTLS is the preferred replacement and is blocked on unestablished agent-host client-certificate support. | +| G-DOS-3 | DoS-Med | Managed settings expose one telemetry endpoint to both the extension and the agent host, so pointing it at the relay makes relay health a prerequisite for all of a workstation's Copilot telemetry. A stopped, missing, or unhealthy relay loses that telemetry outright rather than queueing it. | Open by design trade. The alternative topologies either distribute the fleet credential into every agent-spawned subprocess or leave agent-host telemetry dropped fleet-wide. Bounded by `restart: unless-stopped`, a health endpoint, relay-first rollout with positive verification before the managed endpoint changes, a documented rollback to the previous endpoint, and a detection table whose first symptom is total signal absence from one workstation. | +| G-INF-12 | InfoDisc-Low | The content scrub runs with `error_mode: ignore`, so an OTTL statement that errors on one record shape is skipped and that record's carrier reaches the exporter unscrubbed. This makes the scrub fail-open per statement while its sibling allow-list is fail-closed. | Open and load-bearing rather than incidental: a map-valued log body keeps its allow-listed subset precisely because `set(log.body, ...)` does not apply to it. Bounded by digest-pinned images and by `tests/test_collector_carriers.py`, which asserts the resulting carrier map against the pinned runtime, so a change in that behaviour fails the suite rather than passing quietly. Recorded here rather than left implicit in the configuration comment. | +| G-INF-8 | InfoDisc-Med | Span names, metric names, metric descriptions, and metric units reach storage unfiltered. A span name is emitter-composed and can carry request text. | Open by choice, not by oversight. Three dashboard TraceQL queries match on span name, `baseline.py` reads Tempo `rootTraceName` as its injection discriminator, and every shipped Prometheus query selects by metric name; scrubbing any of them breaks shipped functionality. Recorded by `tests/test_collector_carriers.py`, which fails if a scrub rule is later added that targets them. | +| G-INF-9 | InfoDisc-Med | Span link attributes, span link trace state, and metric exemplar filtered attributes reach storage unfiltered. | Open by capability. Verified against the pinned Collector: the `redaction` processor does not traverse links or exemplars, OTTL has no `spanlink` context and refuses to index `links`, and assigning to `datapoint.exemplars` parses without effect. No processor in this distribution reaches them. Whether Copilot populates links or exemplars in practice is unmeasured, so the exposure is recorded unbounded rather than narrowed by an assumption; settling it needs a live capture. | +| G-INF-10 | InfoDisc-Low | The carrier map is established against the OTLP/HTTP receiver. The shipped configuration also publishes OTLP/gRPC, whose content handling is unprobed. | Open. Both receivers feed the same processor chain, so divergence is unlikely rather than excluded. Closing it means extending the harness to the gRPC path. | +| G-INF-11 | InfoDisc-Low | Instrumentation scope name and version and the resource and scope schema URLs reach storage unfiltered. They normally carry library identity, but the receiver is unauthenticated and no processor filters them, so a local process can place arbitrary text there. | Open by low expected content, the weakest of this document's three justifications for an open carrier. Recorded as a gap rather than dismissed by rationale, because "identity, not content" is the same argument the harness disproved for span links. Closing it means extending `transform/scrub` to these fields, which is possible for the schema URLs and would need a consumer check for scope name. | ## References diff --git a/.github/skills/experimental/copilot-otel-metrics/SKILL.md b/.github/skills/experimental/copilot-otel-metrics/SKILL.md index a0d23131c..b2d402279 100644 --- a/.github/skills/experimental/copilot-otel-metrics/SKILL.md +++ b/.github/skills/experimental/copilot-otel-metrics/SKILL.md @@ -9,7 +9,7 @@ compatibility: 'VS Code with GitHub Copilot Chat. Docker with Compose v2 for the metadata: authors: "microsoft/hve-core" spec_version: "1.0" - last_updated: "2026-07-27" + last_updated: "2026-08-18" --- # Copilot OpenTelemetry Metrics @@ -120,12 +120,13 @@ Read the references. Copy or adapt the seeds. Offer the helpers to the user with | `examples/otel-collector-local.yaml` | Copy alongside `compose.yaml`; carries the fail-closed local attribute allow-list and the content scrub for the carriers it cannot reach | | `examples/dashboards/copilot-otel.json` | Copy as the seed for a generated local PromQL dashboard | | `examples/dashboards/copilot-otel-azure.json` | Copy as the seed for a generated Azure KQL dashboard | -| `examples/azure/` | Copy as the seed for the collector configuration and infrastructure templates | +| `examples/azure/` | Copy as the seed for the fleet collector configuration and infrastructure templates | +| `examples/azure/agent-host-relay/` | Copy as the per-workstation relay every fleet deployment runs; it must be healthy before managed settings point at loopback | | `examples/verify.py` | Offer to the user as a stored-signal check they run themselves | | `examples/inspect_metrics.py` | Offer to the user to enumerate the metric surface their build emits | | `examples/baseline.py` | Offer to the user to separate real telemetry from residue | | `examples/validate_dashboard.py` | Offer to the user for a **local** PromQL or TraceQL dashboard only; it has no Azure Monitor path and it overwrites by uid | -| `examples/settings_upsert.py` | Offer to the user for the assisted settings write; it enforces the schema, backs up, and restores on a parse failure | +| `examples/settings_upsert.py` | Offer to the user for the assisted settings write; it enforces the schema, backs up, and replaces the file atomically | | `examples/README.md`, `examples/azure/README.md` | Written for the user, not part of the agent's reading path. Point the user at them rather than summarizing them | ## References diff --git a/TRANSPARENCY-NOTE.md b/TRANSPARENCY-NOTE.md index 0ac68f166..30760b2f9 100644 --- a/TRANSPARENCY-NOTE.md +++ b/TRANSPARENCY-NOTE.md @@ -2,7 +2,7 @@ title: "Transparency Note: HVE Core" description: "Public Transparency Note for HVE Core, a prompt-engineering and agentic-customization framework distributed by microsoft/hve-core." author: HVE Core Maintainers -ms.date: 2026-08-13 +ms.date: 2026-08-18 ms.topic: overview keywords: - responsible-ai @@ -20,7 +20,7 @@ This note covers HVE Core: the files in the `microsoft/hve-core` repository. It ## The basics of HVE Core -HVE Core is a collection of text files and supporting tools that shape how GitHub Copilot behaves. It ships custom agents, prompts, instructions, skills, hooks, PowerShell scripts, GitHub Actions workflows, and one complete Visual Studio Code extension and Copilot plugin identity. The point is to give engineering teams a ready-made, review-friendly starting point for AI-assisted software work. +HVE Core is a collection of text files and supporting tools that shape how GitHub Copilot behaves. It ships custom agents, prompts, instructions, skills, PowerShell scripts, GitHub Actions workflows, and one complete Visual Studio Code extension and Copilot plugin identity. The point is to give engineering teams a ready-made, review-friendly starting point for AI-assisted software work. HVE Core does not run any AI model itself. It does not train models, host inference, call external services while you use it, or process personal data on its own. All of the AI work happens on the host platform (Copilot Chat or the Copilot CLI). That leaves three areas where HVE Core still carries Responsible AI weight: diff --git a/docs/contributing/hooks.md b/docs/contributing/hooks.md index 4367240e8..4bae9ca89 100644 --- a/docs/contributing/hooks.md +++ b/docs/contributing/hooks.md @@ -3,7 +3,7 @@ title: Contributing Hooks description: How to implement, register, and validate hook artifacts in hve-core sidebar_position: 7 author: Microsoft -ms.date: 2026-08-13 +ms.date: 2026-08-18 ms.topic: how-to keywords: - hooks @@ -78,7 +78,7 @@ For reliability and portability, hook scripts should follow these rules: ## Manifest Schema and Validation -Manifests are validated against `scripts/linting/schemas/hook-manifest.schema.json`, the authoritative contract. The schema enforces the allowed top-level keys (`version`, `description`, `hooks`), the ten CLI-lowercase event names (`sessionStart`, `sessionEnd`, `userPromptSubmit`, `userPromptSubmitted`, `preToolUse`, `postToolUse`, `preCompact`, `subagentStart`, `subagentStop`, `stop`), and the permitted command properties. +Manifests are validated against `scripts/linting/schemas/hook-manifest.schema.json`, the authoritative contract. The schema enforces the allowed top-level keys (`version`, `description`, `hooks`), the eleven CLI-lowercase event names (`sessionStart`, `sessionEnd`, `userPromptSubmit`, `userPromptSubmitted`, `preToolUse`, `postToolUse`, `preCompact`, `subagentStart`, `subagentStop`, `stop`, `agentStop`), and the permitted command properties. Run `npm run lint:hooks` to validate every package-scoped manifest. On failure, the validator prints each error and the schema path so you can reconcile the manifest against the contract. diff --git a/docs/customization/README.md b/docs/customization/README.md index b4d1198c6..f99dc306f 100644 --- a/docs/customization/README.md +++ b/docs/customization/README.md @@ -2,7 +2,7 @@ title: Customizing HVE Core description: Overview of customization approaches from lightweight settings to full fork-and-extend, with role-based entry points author: Microsoft -ms.date: 2026-08-13 +ms.date: 2026-08-18 ms.topic: overview sidebar_position: 1 keywords: @@ -96,20 +96,20 @@ with type-specific examples. ## Role-Based Entry Points -Each HVE role benefits from different customization techniques. The table below maps the nine roles to the guides most relevant to their workflow. - -| Role | Recommended Guides | Rationale | -|--------------------------|-------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------| -| Engineer | [Instructions](instructions.md), [Agents](custom-agents.md) | Coding standards and specialized review agents accelerate daily development | -| TPM | [Prompts](prompts.md), [Plugin Manifest](packages.md) | Reusable planning prompts and managed distribution standardize project workflows | -| Tech Lead / Architect | [Instructions](instructions.md), [Agents](custom-agents.md), [Skills](skills.md) | Standards enforcement, architecture review agents, and deep domain knowledge | -| Security Architect | [Skills](skills.md), [Instructions](instructions.md) | Compliance knowledge packages and security-focused coding conventions | -| Data Scientist | [Skills](skills.md), [Prompts](prompts.md) | Analytical domain bundles and repeatable notebook workflows | -| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md) | Infrastructure conventions and DevContainer tuning | -| Platform / Observability | [Copilot OTel Metrics](copilot-otel-metrics.md) | Agent usage, token cost, and latency measurement through OpenTelemetry | -| Business Program Manager | [Prompts](prompts.md), [Team Adoption](team-adoption.md) | Sprint-planning prompts and governance patterns for stakeholder alignment | -| New Contributor | [Instructions](instructions.md), [Environment](environment.md) | Quick onboarding through conventions and a ready-to-use development environment | -| Utility | [Plugin Manifest](packages.md), [Build System](build-system.md) | Cross-cutting tooling assembly and validation pipeline customization | +Each HVE role benefits from different customization techniques. The table below maps each role to the guides most relevant to their workflow. + +| Role | Recommended Guides | Rationale | +|--------------------------|----------------------------------------------------------------------------------|----------------------------------------------------------------------------------| +| Engineer | [Instructions](instructions.md), [Agents](custom-agents.md) | Coding standards and specialized review agents accelerate daily development | +| TPM | [Prompts](prompts.md), [Plugin Manifest](packages.md) | Reusable planning prompts and managed distribution standardize project workflows | +| Tech Lead / Architect | [Instructions](instructions.md), [Agents](custom-agents.md), [Skills](skills.md) | Standards enforcement, architecture review agents, and deep domain knowledge | +| Security Architect | [Skills](skills.md), [Instructions](instructions.md) | Compliance knowledge packages and security-focused coding conventions | +| Data Scientist | [Skills](skills.md), [Prompts](prompts.md) | Analytical domain bundles and repeatable notebook workflows | +| SRE / Operations | [Instructions](instructions.md), [Environment](environment.md) | Infrastructure conventions and DevContainer tuning | +| Platform / Observability | [Copilot OTel Metrics](copilot-otel-metrics.md) | Agent usage, token cost, and latency measurement through OpenTelemetry | +| Business Program Manager | [Prompts](prompts.md), [Team Adoption](team-adoption.md) | Sprint-planning prompts and governance patterns for stakeholder alignment | +| New Contributor | [Instructions](instructions.md), [Environment](environment.md) | Quick onboarding through conventions and a ready-to-use development environment | +| Utility | [Plugin Manifest](packages.md), [Build System](build-system.md) | Cross-cutting tooling assembly and validation pipeline customization | ## File Index @@ -122,7 +122,8 @@ Each HVE role benefits from different customization techniques. The table below 7. [Forking and Extending](forking.md): Full fork-and-extend customization 8. [Environment Customization](environment.md): DevContainers, VS Code settings, MCP servers 9. [Team Adoption and Governance](team-adoption.md): Governance, naming, onboarding, change management -10. [Copilot OpenTelemetry Metrics](copilot-otel-metrics.md): Export Copilot OTel signals to a local Grafana stack, or to a fleet-wide Azure pipeline, and query agent, token, and latency data +10. [Enterprise Artifact Hub](enterprise-artifact-hub.md): Distribute and govern artifacts across an organization +11. [Copilot OpenTelemetry Metrics](copilot-otel-metrics.md): Export Copilot OTel signals to a local Grafana stack, or to a fleet-wide Azure pipeline, and query agent, token, and latency data ## Related Resources diff --git a/docs/customization/copilot-otel-metrics.md b/docs/customization/copilot-otel-metrics.md index 0a13a9f6d..5962163cb 100644 --- a/docs/customization/copilot-otel-metrics.md +++ b/docs/customization/copilot-otel-metrics.md @@ -3,7 +3,7 @@ title: Copilot OpenTelemetry Metrics description: Capture GitHub Copilot Chat OpenTelemetry signals on your own machine in a local Grafana stack, or across an organization through Azure sidebar_position: 11 author: Microsoft -ms.date: 2026-08-13 +ms.date: 2026-08-18 ms.topic: how-to keywords: - opentelemetry @@ -178,12 +178,13 @@ Several details in that file are deliberate: The filter decides what is stored. It does not change what the extension emits, so content attributes still leave the editor and still cross the loopback hop in plaintext before the Collector drops them. -Everything in this guide is copy-pasteable, but the dashboard is not: it is roughly nineteen kilobytes of JSON. -Runnable copies of the compose file, the dashboard, and the verification helpers live in the +Two artifacts in this guide are not copy-pasteable. The dashboard is roughly nineteen kilobytes of JSON, and the Collector configuration is shown above only as an excerpt. The compose file bind-mounts `./otel-collector-local.yaml`, so the Collector cannot start without the complete file: Docker creates an empty directory at that path instead. Take both from the skill rather than from this page. +Runnable copies of the compose file, the full Collector configuration, the dashboard, and the helper scripts live in the [copilot-otel-metrics skill](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SKILL.md), whose [examples directory](https://github.com/microsoft/hve-core/tree/main/.github/skills/experimental/copilot-otel-metrics/examples) -holds [the dashboard JSON](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/examples/dashboards/copilot-otel.json) -to import into Grafana. The YAML above mirrors the file shipped there. +holds [the Collector configuration](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml) +and [the dashboard JSON](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/examples/dashboards/copilot-otel.json) +to import into Grafana. The YAML above mirrors the files shipped there. ## Turn on export @@ -373,7 +374,7 @@ Four small Python scripts sit beside the dashboard in the skill's [examples dire | `baseline.py` | Is this telemetry genuinely Copilot's, or residue from something else | | `validate_dashboard.py` | Does every panel in this dashboard return data | -Download them, then run them from wherever you put them: +All four import `_input_policy.py`, a fifth file in the same directory that holds their shared endpoint and path rules. Download it alongside them and keep it in the same directory, or every one of them fails at import: ```bash python3 verify.py diff --git a/docs/security/security-model.md b/docs/security/security-model.md index 056dbc5ba..b760cd4ee 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -3,7 +3,7 @@ title: Security Assurance Case and Security Model description: Comprehensive security model and security assurance documentation demonstrating enterprise security practices sidebar_position: 2 author: Microsoft -ms.date: 2026-08-14 +ms.date: 2026-08-18 ms.topic: reference keywords: - security @@ -1669,7 +1669,7 @@ Likelihood, Impact, and Residual Risk are taken from the named per-skill risk-ra ### Copilot Telemetry Skill Threats -The `copilot-otel-metrics` skill turns on GitHub Copilot Chat's OTLP export and stands up somewhere for the data to land, locally in a containerized Grafana/Prometheus/Tempo/Loki/Pyroscope stack or in an operator-deployed Azure Monitor workspace. +The `copilot-otel-metrics` skill turns on GitHub Copilot Chat's OTLP export and stands up somewhere for the data to land, locally in a containerized Grafana/Prometheus/Tempo/Loki stack or in an operator-deployed Azure Monitor workspace. The `otel-lgtm` image also runs Pyroscope, which no shipped dashboard or helper queries; it shares the same data volume, so it is part of the asset below even though nothing in the skill sends it profiles. The primary asset is the payload rather than the code. Spans emitted by the extension were directly observed carrying full prompt text, tool-call arguments and results, and system instructions on a configuration where content capture was left at its documented default, so following the skill accumulates a durable corpus of prompt content. The threats below are grouped by the skill's own trust buckets: B1 editor OTLP ingest, B2 telemetry at rest and query surfaces, B3 reference helper scripts, B4 container image supply chain, B5 editor-global configuration mutation, B6 host process control, and B7 cloud control-plane artifact generation. @@ -2259,20 +2259,20 @@ Most skills are markdown knowledge packs with no runtime and are covered by the Skills that ship an executable runtime (network egress, credential handling, subprocess execution, or untrusted document/content parsing) carry their own per-skill STRIDE threat model in a `SECURITY.md` next to their `SKILL.md`. Those models follow a shared structure (assets, adversaries, trust buckets with per-bucket STRIDE mitigations, and an Enterprise Readiness Gaps register) and are the authoritative source for each skill's residual risk. -| Skill | Runtime surface | Primary residual gaps | Security model | -|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| -| jira | REST CLI; environment credentials | No token revocation; best-effort redaction; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/jira/SECURITY.md) | -| gitlab | REST CLI; environment credentials; git-remote subprocess | Untrusted CI-trace egress; insecure-transport opt-out; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/gitlab/SECURITY.md) | -| mural (experimental) | REST CLI; embedded stdio MCP server; OAuth token store | OAuth audit gaps; keyring backend toggle is code-execution surface | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/mural/SECURITY.md) | -| tts-voiceover (experimental) | Azure Speech egress; key/Entra credentials; SSML + PPTX parsing | Content egress to Azure region; broad credential chain | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/tts-voiceover/SECURITY.md) | -| accessibility | Arbitrary-URL scan egress; version-pinned `npx @axe-core/cli@4.12.1` subprocess; design-intent verification adapter | Runtime integrity is best-effort without lockfile/integrity hash; no egress allow-list (SSRF); headless-browser surface; consuming-project validation boundary is repository-scoped | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/accessibility/accessibility/SECURITY.md) | -| powerpoint (experimental) | Sandboxed `content-extra.py` execution; LibreOffice/MuPDF document parsing | Denylist confinement is not OS-level; external-parser CVE exposure | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/powerpoint/SECURITY.md) | -| video-to-gif (experimental) | Local CLI (bash + PowerShell); FFmpeg/ffprobe subprocess; untrusted media parsing | Inherited FFmpeg decoder CVE exposure; bare-filename search resolution | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/video-to-gif/SECURITY.md) | -| copilot-otel-metrics (experimental) | Schema-enforced, backed-up, audited write into the user's global settings.json; loopback OTLP ingest through a fail-closed filtering Collector in front of a containerized Grafana/Prometheus/Tempo stack; five stdlib Python reference helpers plus a shared input-policy module querying local APIs; generated collector configuration, Bicep, Terraform, and Azure CLI templates the operator deploys | Prompt content present in spans before the Collector filters it and in plaintext across the loopback hop; content carriers the Collector governs incompletely, namely span names and metric metadata preserved for shipped dashboard queries, and span links and metric exemplars unreachable by any processor in this distribution, and the instrumentation scope and schema fields left unfiltered on an assumption about emitter behavior; shared fleet-wide ingest credential with no per-user binding or in-place rotation, delivered to the extension exporter but not the agent host, whose telemetry a per-workstation loopback relay carries at the cost of an unauthenticated local listener; unauthenticated Prometheus and Tempo query APIs; exporter-side certificate validation outside skill control; retention is the only deletion mechanism; the no-execution boundary on Docker and infrastructure commands is advisory rather than enforced | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SECURITY.md) | -| gh-code-scanning | GitHub code-scanning read via `gh` CLI subprocess; stdout only | Unpinned `gh`/`jq` PATH dependencies; TLS delegated to `gh` | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/gh-code-scanning/SECURITY.md) | -| customer-card-render (experimental) | Local Python CLI; regex parse of untrusted DT markdown; YAML emission | Inherited powerpoint build toolchain; confidential DT prose egress | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/customer-card-render/SECURITY.md) | -| security-planning | Local Python generator, native TMT validation harness, Windows UI Automation, screenshot capture, and overlay/evidence handling | Screenshot evidence is never redacted; UI Automation drives a real desktop session; overlays stay pending until a human promotes them | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/security-planning/SECURITY.md) | -| vex | Local Python gate (`vex_gate.py`); anchored-regex parse of untrusted detection-issue body; `json.loads` of local OpenVEX doc; exit code only | Gate-suppression by issue-edit access; forced-proceed AI-credit consumption | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/vex/SECURITY.md) | +| Skill | Runtime surface | Primary residual gaps | Security model | +|-------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| jira | REST CLI; environment credentials | No token revocation; best-effort redaction; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/jira/SECURITY.md) | +| gitlab | REST CLI; environment credentials; git-remote subprocess | Untrusted CI-trace egress; insecure-transport opt-out; no cert pinning | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/gitlab/SECURITY.md) | +| mural (experimental) | REST CLI; embedded stdio MCP server; OAuth token store | OAuth audit gaps; keyring backend toggle is code-execution surface | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/mural/SECURITY.md) | +| tts-voiceover (experimental) | Azure Speech egress; key/Entra credentials; SSML + PPTX parsing | Content egress to Azure region; broad credential chain | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/tts-voiceover/SECURITY.md) | +| accessibility | Arbitrary-URL scan egress; version-pinned `npx @axe-core/cli@4.12.1` subprocess; design-intent verification adapter | Runtime integrity is best-effort without lockfile/integrity hash; no egress allow-list (SSRF); headless-browser surface; consuming-project validation boundary is repository-scoped | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/accessibility/accessibility/SECURITY.md) | +| powerpoint (experimental) | Sandboxed `content-extra.py` execution; LibreOffice/MuPDF document parsing | Denylist confinement is not OS-level; external-parser CVE exposure | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/powerpoint/SECURITY.md) | +| video-to-gif (experimental) | Local CLI (bash + PowerShell); FFmpeg/ffprobe subprocess; untrusted media parsing | Inherited FFmpeg decoder CVE exposure; bare-filename search resolution | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/video-to-gif/SECURITY.md) | +| copilot-otel-metrics (experimental) | Schema-enforced, backed-up, audited, atomically replaced write into the user's global settings.json; loopback OTLP ingest through a fail-closed filtering Collector in front of a containerized Grafana/Prometheus/Tempo stack; five stdlib Python reference helpers plus a shared input-policy module querying local APIs; generated fleet collector configuration, a per-workstation relay, Bicep, Terraform, and Azure CLI templates the operator deploys | Prompt content present in spans before the Collector filters it and in plaintext across the loopback hop; content carriers the Collector governs incompletely, namely span names and metric metadata preserved for shipped dashboard queries, and span links and metric exemplars unreachable by any processor in this distribution, and the instrumentation scope and schema fields left unfiltered on an assumption about emitter behavior; the content scrub is fail-open per statement while its sibling allow-list is fail-closed; shared fleet-wide ingest credential with no per-user binding or in-place rotation, held in the workstation relay's runtime environment rather than the editor's, at the cost of an unauthenticated local listener and of making relay health a prerequisite for all of that workstation's telemetry; unauthenticated Prometheus and Tempo query APIs; exporter-side certificate validation outside skill control; retention is the only deletion mechanism; the no-execution boundary on Docker and infrastructure commands is advisory rather than enforced | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/copilot-otel-metrics/SECURITY.md) | +| gh-code-scanning | GitHub code-scanning read via `gh` CLI subprocess; stdout only | Unpinned `gh`/`jq` PATH dependencies; TLS delegated to `gh` | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/gh-code-scanning/SECURITY.md) | +| customer-card-render (experimental) | Local Python CLI; regex parse of untrusted DT markdown; YAML emission | Inherited powerpoint build toolchain; confidential DT prose egress | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/experimental/customer-card-render/SECURITY.md) | +| security-planning | Local Python generator, native TMT validation harness, Windows UI Automation, screenshot capture, and overlay/evidence handling | Screenshot evidence is never redacted; UI Automation drives a real desktop session; overlays stay pending until a human promotes them | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/project-planning/security-planning/SECURITY.md) | +| vex | Local Python gate (`vex_gate.py`); anchored-regex parse of untrusted detection-issue body; `json.loads` of local OpenVEX doc; exit code only | Gate-suppression by issue-edit access; forced-proceed AI-credit consumption | [SECURITY.md](https://github.com/microsoft/hve-core/blob/main/.github/skills/security/vex/SECURITY.md) | Skills whose scripts perform only local validation with no external surface (for example `adr-author` and `vally-tests`) do not require a dedicated model; their risk is bounded by the repository-level controls. When a new skill adds an executable runtime with any of the surfaces above, add a `SECURITY.md` following the shared structure and register it in this table and in the [security documentation index](README.md#skill-security-models). From 78b218c4caf6b919f77a379479c4990aa51d65a3 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Wed, 19 Aug 2026 12:27:42 -0700 Subject: [PATCH 14/28] docs(skills): replace narrative code comments with behavior and invariants Multi-paragraph rationale essays in the Collector configurations, Compose files, and Python helpers duplicated material already owned by SECURITY.md and the references pages. Comments now state behavior, intent, invariants, and edge cases, and point to the documents that own the rationale. Also removes revision-history narration from module and function docstrings, which described how earlier versions of the skill behaved rather than the current contract. --- .../examples/_input_policy.py | 8 +- .../azure/agent-host-relay/compose.yaml | 52 ++++------ .../otel-collector-config.yaml | 58 +++++------ .../examples/azure/otel-collector-config.yaml | 94 +++++++----------- .../examples/compose.yaml | 69 ++++---------- .../examples/otel-collector-local.yaml | 95 ++++++------------- .../examples/settings_upsert.py | 58 +++++------ .../tests/test_collector_carriers.py | 21 ++-- 8 files changed, 161 insertions(+), 294 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py index 83872a3ae..989c2484f 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py @@ -4,12 +4,8 @@ """Shared input policy for the bundled helper scripts. Every helper takes a configurable endpoint or path from the environment or the -command line. Each one previously guarded itself, or did not, and the guards -disagreed: the dashboard helper checked a hostname while the metric and trace -endpoints were unchecked, and no helper rechecked the target after a redirect. - -This module is the single place those decisions are made, so a helper cannot be -hardened by accident in one place and left open in another. +command line. This module is the single place those decisions are made, so a +helper cannot be hardened in one place and left open in another. Nothing here makes an arbitrary PromQL or TraceQL query safe. It constrains where a request may go and where a file may be written, not what is asked for. diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml index b52bdb12a..59788e927 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/compose.yaml @@ -1,29 +1,20 @@ # Per-workstation agent-host relay for organization-managed Copilot telemetry. # -# Deploy this on every workstation that reports to the fleet endpoint, and -# verify it is healthy, before switching managed settings to the loopback -# endpoint. After that switch this relay carries all of that workstation's -# Copilot telemetry: if it is stopped or missing, telemetry is not queued -# elsewhere, it is lost. +# Deploy and verify this on a workstation before switching its managed settings +# to the loopback endpoint. After that switch the relay carries all of that +# workstation's Copilot telemetry: if it is stopped or missing, telemetry is +# lost rather than queued. # -# Start it independently of VS Code so it does not inherit or share the -# editor's environment, and so `restart: unless-stopped` returns it after a -# reboot: -# docker compose --env-file /absolute/path/copilot-otel-relay.env \ -# -f /absolute/path/agent-host-relay/compose.yaml up -d +# Start it independently of VS Code so it does not inherit the editor's +# environment. Runtime inputs come from an env file outside this repository +# that is readable only by the operator account: +# COPILOT_OTEL_FLEET_ENDPOINT, COPILOT_OTEL_INGEST_TOKEN, +# COPILOT_OTEL_FLEET_CA_BUNDLE # -# The env file lives outside this repository, is readable only by the operator -# account, and supplies: -# COPILOT_OTEL_FLEET_ENDPOINT=https://otel.example.internal -# COPILOT_OTEL_INGEST_TOKEN= -# COPILOT_OTEL_FLEET_CA_BUNDLE=/absolute/path/fleet-ca.pem +# The image is pinned by manifest digest, matching the local stack. # -# Rotate the token by editing that file and running the same command with -# --force-recreate. Delete the file when decommissioning the relay; removing -# the container does not remove it. -# -# The image is pinned by manifest digest rather than tag, matching the local -# stack, so a mutable tag cannot change this relay underneath a review. +# Exact commands, rotation, and decommissioning: ../README.md and +# ../../../references/org-distribution.md. services: otel-relay: @@ -33,21 +24,18 @@ services: restart: unless-stopped command: ["--config=/etc/otelcol-contrib/config.yaml"] ports: - # Loopback only. The relay is reachable from this workstation and from - # nothing on the network; the receiver inside the container binds the - # container interface, and this mapping is what decides host exposure. + # Loopback only. The receiver binds the container interface; this mapping + # decides host exposure. - "127.0.0.1:4318:4318" # OTLP HTTP <- both Copilot emitters export here - "127.0.0.1:13133:13133" # health, for the post-deployment check environment: - # Required, with no default. The :? form fails the up with this message - # rather than starting a relay that would drop everything it receives. + # Required with no default; the :? form fails the up with this message. COPILOT_OTEL_FLEET_ENDPOINT: ${COPILOT_OTEL_FLEET_ENDPOINT:?set COPILOT_OTEL_FLEET_ENDPOINT in the relay env file} COPILOT_OTEL_INGEST_TOKEN: ${COPILOT_OTEL_INGEST_TOKEN:?set COPILOT_OTEL_INGEST_TOKEN in the relay env file} volumes: - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro - # The operator's CA bundle, read-only. An absolute host path is required - # because a relative one would resolve against whichever directory the - # up was run from. + # The operator's CA bundle, read-only. An absolute host path is required; + # a relative one would resolve against the directory the up was run from. - ${COPILOT_OTEL_FLEET_CA_BUNDLE:?set COPILOT_OTEL_FLEET_CA_BUNDLE to the absolute path of the fleet CA bundle}:/run/copilot-otel/fleet-ca.pem:ro networks: - copilot-otel-relay @@ -57,11 +45,11 @@ services: security_opt: - no-new-privileges:true # Bounds the container so the cgroup limit and the Collector's own - # memory_limiter agree rather than contradicting each other. + # memory_limiter agree. mem_limit: 512m networks: - # Explicit and dedicated, so this relay shares no network with the local - # development stack or anything else running on the workstation. + # Dedicated, so this relay shares no network with the local development + # stack. copilot-otel-relay: driver: bridge diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml index 8fb27fe67..c6ae3c5e1 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/agent-host-relay/otel-collector-config.yaml @@ -1,20 +1,15 @@ # OpenTelemetry Collector configuration for the per-workstation agent-host relay. # # Managed VS Code settings apply one telemetry endpoint to both the extension -# and the agent host, and managed `telemetry.headers` reach only the extension. -# Sending both to this relay removes that split: neither emitter carries the -# fleet credential, this relay adds it on the upstream hop, and the credential -# never enters the environment VS Code hands to agent-spawned subprocesses. +# and the agent host, and managed telemetry.headers reach only the extension. +# Both emitters send here without a credential; this relay adds the fleet token +# on the upstream hop, so it never enters the environment VS Code hands to +# agent-spawned subprocesses. # -# Ingress is OTLP/HTTP only. Host exposure is decided by the loopback port -# mapping in compose.yaml, not here, so the receiver binds the container -# interface exactly as the local stack's Collector does. There is deliberately -# no gRPC listener: managed settings pin `otlp-http`, and a second protocol -# would be an unexercised ingress. +# Ingress is OTLP/HTTP only, matching the protocol managed settings pin. Host +# exposure is decided by the loopback port mapping in compose.yaml, not here. # -# Every runtime input arrives through the environment or a read-only mount. -# Nothing in this file is a credential, and nothing in this file is a path -# inside the repository: +# Every runtime input arrives through the environment or a read-only mount: # COPILOT_OTEL_FLEET_ENDPOINT https:// base URL of the fleet receiver # COPILOT_OTEL_INGEST_TOKEN the fleet ingest token # COPILOT_OTEL_FLEET_CA_BUNDLE absolute host path to the CA bundle that @@ -23,6 +18,9 @@ # # Requires the OpenTelemetry Collector Contrib distribution; the redaction # processor is not in the core distribution. +# +# Rollout, rotation, and rollback: ../README.md and +# ../../../references/org-distribution.md. receivers: otlp: @@ -38,9 +36,8 @@ processors: spike_limit_percentage: 20 # The same fail-closed policy the local stack and the fleet Collector apply. - # Filtering here as well as upstream is deliberate: content is removed on the - # workstation that produced it, before it crosses the network, so a fleet - # receiver misconfiguration cannot turn this hop into the leak. + # Filtering here as well as upstream removes content on the workstation that + # produced it, before it crosses the network. redaction: allow_all_keys: false allowed_keys: @@ -98,16 +95,14 @@ processors: # keys it removed, which would reintroduce the drift it exists to stop. summary: silent - # Everything the redaction processor cannot reach. redaction traverses + # Carriers the redaction processor cannot reach. redaction traverses # attribute maps only, so span status messages, span event names, trace - # state, and every non-map log body shape would otherwise leave the - # workstation verbatim. + # state, and non-map log bodies are overwritten here. # - # Span names, metric names, descriptions, and units are absent because - # shipped dashboard queries and baseline.py read them; span link attributes, - # span link trace state, and metric exemplar filtered attributes are absent - # because this distribution provides no way to reach them. Both kinds of - # residual are recorded in SECURITY.md. + # Invariant: a carrier the allow-list does not reach is scrubbed unless a + # shipped consumer reads it. Span names and metric metadata are therefore + # absent, as are carriers this distribution cannot address. Both residual + # kinds are registered in SECURITY.md. transform/scrub: error_mode: ignore trace_statements: @@ -124,12 +119,10 @@ processors: send_batch_size: 1024 exporters: - # The upstream hop is the only place the fleet credential exists on this - # workstation. TLS verification is explicit: `ca_file` names the operator's - # bundle and there is no `insecure` flag, so a receiver presenting an - # untrusted certificate or a mismatched hostname is refused rather than - # trusted. The bundle is mounted from a host path the operator chooses; this - # skill ships no certificate authority. + # The only place the fleet credential exists on this workstation. TLS + # verification is explicit: ca_file names the operator's bundle and there is + # no insecure flag, so an untrusted certificate or mismatched hostname is + # refused. This skill ships no certificate authority. otlp_http/fleet: endpoint: ${env:COPILOT_OTEL_FLEET_ENDPOINT} auth: @@ -141,10 +134,9 @@ extensions: health_check: endpoint: 0.0.0.0:13133 - # The fleet ingest token. It authenticates the fleet rather than a person and - # is identical on every workstation, so holding it here narrows where it is - # readable without binding it to anyone. Any process running as the same - # operating-system user can still read the environment file that supplies it. + # The fleet ingest token, identical on every workstation, so it authenticates + # the fleet rather than a person. Residual: any process running as the same + # operating-system user can read the environment file that supplies it. bearertokenauth: scheme: Bearer token: ${env:COPILOT_OTEL_INGEST_TOKEN} diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml index 13b219924..8ecc0dc9b 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/azure/otel-collector-config.yaml @@ -17,14 +17,10 @@ receivers: otlp: protocols: - # Authentication and TLS are active, not suggested. An unauthenticated - # plaintext receiver reachable by a fleet accepts spans from anything that - # can route to it, and every span it accepts is billed and stored. - # - # If TLS is terminated by an ingress controller or load balancer in front - # of this collector, delete the tls blocks below and record that the - # terminating hop owns certificate lifecycle and cipher policy. Do not - # delete them merely because supplying certificates is inconvenient. + # Authentication and TLS are required on both protocols. If TLS is + # terminated by an ingress controller in front of this collector, delete + # the tls blocks and record that the terminating hop owns certificate + # lifecycle and cipher policy. http: endpoint: 0.0.0.0:4318 auth: @@ -47,14 +43,10 @@ processors: limit_percentage: 80 spike_limit_percentage: 20 - # The fleet minimization control. It is the local stack's policy verbatim, - # and tests/test_azure_templates.py asserts that parity: a shared, retained - # workspace is a stricter destination than one workstation, so this - # configuration may not filter less than otel-collector-local.yaml. - # - # allow_all_keys: false makes allowed_keys authoritative. An attribute a - # future emitter release introduces is dropped rather than stored, which a - # delete-list cannot do. Adding a key here stores that attribute fleet-wide. + # Fleet minimization. This is otel-collector-local.yaml's policy verbatim, + # and tests/test_azure_templates.py asserts that parity: this configuration + # may not filter less than the local stack. allow_all_keys: false makes + # allowed_keys authoritative, so adding a key here stores it fleet-wide. redaction: allow_all_keys: false allowed_keys: @@ -77,8 +69,7 @@ processors: # Prometheus stores this as the gen_ai_token_type label; without it, # input and output tokens collapse into one indistinguishable series. - gen_ai.token.type - # The discriminator baseline.py reads through the session_id label. It - # is a detection control, not a convenience. + # The discriminator baseline.py reads through the session_id label. - session.id # Agent and tool identity. Names only; arguments and results are not # allowed and therefore cannot reach storage. @@ -89,9 +80,8 @@ processors: - copilot_chat.turn_count - copilot_chat.copilot_usage_nano_aiu - github.copilot.agent.type - # Both spellings collapse to the copilot_chat_edit_source label and - # normalization is not reversible, so allow-listing both keeps a wrong - # single guess from silently emptying the edit-acceptance panel. + # Both spellings normalize to the copilot_chat_edit_source label and + # normalization is not reversible, so both are allowed. - copilot_chat.edit_source - copilot_chat.edit.source # Failure triage without failure content. @@ -112,17 +102,14 @@ processors: # keys it removed, which would reintroduce the drift it exists to stop. summary: silent - # Everything the redaction processor cannot reach. redaction traverses + # Carriers the redaction processor cannot reach. redaction traverses # attribute maps only, so span status messages, span event names, trace - # state, and every non-map log body shape would otherwise reach the exporter - # verbatim. + # state, and non-map log bodies are overwritten here. # - # A carrier the allow-list does not reach is scrubbed unless a shipped - # consumer reads it. Span names, metric names, descriptions, and units are - # absent because dashboard queries and baseline.py read them; span link - # attributes, span link trace state, and metric exemplar filtered attributes - # are absent because this distribution provides no way to reach them. Both - # kinds of residual are recorded in SECURITY.md. + # Invariant: a carrier the allow-list does not reach is scrubbed unless a + # shipped consumer reads it. Span names and metric metadata are therefore + # absent, as are carriers this distribution cannot address. Both residual + # kinds are registered in SECURITY.md. transform/scrub: error_mode: ignore trace_statements: @@ -137,11 +124,9 @@ processors: # Fleet dimensions for grouping. Keep these to org structure; anything that # identifies an individual lands on every span they produce. # - # These are labels, not boundaries. An attribute set by the collector is a - # grouping key that anyone with read access to this workspace can filter on; - # it does not stop them reading any other environment's data. Separation comes - # from deploying one collector and one workspace per environment, which is why - # the environment name is a required input rather than a default. + # These are grouping keys, not boundaries. Separation comes from deploying + # one collector and one workspace per environment, which is why the + # environment name is a required input rather than a default. resource/fleet: attributes: - key: service.namespace @@ -163,34 +148,21 @@ extensions: health_check: endpoint: 0.0.0.0:13133 - # Copilot can only send a fixed header set, so a static bearer token is the - # available fleet authentication mechanism, and it is active above. Be clear - # about what it is and is not: every workstation holds the same value, so it - # authenticates the fleet rather than a person, it cannot be rotated for one - # user, and anyone who extracts it from a workstation can write spans as the - # fleet. It is write-side only, and it is still worth having, because without - # it the receiver accepts spans from anything that can reach it. - # - # This authentication is mandatory, and that has a consequence worth stating - # here rather than only in the README. Managed telemetry.headers reach the - # extension exporter but deliberately not the agent host, because delivering - # them through the environment would put this token inside every tool - # subprocess the agent spawns. So the agent host exports without a - # credential, this receiver answers 401 or UNAUTHENTICATED, and OTLP treats - # neither as retryable: that telemetry is dropped, not queued. + # Copilot sends a fixed header set, so a static bearer token is the available + # fleet authentication mechanism. Every workstation holds the same value: it + # authenticates the fleet rather than a person, cannot be rotated for one + # user, and is write-side only. # - # Do not resolve that by exporting the token into the environment, and do not - # resolve it by removing this authenticator. Run a loopback relay on each - # workstation that accepts the unauthenticated local export and adds this - # credential upstream. See README.md, "Agent-host telemetry needs a relay". + # Because this authenticator is mandatory and managed telemetry.headers do + # not reach the agent host, an agent host exporting directly receives 401 or + # UNAUTHENTICATED and OTLP drops that telemetry rather than queuing it. Run + # the loopback relay in agent-host-relay/ instead of exporting the token into + # the environment or removing this authenticator. See README.md. # - # Mutual TLS would authenticate each workstation rather than the fleet, and - # this receiver can already require it: adding client_ca_file beside - # cert_file and key_file in the otlp receiver selects - # RequireAndVerifyClientCert on both protocols. It is not enabled because - # whether the VS Code agent host can present a client certificate is - # unestablished, and configuring a receiver to require one the sender cannot - # supply would replace a documented gap with fleet-wide data loss. + # Mutual TLS would authenticate each workstation instead. Adding + # client_ca_file beside cert_file and key_file selects + # RequireAndVerifyClientCert; it is not enabled because whether the agent + # host can present a client certificate is unestablished. See SECURITY.md. bearertokenauth: scheme: Bearer token: ${env:COPILOT_OTEL_INGEST_TOKEN} diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml index 6a54f0291..69c3df928 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml @@ -1,34 +1,16 @@ # Local Copilot OTel stack: a filtering Collector in front of Grafana, # Prometheus, Tempo, Loki, and Pyroscope. # -# Copilot exports to the Collector, never to LGTM directly. The Collector -# applies the fail-closed attribute allow-list in otel-collector-local.yaml and -# is the only OTLP listener published on the host, so nothing reaches storage -# without passing that policy. LGTM's own OTLP ports are deliberately not -# mapped: publishing them would let any local process bypass the filter, which -# loopback binding alone does not prevent. +# Copilot exports to the Collector, never to LGTM directly. The Collector is +# the only OTLP listener published on the host; LGTM's OTLP ports are +# deliberately unmapped so no local process can bypass the filter. # -# Both images are pinned by manifest digest rather than tag. A tag is mutable, -# so a tag-pinned stack can change underneath a reviewed configuration. +# Images are pinned by manifest digest. Grafana credentials are required with +# no default, and the data volume is external so this file binds the existing +# copilot-otel-data volume rather than creating a project-prefixed duplicate. # -# Grafana credentials are required, not defaulted. Set them before the first -# up; the stack refuses to start otherwise. Read the password rather than -# typing it inline, so it does not land in the terminal scrollback or the -# shell history file: -# export COPILOT_OTEL_GRAFANA_USER=admin -# read -rs -p 'Grafana password: ' COPILOT_OTEL_GRAFANA_PASSWORD; echo -# export COPILOT_OTEL_GRAFANA_PASSWORD -# -# In PowerShell, use Read-Host -AsSecureString for the same reason. A .env file -# reaches these containers but not the bundled Python helpers, which read the -# same two variables from the environment of the shell that runs them. -# -# The data volume is declared external so this file binds the existing -# copilot-otel-data volume rather than creating a project-prefixed duplicate, -# which would silently orphan the accumulated history. -# -# Create it once before the first up: -# docker volume create copilot-otel-data +# Prerequisites, credential handling, and first-run steps: +# references/local-setup.md. services: otel-collector: @@ -46,9 +28,7 @@ services: networks: - copilot-otel # memory_limiter resolves its percentages against the container's cgroup - # limit when one exists and against total host memory when one does not. - # Without this line its stated bound is 80% of the workstation's RAM, in - # front of an ingress any local process can reach. + # limit when one exists and against total host memory when it does not. mem_limit: 512m read_only: true cap_drop: @@ -75,32 +55,25 @@ services: - copilot-otel environment: # This image enables anonymous Grafana access with the Admin role and - # hides the login form. Left alone, every panel and every stored prompt - # fragment is readable by anything that can reach port 3000 without - # presenting a credential. All three flags are needed: disabling - # anonymous access without restoring the login form leaves no way in. + # hides the login form. All three flags are required: disabling anonymous + # access without restoring the login form leaves no way in. GF_AUTH_ANONYMOUS_ENABLED: "false" GF_AUTH_DISABLE_LOGIN_FORM: "false" GF_AUTH_BASIC_ENABLED: "true" - # Required, with no default. The :? form fails the up with this message - # rather than silently starting on a guessable credential, and it is the - # same pair validate_dashboard.py reads. + # Required with no default; the :? form fails the up with this message. + # validate_dashboard.py reads the same pair. GF_SECURITY_ADMIN_USER: ${COPILOT_OTEL_GRAFANA_USER:?set COPILOT_OTEL_GRAFANA_USER before starting the stack} GF_SECURITY_ADMIN_PASSWORD: ${COPILOT_OTEL_GRAFANA_PASSWORD:?set COPILOT_OTEL_GRAFANA_PASSWORD before starting the stack} - # Two flags, space separated, appended to the Prometheus command line. - # - # otlp-deltatocumulative is applied defensively. Prometheus drops - # delta-temporality metrics by default, and a dropped delta metric was - # observed to fail the whole batched write, taking co-batched cumulative - # metrics with it. This converts instead of dropping, and is inert for - # cumulative-only traffic. Its per-series state resets on restart. - # + # Space-separated flags appended to the Prometheus command line. + # otlp-deltatocumulative converts delta-temporality metrics that + # Prometheus would otherwise drop, failing the whole batched write; it is + # inert for cumulative-only traffic and its state resets on restart. # Retention is raised from the 15d default so per-month token totals are - # real rather than silently truncated at 15 days. + # not silently truncated. PROMETHEUS_EXTRA_ARGS: "--enable-feature=otlp-deltatocumulative --storage.tsdb.retention.time=120d" volumes: # Everything persistent lives under /data: grafana, prometheus, tempo, - # loki, pyroscope. One mount covers all five. + # loki, pyroscope. - copilot-otel-data:/data healthcheck: test: @@ -114,8 +87,8 @@ services: start_period: 30s networks: - # Explicit rather than implicit so the LGTM OTLP listener is reachable from - # the Collector and from nothing else. + # Explicit so the LGTM OTLP listener is reachable from the Collector and from + # nothing else. copilot-otel: driver: bridge diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml index b4f042921..3f81fb099 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/otel-collector-local.yaml @@ -1,17 +1,13 @@ # OpenTelemetry Collector configuration for the local Copilot stack. # -# This Collector is the only OTLP listener published on the host. It sits in -# front of the LGTM container, which has no host OTLP mapping, so nothing -# reaches storage without passing through the policy declared here. -# -# The policy is fail-closed: allowed_keys is an allow-list, so an attribute -# this configuration has never heard of is dropped rather than stored. That -# matters because the emitter's attribute set is not a stable contract. A -# future extension release can add a new prompt-bearing attribute, and a -# delete-list would silently pass it through while this allow-list will not. +# The only OTLP listener published on the host. LGTM has no host OTLP mapping, +# so nothing reaches storage without passing the policy declared here. # # Requires the OpenTelemetry Collector Contrib distribution; the redaction # processor is not in the core distribution. +# +# Policy rationale, carrier coverage, and residual gaps: SECURITY.md and +# references/local-stack.md. receivers: otlp: @@ -30,17 +26,10 @@ processors: limit_percentage: 80 spike_limit_percentage: 20 - # The entire minimization control for the local stack. - # - # allow_all_keys: false makes allowed_keys authoritative. Every key below - # exists because a shipped dashboard panel or a datasource lookup reads it. - # Adding a key here is a deliberate decision to store that attribute; the - # content attributes that were observed carrying plaintext are absent by - # construction rather than by enumeration. - # - # blocked_values is a second, independent pass over the values that survive. - # An allowed key can still carry a credential pasted into a model name or a - # tool argument, so recognizable secret shapes are masked rather than stored. + # Fail-closed attribute minimization. allow_all_keys: false makes + # allowed_keys authoritative, so an unlisted attribute is dropped rather + # than stored. blocked_values is a second pass that masks credential shapes + # in the values that survive. redaction: allow_all_keys: false allowed_keys: @@ -64,9 +53,8 @@ processors: # gen_ai_token_type label; dropping it collapsed input and output tokens # into one indistinguishable series. - gen_ai.token.type - # Read by baseline.py through the session_id label. It is the - # discriminator that distinguishes injected series from real ones, so - # dropping it disabled a detection control the threat model credits. + # Read by baseline.py through the session_id label; it discriminates + # injected series from real ones. - session.id # Agent and tool identity. Names only; arguments and results are not # allowed and therefore cannot reach storage. @@ -77,19 +65,16 @@ processors: - copilot_chat.turn_count - copilot_chat.copilot_usage_nano_aiu - github.copilot.agent.type - # The edit-acceptance panel groups by the copilot_chat_edit_source label. - # Which OTLP key produces that label is not established: normalization to - # a Prometheus label is not reversible, and both spellings below collapse - # to it. Allow-listing both means a wrong single guess cannot silently - # empty the panel. Settling this needs live emitter output, and the gap - # is recorded in SECURITY.md rather than guessed away here. + # Both spellings normalize to the copilot_chat_edit_source label and + # normalization is not reversible, so both are allowed. Unresolved gap: + # SECURITY.md. - copilot_chat.edit_source - copilot_chat.edit.source # Failure triage without failure content. - error.type - # Masked, not dropped, so a panel keeps its shape while the value is gone. - # These are recognizable shapes, not secret detection: a credential with no - # distinctive prefix passes, and that limit is recorded in SECURITY.md. + # Masked rather than dropped, so a panel keeps its shape. Recognizable + # shapes only; a credential with no distinctive prefix passes. See + # SECURITY.md. blocked_values: - "gh[pousr]_[0-9A-Za-z]{16,}" - "github_pat_[0-9A-Za-z_]{20,}" @@ -110,35 +95,18 @@ processors: # keys it removed, which would reintroduce the drift it exists to stop. summary: silent - # Everything the redaction processor cannot reach. - # - # redaction traverses attribute maps. Running the pinned Collector against - # payloads that placed a distinct marker in every OTLP carrier showed which - # carriers that leaves open: span status messages, span event names, trace - # state, and every log body shape except a map all reached the exporter - # verbatim. An allow-list that never sees a field cannot fail closed on it, - # so those carriers are overwritten here. + # Carriers the redaction processor cannot reach. redaction traverses + # attribute maps only, so span status messages, span event names, trace + # state, and non-map log bodies are overwritten here. # - # The rule is: a carrier the allow-list does not reach is scrubbed unless a - # shipped consumer reads it. That is why span names are absent from this - # list even though they are the carrier most likely to hold prompt text. - # Three dashboard queries match on span name and baseline.py reads Tempo's - # rootTraceName, so scrubbing them would break shipped functionality. Metric - # names, descriptions, and units are absent for the same reason. Those - # residual exposures are recorded in SECURITY.md rather than hidden here. + # Invariant: a carrier the allow-list does not reach is scrubbed unless a + # shipped consumer reads it. Span names and metric metadata are therefore + # absent, as are carriers this distribution cannot address. Both residual + # kinds are registered in SECURITY.md. # - # Span link attributes, span link trace state, and metric exemplar filtered - # attributes are also open and are also absent, but for a different reason: - # this distribution provides no way to reach them. OTTL has no `spanlink` - # context and refuses to index `links`, and assigning to `datapoint.exemplars` - # parses but does nothing. Both are recorded as gaps. - # `error_mode: ignore` is deliberate and load-bearing. A statement that - # errors on one record shape is skipped for that record, which is why a - # map-valued log body keeps the allow-listed subset rather than being - # replaced wholesale. It also makes this processor fail-open per statement, - # unlike the allow-list above, which is fail-closed. The digest pins and the - # runtime carrier harness are what bound that; the residual is registered in - # SECURITY.md rather than left implicit. + # error_mode: ignore is deliberate and fail-open per statement: a statement + # that errors on one record shape is skipped for that record, which is what + # lets a map-valued log body keep its allow-listed subset. transform/scrub: error_mode: ignore trace_statements: @@ -155,12 +123,9 @@ processors: send_batch_size: 1024 exporters: - # The LGTM container is reachable only on the internal Compose network. TLS - # is not used on that hop and the insecure flag says so explicitly rather - # than leaving it to a default. - # - # `otlp_grpc` rather than `otlp`: the pinned Collector logs a deprecation - # warning for the `otlp` exporter alias once per signal at every start. + # Reachable only on the internal Compose network; TLS is not used on that hop + # and insecure states so explicitly rather than relying on a default. + # otlp_grpc rather than otlp: the pinned Collector deprecates the otlp alias. otlp_grpc/lgtm: endpoint: lgtm:4317 tls: diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py index ad8705092..a98e8c370 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py @@ -3,12 +3,10 @@ # SPDX-License-Identifier: MIT """Reversible, audited upsert of Copilot OTel settings into a JSONC settings file. -The skill's written rules for this edit were strong and entirely advisory: an -agent could describe a careful diff-and-confirm write and then perform any edit -at all. This makes the rules executable. Every key, type, and value combination -is checked against a schema before a byte is written, non-target bytes are -preserved exactly, the previous file is backed up, and a failure to parse the -result restores the backup. +Every key, type, and value combination is checked against a schema before a +byte is written. Non-target bytes are preserved exactly, the previous file is +backed up, the write is staged and replaced atomically, and a result that fails +to parse restores the backup. settings_upsert.py --settings --set key=value [--set ...] # dry run settings_upsert.py --settings --set key=value --apply # write @@ -34,21 +32,19 @@ LOGGER = logging.getLogger("settings_upsert") -# Exit codes. 2 separates "you asked for something that cannot be done" from a -# crash, and 130 is the shell's convention for an interrupt. +# Exit codes. 2 marks a refused request, distinct from a crash; 130 is the +# shell convention for an interrupt. EXIT_OK = 0 EXIT_REFUSED = 2 EXIT_INTERRUPTED = 130 -# Backups accumulate on every apply. Five is enough to walk back a bad session -# and few enough that a settings directory does not fill with copies of a file -# that may name an output path. +# Backups accumulate on every apply. Bounded so a settings directory does not +# fill with copies of a file that may name an output path. BACKUP_RETENTION = 5 UTF8_BOM = "\ufeff" -# Provenance for the schema below. This is the authoritative manifest that was -# read, not a recollection of one. +# Provenance for the schema below. SCHEMA_SOURCE = { "artifact": "GitHub Copilot Chat extension manifest (package.json)", "extension": "GitHub.copilot-chat", @@ -57,11 +53,9 @@ "verified": "2026-08-13", } -# The OTel settings this build declares, with their declared types. Seven, not -# the eleven earlier revisions of this skill documented: protocol, headers, -# serviceName, and resourceAttributes are absent from this manifest. A key that -# is not here is refused rather than written, because writing a key the build -# does not declare produces a setting that is silently inert. +# The OTel settings this build declares, with their declared types. A key that +# is not here is refused rather than written: writing a key the build does not +# declare produces a setting that is silently inert. SCHEMA: dict[str, type | tuple[type, ...]] = { "github.copilot.chat.otel.enabled": bool, "github.copilot.chat.otel.exporterType": str, @@ -214,15 +208,12 @@ def check_policy( def strip_jsonc(text: str) -> str: """Return the text with comments and trailing commas blanked out. - Offsets are preserved so the result can be parsed for validation while the - spans found by the scanner still line up with the original bytes. Nothing - is removed; every blanked byte becomes a space in the same position. + Offsets are preserved: nothing is removed, and every blanked byte becomes a + space in the same position, so spans found by the scanner still line up + with the original bytes. - Trailing commas are handled here rather than by a regex over the whole - document. A regex cannot tell a comma inside a string literal from a - structural one, and the untouched-key comparison that guards this edit - could not catch the difference, because both sides of that comparison would - be produced by the same corrupting pass. + Trailing commas are handled by this scanner rather than a regex, which + cannot distinguish a comma inside a string literal from a structural one. """ out = list(text) index, length = 0, len(text) @@ -403,13 +394,9 @@ def backup_path_for( ) -> pathlib.Path: """Return a backup path that cannot overwrite an existing backup. - A fixed `.bak` name makes the second run destroy the only copy of the - original, which is the opposite of what a backup is for. The timestamp is - to the second, so repeated runs accumulate rather than overwrite. - - The counter is always present and zero-padded. An optional suffix would - sort `...Z-1.bak` before `...Z.bak`, and retention reads that order to - decide what to delete, so the newest backups would be the ones pruned. + The timestamp is to the second, so repeated runs accumulate. The counter is + always present and zero-padded: an optional suffix would sort `...Z-1.bak` + before `...Z.bak`, and retention reads name order to decide what to delete. """ stamp = (now or datetime.datetime.now(datetime.UTC)).strftime("%Y%m%dT%H%M%SZ") collision = 0 @@ -423,9 +410,8 @@ def backup_path_for( def existing_backups(settings_path: pathlib.Path) -> list[pathlib.Path]: """Backups this tool wrote for one settings file, oldest first. - Sorted by name rather than by modification time, because the name carries - the stamp this tool assigned and a copy operation does not preserve - ordering the way that stamp does. + Sorted by name, which carries the stamp this tool assigned; a copy + operation does not preserve modification-time ordering. """ return sorted(settings_path.parent.glob(f"{settings_path.name}.*.bak"), key=lambda p: p.name) diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py index e3be70cd0..5f202b7e3 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py @@ -2,19 +2,14 @@ # SPDX-License-Identifier: MIT """Runtime carrier map for the shipped local Collector configuration. -Every other test module in this suite parses committed files and simulates the -declared policy. That is how this skill came to ship control claims no test -could support: a static check confirmed the YAML said the right thing, and a -runtime claim was written on that basis. - -This module starts the pinned Collector, sends payloads that place a unique -marker in every OTLP carrier the signal model can transport, and records what -the shipped policy does to each one. The recorded map is the citable evidence -behind the minimization claims in `SECURITY.md` and the references, and it is -asserted as a regression boundary, so a configuration or image change that -opens a carrier fails here rather than passing quietly. - -Absence alone proves nothing. A marker can be missing because policy dropped it +Starts the pinned Collector, sends payloads that place a unique marker in every +OTLP carrier the signal model can transport, and records what the shipped +policy does to each one. The recorded map is the citable evidence behind the +minimization claims in `SECURITY.md` and the references, and it is asserted as +a regression boundary, so a configuration or image change that opens a carrier +fails here. + +Absence alone proves nothing: a marker can be missing because policy dropped it or because the exporter never renders that field. Every run is therefore paired with a negative control that removes the content-minimization processors from the same configuration. A marker the control does not render is `unobservable`, From e0d0d4385edb1d587e19167555381012ae9a7ad3 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Wed, 19 Aug 2026 15:40:49 -0700 Subject: [PATCH 15/28] fix(skills): stop reusing a pruned backup index within the same second backup_path_for filled the lowest free collision slot, so when retention deleted a low index and another apply landed inside the same second, the new backup reclaimed that index and sorted ahead of older backups. Retention reads name order, so it would again delete the newest copy. The counter now resumes after the highest index already used for that second. The regression test pins the interaction with a fixed timestamp rather than depending on execution speed. --- .../examples/settings_upsert.py | 22 ++++++++++++------ .../tests/test_settings_upsert.py | 23 +++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py index a98e8c370..ed3eec7e9 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py @@ -392,18 +392,26 @@ def write_audit(audit_path: pathlib.Path, record: dict[str, object]) -> None: def backup_path_for( settings_path: pathlib.Path, *, now: datetime.datetime | None = None ) -> pathlib.Path: - """Return a backup path that cannot overwrite an existing backup. + """Return a backup path that sorts after every existing backup. - The timestamp is to the second, so repeated runs accumulate. The counter is - always present and zero-padded: an optional suffix would sort `...Z-1.bak` - before `...Z.bak`, and retention reads name order to decide what to delete. + The timestamp is to the second and the counter is always present and + zero-padded, so name order is creation order. The counter resumes after the + highest index already used for this second rather than filling the lowest + free slot: retention can delete a low index, and reusing it would place a + newer backup ahead of an older one in the order retention reads. """ stamp = (now or datetime.datetime.now(datetime.UTC)).strftime("%Y%m%dT%H%M%SZ") - collision = 0 - candidate = settings_path.with_name(f"{settings_path.name}.{stamp}-{collision:03d}.bak") + prefix = f"{settings_path.name}.{stamp}-" + used = [] + for sibling in settings_path.parent.glob(f"{prefix}*.bak"): + index = sibling.name[len(prefix) : -len(".bak")] + if index.isdigit(): + used.append(int(index)) + collision = max(used) + 1 if used else 0 + candidate = settings_path.with_name(f"{prefix}{collision:03d}.bak") while candidate.exists(): collision += 1 - candidate = settings_path.with_name(f"{settings_path.name}.{stamp}-{collision:03d}.bak") + candidate = settings_path.with_name(f"{prefix}{collision:03d}.bak") return candidate diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py index 4a16bcef5..db21faf9e 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py @@ -8,6 +8,7 @@ from __future__ import annotations +import datetime import json import pathlib @@ -603,6 +604,28 @@ def test_a_later_backup_sorts_after_an_earlier_one(self, tmp_path: pathlib.Path) names.append(candidate.name) assert names == sorted(names) + def test_a_pruned_index_is_not_reused_within_the_same_second( + self, tmp_path: pathlib.Path + ) -> None: + """Retention frees a low index; reusing it would invert creation order. + + Every name here shares one timestamp, which is what happens when + several applies land inside the same second. + """ + settings = tmp_path / "settings.json" + settings.write_text("{}\n", encoding="utf-8") + stamp = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC) + first = backup_path_for(settings, now=stamp) + first.write_text("oldest", encoding="utf-8") + second = backup_path_for(settings, now=stamp) + second.write_text("newer", encoding="utf-8") + first.unlink() + + third = backup_path_for(settings, now=stamp) + + assert third.name != first.name + assert third.name > second.name + def test_the_backup_sits_beside_the_settings_file(self, tmp_path: pathlib.Path) -> None: settings = tmp_path / "settings.json" assert backup_path_for(settings).parent == tmp_path From cebb4ee020537c695438eed7616a6731e0c5d56b Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Thu, 20 Aug 2026 16:54:48 -0700 Subject: [PATCH 16/28] fix(skills): make the copilot-otel-metrics CI lanes pass on Linux The relay TLS cases could not read their own private key on the runner. OpenSSL writes keys 0600 for the creating user, the Collector image runs as 10001:10001, and the bind mount carries host mode through, so every case failed at receiver startup. They passed on Windows only because the WSL mount presents files as world-readable. Backup retention could also delete the newest backup. Name order is creation order only while the clock advances; the zero-padded counter orders within a second but nothing held the stamp when the wall clock stepped back, so the newest backup took the lowest name and was pruned first. The stamp is now held at the newest backup on disk, with a regression test driving a clock that moves backwards. --- .../examples/settings_upsert.py | 8 ++++++++ .../tests/test_collector_carriers.py | 5 +++++ .../tests/test_settings_upsert.py | 20 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py index ed3eec7e9..5fc19b768 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py @@ -399,8 +399,16 @@ def backup_path_for( highest index already used for this second rather than filling the lowest free slot: retention can delete a low index, and reusing it would place a newer backup ahead of an older one in the order retention reads. + + The counter only orders backups within one second, so a clock that steps + backwards would name the newest backup lowest and retention would delete it + first. A regressing stamp is held at the newest one on disk. """ stamp = (now or datetime.datetime.now(datetime.UTC)).strftime("%Y%m%dT%H%M%SZ") + existing = existing_backups(settings_path) + if existing: + newest = existing[-1].name[len(f"{settings_path.name}.") :].rsplit("-", 1)[0] + stamp = max(stamp, newest) prefix = f"{settings_path.name}.{stamp}-" used = [] for sibling in settings_path.parent.glob(f"{prefix}*.bak"): diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py index 5f202b7e3..3da86e506 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py @@ -1238,6 +1238,11 @@ def _issue_relay_material(runtime: ContainerRuntime, workdir: pathlib.Path) -> N if result.returncode != 0: raise RuntimeError(f"openssl {command[0]} failed: {result.stderr.strip()}") + # OpenSSL writes keys 0600 for the creating user; the Collector image runs + # as a different UID and the bind mount carries host mode through. + for key in workdir.glob("*.key"): + key.chmod(0o644) + def _fleet_receiver_config() -> dict: """A disposable stand-in for the fleet receiver: TLS, bearer auth, and a log.""" diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py index db21faf9e..b3076347d 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py @@ -655,6 +655,26 @@ def test_pruning_reports_what_it_removed(self, tmp_path: pathlib.Path) -> None: assert removed == created[:2] assert all(not path.exists() for path in removed) + def test_a_backwards_clock_still_sorts_the_newest_backup_last( + self, tmp_path: pathlib.Path + ) -> None: + """Retention reads name order, so a regressing clock must not reorder it. + + An NTP correction on a shared runner is enough to move the wall clock + back a second. Without this the newest backup takes the lowest name and + retention deletes it first, which loses the only copy that matters. + """ + settings = tmp_path / "settings.json" + settings.write_text("{}\n", encoding="utf-8") + later = datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.UTC) + earlier = later - datetime.timedelta(seconds=30) + + first = backup_path_for(settings, now=later) + first.write_text("{}\n", encoding="utf-8") + second = backup_path_for(settings, now=earlier) + + assert second.name > first.name + def test_pruning_keeps_the_newest_backup(self, settings_file: pathlib.Path) -> None: for index in range(BACKUP_RETENTION + 2): apply_changes(settings_file, {MAXSIZE: index + 1}, apply=True) From 05fb2305dcec96d17abc9627377e247ee0afa73f Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Thu, 20 Aug 2026 17:10:45 -0700 Subject: [PATCH 17/28] fix(skills): clear the CodeQL findings in copilot-otel-metrics Close the descriptor that silence_broken_pipe duplicates onto stdout (py/file-not-closed). Give resolve_runtime a single exit so no path falls off the end implicitly (py/mixed-returns). Patch resolve_addresses by name so _input_policy is imported one way only (py/import-and-import-from). --- .../examples/settings_upsert.py | 6 ++++- .../tests/test_collector_carriers.py | 22 +++++++++---------- .../tests/test_helpers.py | 3 +-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py index 5fc19b768..0d522383f 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py @@ -596,7 +596,11 @@ def silence_broken_pipe() -> None: flushing during shutdown, turning a pager quit into a second error message and a non-zero status. """ - os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + devnull = os.open(os.devnull, os.O_WRONLY) + try: + os.dup2(devnull, sys.stdout.fileno()) + finally: + os.close(devnull) def configure_logging(verbose: bool = False) -> None: diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py index 3da86e506..aefeba112 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py @@ -676,17 +676,17 @@ def strict_runtime_required(env: Mapping[str, str] | None = None) -> bool: def resolve_runtime() -> ContainerRuntime: """Return a reachable runtime, or stop this module the way strictness requires.""" found = detect_runtime() - if found is not None: - return found - if strict_runtime_required(): - pytest.fail( - f"{NO_RUNTIME_REASON} Strict mode is active, so this is a failure and " - f"not a skip: the runtime-backed claims in SECURITY.md cite this " - f"module, and a silent skip would leave them citing a run that never " - f"happened. Set {STRICT_RUNTIME_ENV}=0 to allow the skip.", - pytrace=False, - ) - pytest.skip(NO_RUNTIME_REASON) + if found is None: + if strict_runtime_required(): + pytest.fail( + f"{NO_RUNTIME_REASON} Strict mode is active, so this is a failure and " + f"not a skip: the runtime-backed claims in SECURITY.md cite this " + f"module, and a silent skip would leave them citing a run that never " + f"happened. Set {STRICT_RUNTIME_ENV}=0 to allow the skip.", + pytrace=False, + ) + pytest.skip(NO_RUNTIME_REASON) + return found def _free_port() -> int: diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py index dc17e9b96..e7ad1306c 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py @@ -18,7 +18,6 @@ import sys import urllib.request -import _input_policy import inspect_metrics import pytest import validate_dashboard @@ -63,7 +62,7 @@ def fake(host: str) -> tuple: raise PolicyError(f"refusing '{host}': it does not resolve") return tuple(ipaddress.ip_address(value) for value in mapping[host]) - monkeypatch.setattr(_input_policy, "resolve_addresses", fake) + monkeypatch.setattr("_input_policy.resolve_addresses", fake) return apply From a22b48ca7be0691ad9632a927c3fcaff9d63e61a Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Thu, 20 Aug 2026 17:10:47 -0700 Subject: [PATCH 18/28] docs: correct stale hook and residual-carrier statements The telemetry hook is retired, so the plugin identity and extension sections no longer claim it ships or needs manual configuration. The customization page said three carriers reach the store unfiltered and listed two; the instrumentation scope and schema fields are now stated as the third, matching the gap registered in the skill security model. --- docs/architecture/ai-artifacts.md | 4 ++-- docs/customization/copilot-otel-metrics.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/architecture/ai-artifacts.md b/docs/architecture/ai-artifacts.md index eef63feae..13b842e46 100644 --- a/docs/architecture/ai-artifacts.md +++ b/docs/architecture/ai-artifacts.md @@ -217,7 +217,7 @@ Copilot discovers skills automatically when their description matches the curren Root `plugin.json` is the sole component-membership authority for the `hve-core` plugin and VSIX. `.github/plugin/marketplace.json` contains one relative locator to the repository root and does not repeat membership. The plugin details surface resolves root `README.md` and `LICENSE`; the VSIX keeps `extension/README.md` and `extension/LICENSE` as its separate metadata surface. -The one product identity includes every distributable agent, prompt, instruction, and skill plus the fixed telemetry hook. Stable and PreRelease use the same manifest membership. +The one product identity includes every distributable agent, prompt, instruction, and skill. Stable and PreRelease use the same manifest membership. ### Deterministic Membership @@ -259,7 +259,7 @@ HVE Core has one Copilot plugin root at the repository root and one VSIX identit The VS Code extension is prepared with `Prepare-Extension.ps1` and packaged with `Package-Extension.ps1`. Both Stable and PreRelease preparation write the same component set to the single extension manifest and README. No Copilot package assembly step exists. -The plugin includes the telemetry hook. VS Code has no declarative hook contribution point, so extension users configure its location manually. +The plugin ships no hooks. Telemetry is opt-in through the `copilot-otel-metrics` skill, which the user installs and configures deliberately. For a repository-owned selection, the installer validates chosen component paths against root `plugin.json`. It converts repository-relative manifest declarations to installer form, then copies agents, prompts, instructions, and complete distributable skill directories while preserving canonical `.github` target paths; hooks are not copied. diff --git a/docs/customization/copilot-otel-metrics.md b/docs/customization/copilot-otel-metrics.md index 5962163cb..bfaacda41 100644 --- a/docs/customization/copilot-otel-metrics.md +++ b/docs/customization/copilot-otel-metrics.md @@ -152,6 +152,7 @@ Three things still reach the store unfiltered, and they are recorded as gaps rat * **Span names and metric metadata**, because dashboard queries match on them. A span name is composed by the emitter, so this is the residual worth watching. * **Span link attributes and metric exemplar attributes**, because this Collector distribution offers no way to reach them. OTTL has no `spanlink` context, refuses to index `links`, and silently ignores an assignment to `datapoint.exemplars`. +* **Instrumentation scope name and version, and the resource and scope schema URLs**, because they are expected to carry library identity rather than content. That is an expectation about a well-behaved emitter, not a control: nothing filters these fields, so anything placed in them is stored. None of that is inferred from reading the configuration. `tests/test_collector_carriers.py` starts the pinned Collector, sends a distinct marker through each of 28 carriers, and records what survives, with a paired control run proving each marker is visible when nothing filters it. The recorded map is asserted, so a change that opens a carrier fails the test suite. From 2fd2e48950c3dc5b49ff301cbdcb758d52880b27 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:08:09 -0700 Subject: [PATCH 19/28] fix(skills): close proxy egress, credential adoption, and settings policy gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pin the opener to an empty proxy map, keeping the redirect handler - give Grafana its own volume so it adopts the configured credential - check captureContent and otlpEndpoint against the merged document - correct the gap register to 29 registered, 15 open πŸ”’ - Generated by Copilot --- .../copilot-otel-metrics/SECURITY.md | 119 ++++++++-------- .../copilot-otel-metrics/examples/README.md | 92 +++++++++---- .../examples/_input_policy.py | 26 +++- .../examples/compose.yaml | 24 +++- .../examples/settings_upsert.py | 32 ++++- .../copilot-otel-metrics/examples/verify.py | 97 ++++++++++++- .../references/local-stack.md | 9 +- .../tests/test_helpers.py | 128 ++++++++++++++++++ .../tests/test_settings_upsert.py | 68 ++++++++++ 9 files changed, 488 insertions(+), 107 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/SECURITY.md b/.github/skills/experimental/copilot-otel-metrics/SECURITY.md index 1ebdfd332..d5aa2001b 100644 --- a/.github/skills/experimental/copilot-otel-metrics/SECURITY.md +++ b/.github/skills/experimental/copilot-otel-metrics/SECURITY.md @@ -44,7 +44,7 @@ The assistant behaviors add three bounded exposures. The settings write mutates | Credentials | Grafana admin credentials supplied by the operator through required environment variables, with no default and no committed value; an Application Insights connection string and a fleet ingest token for the Azure path, both operator-supplied and never written by the skill | | Network egress | None locally after the image pull. The Azure path sends telemetry to a collector the operator runs, which forwards to Application Insights over TLS | | Agent execution | Writes the user's global `settings.json` after an approved diff; writes generated artifacts to disk. Never starts a service and never provisions infrastructure | -| Open residual gaps | 27 registered, 13 open (highest: InfoDisc-Med, the carriers the filter does not reach and the shared fleet ingest credential). G-INF-1 is the highest-severity entry overall and is partially mitigated rather than open | +| Open residual gaps | 29 registered, 15 open (highest: InfoDisc-Med, the carriers the filter does not reach and the shared fleet ingest credential). G-INF-1 is the highest-severity entry overall and is partially mitigated rather than open | ## Contents @@ -66,7 +66,7 @@ The assistant behaviors add three bounded exposures. The settings write mutates ### Components -1. `examples/compose.yaml` β€” declares a Collector container publishing the only loopback-bound OTLP ports and one `grafana/otel-lgtm` container with no host OTLP mapping, joined by an explicit bridge network, mounting an external named volume at `/data`, requiring operator-supplied Grafana credentials with anonymous access disabled, and passing delta-to-cumulative conversion plus 120-day retention to Prometheus. Both images are digest-pinned, and the Collector service declares a memory limit, a read-only root filesystem, all capabilities dropped, and no-new-privileges. +1. `examples/compose.yaml` β€” declares a Collector container publishing the only loopback-bound OTLP ports and one `grafana/otel-lgtm` container with no host OTLP mapping, joined by an explicit bridge network, mounting an external named volume at `/data` with a separate project-managed volume over Grafana's database at `/data/grafana/data`, requiring operator-supplied Grafana credentials with anonymous access disabled, and passing delta-to-cumulative conversion plus 120-day retention to Prometheus. Both images are digest-pinned, and the Collector service declares a memory limit, a read-only root filesystem, all capabilities dropped, and no-new-privileges. 2. `examples/otel-collector-local.yaml` β€” the local Collector pipeline. Applies a fail-closed `redaction` allow-list across the trace, metric, and log pipelines, masks recognizable credential shapes in surviving values, scrubs the content carriers the allow-list cannot reach, and exports only to the LGTM service over the internal network. What each of its controls actually reaches is recorded by `tests/test_collector_carriers.py` rather than inferred from the configuration. 3. `examples/dashboards/copilot-otel.json` β€” Grafana dashboard referencing the pre-provisioned `prometheus` and `tempo` datasource uids. Contains PromQL and TraceQL queries only. 4. `examples/_input_policy.py` β€” shared, not run directly. Centralizes endpoint scheme, authority, port, remote opt-in, redirect revalidation, path containment, and required-credential checks for the helpers that take configurable input. @@ -211,17 +211,18 @@ flowchart TD ## Assets -| Id | Asset | Lifetime | Notes | -|----|----------------------------------------|---------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| -| A1 | Prompt and tool-call content in spans | Persisted in the volume | Observed present despite content capture being left at its default. Highest-value asset here. | -| A2 | Usage and cost metrics | Persisted, 120-day retention | Token counts, AIU billing proxy, tool call counts. Commercially sensitive in aggregate. | -| A3 | `copilot-otel-data` Docker volume | Persistent until explicitly removed | Unencrypted at rest. Survives `docker compose down` by design. | -| A4 | Grafana instance and dashboards | Persistent | Anonymous access disabled; admin credentials required from the operator's environment. Reachable on loopback only. | -| A5 | Baseline snapshot file | Persistent under the user cache | Contains metric and service names plus session ids, not content. | -| A6 | Stack container images | External, pulled on first run | `grafana/otel-lgtm` and `otel/opentelemetry-collector-contrib`, both pinned by manifest digest. | -| A7 | Global `settings.json` | Persistent, user-owned | JSONC with the user's own comments and formatting. Which file resolves depends on the scope the installed build declares. | -| A8 | Application Insights connection string | Persistent until the component is recreated | Fleet-wide write credential. Supplied by the operator, never written into a generated file. | -| A9 | Generated infrastructure templates | Persistent in the user's workspace | Inert until deployed. Deployment creates billable resources and an optional role assignment. | +| Id | Asset | Lifetime | Notes | +|-----|-----------------------------------------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| A1 | Prompt and tool-call content in spans | Persisted in the volume | Observed present despite content capture being left at its default. Highest-value asset here. | +| A2 | Usage and cost metrics | Persisted, 120-day retention | Token counts, AIU billing proxy, tool call counts. Commercially sensitive in aggregate. | +| A3 | `copilot-otel-data` Docker volume | Persistent until explicitly removed | Unencrypted at rest. Survives `docker compose down` by design. | +| A4 | Grafana instance and dashboards | Persistent | Anonymous access disabled; admin credentials required from the operator's environment. Reachable on loopback only. | +| A10 | `copilot-otel-grafana-db` Docker volume | Persistent until explicitly removed | Grafana's `grafana.db`: admin password hash, users, and dashboards. Project-managed, so it is created per stack and removed by `docker compose down -v`. Separate from A3 so the configured admin credential is the live one. | +| A5 | Baseline snapshot file | Persistent under the user cache | Contains metric and service names plus session ids, not content. | +| A6 | Stack container images | External, pulled on first run | `grafana/otel-lgtm` and `otel/opentelemetry-collector-contrib`, both pinned by manifest digest. | +| A7 | Global `settings.json` | Persistent, user-owned | JSONC with the user's own comments and formatting. Which file resolves depends on the scope the installed build declares. | +| A8 | Application Insights connection string | Persistent until the component is recreated | Fleet-wide write credential. Supplied by the operator, never written into a generated file. | +| A9 | Generated infrastructure templates | Persistent in the user's workspace | Inert until deployed. Deployment creates billable resources and an optional role assignment. | ## Adversaries @@ -288,15 +289,15 @@ Not applicable. The receiver executes no submitted content; OTLP payloads are pa ### Risk Rating -| Threat | Likelihood | Impact | Residual Risk | Status | -|---------------------------------------------------------------------------|------------|--------|---------------|---------------------------------------------------------------------| -| Local process injects synthetic Copilot series | Low | Medium | Medium | Detectable via `baseline.py`; not prevented (G-SPF-1) | -| Prompt content traverses plaintext OTLP | High | High | Medium | Contained by loopback binding only (G-INF-1, G-TLS-1) | -| Content survives in a carrier the filter preserves for a shipped consumer | Medium | High | Medium | Span names and metric metadata pass through by choice (G-INF-8) | -| Content survives in a carrier no processor can reach | Low | Medium | Medium | Span links and exemplars unreachable in this distribution (G-INF-9) | -| Content placed in a carrier assumed to hold only library identity | Low | Medium | Low | Scope and schema fields unfiltered and attacker-settable (G-INF-11) | -| Batched write loss from delta temporality | Low | Low | Low | Mitigated by delta-to-cumulative conversion (G-DOS-1) | -| Local flooding of the ingest endpoint | Low | Low | Low | Accepted for a single-machine demonstration stack | +| Threat | Likelihood | Impact | Residual Risk | Status | +|---------------------------------------------------------------------------|------------|--------|---------------|-----------------------------------------------------------------------| +| Local process injects synthetic Copilot series | Low | Medium | Medium | Detectable via `baseline.py`; not prevented (G-SPF-1) | +| Prompt content traverses plaintext OTLP | High | High | Medium | Contained by loopback binding on the ingest socket (G-INF-1, G-TLS-1) | +| Content survives in a carrier the filter preserves for a shipped consumer | Medium | High | Medium | Span names and metric metadata pass through by choice (G-INF-8) | +| Content survives in a carrier no processor can reach | Low | Medium | Medium | Span links and exemplars unreachable in this distribution (G-INF-9) | +| Content placed in a carrier assumed to hold only library identity | Low | Medium | Low | Scope and schema fields unfiltered and attacker-settable (G-INF-11) | +| Batched write loss from delta temporality | Low | Low | Low | Mitigated by delta-to-cumulative conversion (G-DOS-1) | +| Local flooding of the ingest endpoint | Low | Low | Low | Accepted for a single-machine demonstration stack | ## Bucket B2: Telemetry at rest and query surfaces @@ -328,12 +329,12 @@ Grafana's administrator role is the highest privilege in this bucket. It was pre ### Risk Rating -| Threat | Likelihood | Impact | Residual Risk | Status | -|---------------------------------------------------|------------|--------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Prompt content readable at rest by any local user | Low | High | Medium | Collector drops or scrubs content in every carrier it reaches; span names, metric metadata, links, and exemplars survive, and the volume remains unencrypted (G-INF-2, G-INF-8, G-INF-9) | -| Grafana admin reachable without a credential | Low | Medium | Low | Anonymous access disabled and the admin credential is operator-supplied with no default; loopback-only publishing (G-SPF-1, G-EOP-1) | -| Dashboard overwritten by the validation helper | Low | Low | Low | Non-loopback targets refused by default (G-TAM-1) | -| Unbounded trace growth exhausts disk | Low | Medium | Low | Accepted; operator removes the volume to reclaim | +| Threat | Likelihood | Impact | Residual Risk | Status | +|---------------------------------------------------|------------|--------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Prompt content readable at rest by any local user | Low | High | Medium | Collector drops or scrubs content in every carrier it reaches; span names, metric metadata, links, and exemplars survive, and the volume remains unencrypted (G-INF-2, G-INF-8, G-INF-9) | +| Grafana admin reachable without a credential | Low | Medium | Low | Anonymous access disabled, the admin credential is operator-supplied with no default, and Grafana's database is separated so it adopts that credential; loopback-only publishing (G-SPF-1, G-EOP-1, G-EOP-5) | +| Dashboard overwritten by the validation helper | Low | Low | Low | Non-loopback targets refused by default (G-TAM-1) | +| Unbounded trace growth exhausts disk | Low | Medium | Low | Accepted; operator removes the volume to reclaim | ## Bucket B3: Reference helper scripts to local service APIs @@ -353,7 +354,7 @@ Not applicable. The helpers are operator-invoked interactive tools that print th ### Information Disclosure -The helpers print metric names, label values, and series values to the terminal. `inspect_metrics.py` prints label sets, which include model names, tool names, and session identifiers, but not span content: prompt-bearing attributes live on spans in Tempo and are not enumerated by any shipped helper. `baseline.py` persists metric names, service names, service versions, session ids, and trace names to its snapshot file. No helper writes prompt content to disk, and none transmits anything off the host. Terminal output may still be captured by shell history or a session recorder. +The helpers print metric names, label values, and series values to the terminal. `inspect_metrics.py` prints label sets, which include model names, tool names, and session identifiers, but not span content: prompt-bearing attributes live on spans in Tempo and are not enumerated by any shipped helper. `baseline.py` persists metric names, service names, service versions, session ids, and trace names to its snapshot file. No helper writes prompt content to disk. No helper transmits anything off the host, and that now rests on the transport rather than on the address alone: `open_url` builds its opener with an explicitly empty `ProxyHandler`, because `urllib.request.build_opener` otherwise installs the default proxy handler and `proxy_bypass` does not exempt loopback. Before that pin, a `HTTP_PROXY` value in the operator's environment would have carried a loopback-addressed request β€” including the Grafana Basic credential `validate_dashboard.py` attaches β€” to the proxy host in plaintext, unless `no_proxy` happened to cover loopback. Terminal output may still be captured by shell history or a session recorder. Everything returned from the store is untrusted data, never instructions. B1 Spoofing establishes that any local process can inject series carrying genuine Copilot names with attacker-controlled `service_version` and `session_id` values, and those values are exactly what these helpers print. The same applies to span attributes and trace names read through Tempo. Treat helper output, query results, and dashboard content as data to be inspected, and never act on text embedded in them. @@ -529,35 +530,37 @@ Deployment creates billable Azure resources and, when a principal is supplied, a ## Enterprise Readiness Gaps -| Id | Severity | Gap | Status | -|----------|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| G-INF-1 | InfoDisc-High | Spans carry full prompt text, tool call arguments and results, and system instructions on a configuration where content capture was left at its documented default. The skill cannot change extension behavior. | Partially mitigated. Content still leaves the editor and crosses the loopback hop in plaintext, which remains outside the skill's control. Both the local and organization paths now filter before storage: local uses a fail-closed allow-list at the Collector, so an attribute added by a future extension release is dropped rather than stored, and the Azure path applies the same fail-closed allow-list and content scrub to traces, metrics, and logs before export. | -| G-INF-2 | InfoDisc-Med | The `copilot-otel-data` volume stores whatever survives filtering, unencrypted, and survives `docker compose down` by design. | Open on encryption, reduced on content. The allow-list keeps observed and unknown content attributes out of the volume, so what remains is usage and cost data rather than prompt text. Volume encryption is a Docker and host-platform property the skill does not control. Traces inherit the image's 336h Tempo expiry rather than persisting indefinitely; metrics retain for 120 days by configuration. Teardown documentation states both variants explicitly. | -| G-INF-3 | InfoDisc-Med | Organization capture distributes one static write-side credential to every workstation, with no per-user binding and no documented in-place rotation. Anything holding it can inject telemetry indistinguishable from a real developer's. | Open and now active rather than optional. The receiver requires the token instead of leaving authentication commented out, so the gap is a bounded shared-credential weakness rather than an unauthenticated endpoint. Disclosed before any Azure artifact is generated, with the write-side-only blast radius stated plainly. | -| G-INF-4 | InfoDisc-Low | The assisted settings write reads the whole global `settings.json`, so unrelated values in that file enter model context. | Accepted. Reading the whole document is required to preserve it; only the changed lines are echoed in the presented diff. | -| G-SPF-1 | Spoofing-Med | The Prometheus and Tempo query APIs are unauthenticated, and the local OTLP ingress accepts spans from any local process. | Reduced. Grafana now requires an operator-supplied credential with anonymous access disabled. Prometheus and Tempo remain unauthenticated behind `127.0.0.1` binding, and injected spans are still accepted, though the allow-list bounds what an injector can store. `baseline.py` provides after-the-fact detection of injected series. | -| G-SUP-1 | SupplyChain-Med | Image identity was previously pinned by mutable tag, and no signature or provenance verification is performed before the container runs. | Partially closed. Both images are now pinned by multi-architecture manifest digest, which removes the substitution-under-a-tag vector. Signature and provenance verification remains a documented operator step rather than an enforced one; the update procedure states that digest pinning, publisher verification, and vulnerability review are three separate checks. | -| G-EOP-1 | EoP-Low | Grafana administrator access was previously reachable from any local process using published default credentials. | Closed. Anonymous Admin access is disabled and the admin credential is operator-supplied with no default. The residual observation stands that the same local actor can read the volume directly, so this was never a host-user boundary crossing. | -| G-EOP-2 | EoP-Med | A compromised stack image would execute with Docker daemon authority, which is root-equivalent on a typical developer workstation. | Bounded by digest pinning, by adding no capabilities, and by mounting no host paths beyond the named volume and the read-only Collector configuration. | -| G-TAM-1 | Tampering-Low | `validate_dashboard.py` imports with `overwrite: true`, so it would replace an unrelated dashboard sharing the same uid. | Constrained. All configurable endpoints pass a shared policy that refuses non-http schemes, credentials in the authority, unrelated local ports, and non-loopback hosts without an explicit opt-in, and re-applies the same rules to every redirect target. The dashboard path is contained to the examples directory. | -| G-TAM-2 | Tampering-Med | The assisted settings write mutates a user-owned JSONC file that may hold comments, formatting, and unrelated configuration the skill did not create. | Mitigated structurally and now executably. `examples/settings_upsert.py` enforces the contract the prose described: schema-checked keys and types, value-span splicing that never reserializes, a refusal to write when unrelated settings would change or the result would not parse, backup before write, restore on failure, and a redacted audit record. Concurrent VS Code writes remain unpreventable. | -| G-REP-1 | Repudiation-Low | Settings Sync conflict behavior for a write made outside the running VS Code instance was never confirmed. | Open. Disclosed to the user as a caveat before the write rather than asserted in either direction. | -| G-DOS-1 | DoS-Low | Delta-to-cumulative conversion state is held in memory and resets on container restart, producing a bounded gap in converted series. | Accepted. Without the flag the failure mode is worse, because a dropped delta metric can fail an entire batched write. | -| G-DOS-2 | DoS-Low | The shared fleet credential permits unbounded ingestion against a billed backend, so the practical denial of service is financial. | Mitigated by a 5 GB daily cap defaulted in every generated template, and by naming `captureContent` as the dominant volume multiplier wherever cost is discussed. | -| G-EOP-3 | EoP-Med | Generated infrastructure templates provision billable Azure resources and can create a `Monitoring Reader` role assignment when deployed. | Bounded. The agent never holds Azure credentials and never deploys; the role assignment is opt-in through an empty-by-default parameter and is scoped to a single workspace rather than to the resource group or subscription; required inputs have no defaults. | -| G-TLS-2 | Spoofing-Med | The generated fleet receiver now terminates TLS from operator-supplied material, but how Copilot's exporter validates that certificate is exporter behavior the skill does not control. | Open by ownership. The receiver requires TLS material or an explicitly recorded terminating ingress that owns certificate lifecycle and cipher policy. No claim is made about exporter-side validation or custom CA support, because none was verified. | -| G-INF-5 | InfoDisc-Med | A single shared Log Analytics workspace gives every authorized reader visibility across all environments, and a resource attribute cannot restrict that after the fact. | Mitigated by default. One collector endpoint, ingest token, Application Insights component, and workspace per environment is now the shipped topology; environment is a required input with no default in Bicep, Terraform, and the CLI script. Reader assignments are scoped to a single workspace. Attributes are documented as grouping keys, never as isolation controls. This is environment separation, not customer multi-tenancy, and is stated as such. | -| G-INF-6 | InfoDisc-Low | Retention is the only deletion mechanism. Neither the local stack nor the Azure templates purge a specific person's data on request. | Open and operator-owned. Retention is documented as the deletion boundary in both templates and their guidance, and an erasure obligation is presented as an operator procedure against the workspace rather than a template capability. No purge automation is provided. | -| G-SUP-2 | SupplyChain-Low | Terraform state for the fleet templates contains the Application Insights connection string, a fleet-wide write credential, and `sensitive = true` does not remove it from state. | Documented rather than solved. The configuration, its README, and the Azure reference all state that sensitivity hides CLI display only, and recommend an encrypted remote backend with restricted access and treating the state file as a secret. | -| G-EOP-4 | EoP-Med | The prohibition on the agent running `docker compose`, `az deployment`, or `terraform apply` is advisory prose in `SKILL.md` rather than an enforced control. | Open. Stated in the constraints and the stop rules and exercised by the behavior gate. A hook would make it enforced. | -| G-TLS-1 | InfoDisc-Low | OTLP ingest and all service queries use plaintext HTTP with no transport security. | Acceptable on loopback. Material the moment `otlpEndpoint` targets a remote collector, which the skill states explicitly. | -| G-INF-7 | InfoDisc-Med | The shipped per-workstation relay listens without authentication, so any local process able to reach loopback can inject spans that the relay then forwards under the fleet credential. Its runtime environment file is readable by anything running as the same operating-system account or able to inspect the container. | Open, and the least-bad of three options: the alternatives are delivering the ingest token through an environment agent-spawned subprocesses inherit, or removing receiver authentication entirely. Loopback binding is necessary and not sufficient for provenance. The relay authenticates the relay, not the workstation and not the developer. The relay verifies the fleet receiver's certificate against an operator-supplied CA bundle and refuses an untrusted certificate or a mismatched hostname, which is proven by `tests/test_collector_carriers.py`. mTLS is the preferred replacement and is blocked on unestablished agent-host client-certificate support. | -| G-DOS-3 | DoS-Med | Managed settings expose one telemetry endpoint to both the extension and the agent host, so pointing it at the relay makes relay health a prerequisite for all of a workstation's Copilot telemetry. A stopped, missing, or unhealthy relay loses that telemetry outright rather than queueing it. | Open by design trade. The alternative topologies either distribute the fleet credential into every agent-spawned subprocess or leave agent-host telemetry dropped fleet-wide. Bounded by `restart: unless-stopped`, a health endpoint, relay-first rollout with positive verification before the managed endpoint changes, a documented rollback to the previous endpoint, and a detection table whose first symptom is total signal absence from one workstation. | -| G-INF-12 | InfoDisc-Low | The content scrub runs with `error_mode: ignore`, so an OTTL statement that errors on one record shape is skipped and that record's carrier reaches the exporter unscrubbed. This makes the scrub fail-open per statement while its sibling allow-list is fail-closed. | Open and load-bearing rather than incidental: a map-valued log body keeps its allow-listed subset precisely because `set(log.body, ...)` does not apply to it. Bounded by digest-pinned images and by `tests/test_collector_carriers.py`, which asserts the resulting carrier map against the pinned runtime, so a change in that behaviour fails the suite rather than passing quietly. Recorded here rather than left implicit in the configuration comment. | -| G-INF-8 | InfoDisc-Med | Span names, metric names, metric descriptions, and metric units reach storage unfiltered. A span name is emitter-composed and can carry request text. | Open by choice, not by oversight. Three dashboard TraceQL queries match on span name, `baseline.py` reads Tempo `rootTraceName` as its injection discriminator, and every shipped Prometheus query selects by metric name; scrubbing any of them breaks shipped functionality. Recorded by `tests/test_collector_carriers.py`, which fails if a scrub rule is later added that targets them. | -| G-INF-9 | InfoDisc-Med | Span link attributes, span link trace state, and metric exemplar filtered attributes reach storage unfiltered. | Open by capability. Verified against the pinned Collector: the `redaction` processor does not traverse links or exemplars, OTTL has no `spanlink` context and refuses to index `links`, and assigning to `datapoint.exemplars` parses without effect. No processor in this distribution reaches them. Whether Copilot populates links or exemplars in practice is unmeasured, so the exposure is recorded unbounded rather than narrowed by an assumption; settling it needs a live capture. | -| G-INF-10 | InfoDisc-Low | The carrier map is established against the OTLP/HTTP receiver. The shipped configuration also publishes OTLP/gRPC, whose content handling is unprobed. | Open. Both receivers feed the same processor chain, so divergence is unlikely rather than excluded. Closing it means extending the harness to the gRPC path. | -| G-INF-11 | InfoDisc-Low | Instrumentation scope name and version and the resource and scope schema URLs reach storage unfiltered. They normally carry library identity, but the receiver is unauthenticated and no processor filters them, so a local process can place arbitrary text there. | Open by low expected content, the weakest of this document's three justifications for an open carrier. Recorded as a gap rather than dismissed by rationale, because "identity, not content" is the same argument the harness disproved for span links. Closing it means extending `transform/scrub` to these fields, which is possible for the schema URLs and would need a consumer check for scope name. | +| Id | Severity | Gap | Status | +|----------|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| G-INF-1 | InfoDisc-High | Spans carry full prompt text, tool call arguments and results, and system instructions on a configuration where content capture was left at its documented default. The skill cannot change extension behavior. | Partially mitigated. Content still leaves the editor and crosses the loopback hop in plaintext, which remains outside the skill's control. Both the local and organization paths now filter before storage: local uses a fail-closed allow-list at the Collector, so an attribute added by a future extension release is dropped rather than stored, and the Azure path applies the same fail-closed allow-list and content scrub to traces, metrics, and logs before export. | +| G-INF-2 | InfoDisc-Med | The `copilot-otel-data` volume stores whatever survives filtering, unencrypted, and survives `docker compose down` by design. | Open on encryption, reduced on content. The allow-list keeps observed and unknown content attributes out of the volume, so what remains is usage and cost data rather than prompt text. Volume encryption is a Docker and host-platform property the skill does not control. Traces inherit the image's 336h Tempo expiry rather than persisting indefinitely; metrics retain for 120 days by configuration. Teardown documentation states both variants explicitly. | +| G-INF-3 | InfoDisc-Med | Organization capture distributes one static write-side credential to every workstation, with no per-user binding and no documented in-place rotation. Anything holding it can inject telemetry indistinguishable from a real developer's. | Open and now active rather than optional. The receiver requires the token instead of leaving authentication commented out, so the gap is a bounded shared-credential weakness rather than an unauthenticated endpoint. Disclosed before any Azure artifact is generated, with the write-side-only blast radius stated plainly. | +| G-INF-4 | InfoDisc-Low | The assisted settings write reads the whole global `settings.json`, so unrelated values in that file enter model context. | Accepted. Reading the whole document is required to preserve it; only the changed lines are echoed in the presented diff. | +| G-SPF-1 | Spoofing-Med | The Prometheus and Tempo query APIs are unauthenticated, and the local OTLP ingress accepts spans from any local process. | Reduced. Grafana now requires an operator-supplied credential with anonymous access disabled. Prometheus and Tempo remain unauthenticated behind `127.0.0.1` binding, and injected spans are still accepted, though the allow-list bounds what an injector can store. `baseline.py` provides after-the-fact detection of injected series. | +| G-SUP-1 | SupplyChain-Med | Image identity was previously pinned by mutable tag, and no signature or provenance verification is performed before the container runs. | Partially closed. Both images are now pinned by multi-architecture manifest digest, which removes the substitution-under-a-tag vector. Signature and provenance verification remains a documented operator step rather than an enforced one; the update procedure states that digest pinning, publisher verification, and vulnerability review are three separate checks. | +| G-EOP-1 | EoP-Low | Grafana administrator access was previously reachable from any local process using published default credentials. | Closed, and now closed by mechanism rather than by configuration alone. Anonymous Admin access is disabled and the admin credential is operator-supplied with no default. The earlier closure claim was incomplete: Grafana applies `GF_SECURITY_ADMIN_*` only when it creates its database, and the shipped topology reused an external volume, so a stack could run on the `admin`/`admin` default while the required variables were supplied and ignored. Grafana's database now has its own project-managed volume mounted over `GF_PATHS_DATA`, and `verify.py` reports a live default credential distinctly from a rejected configured one. The residual observation stands that the same local actor can read the volume directly, so this was never a host-user boundary crossing. | +| G-EOP-2 | EoP-Med | A compromised stack image would execute with Docker daemon authority, which is root-equivalent on a typical developer workstation. | Bounded by digest pinning, by adding no capabilities, and by mounting no host paths beyond the named volume and the read-only Collector configuration. | +| G-TAM-1 | Tampering-Low | `validate_dashboard.py` imports with `overwrite: true`, so it would replace an unrelated dashboard sharing the same uid. | Constrained. All configurable endpoints pass a shared policy that refuses non-http schemes, credentials in the authority, unrelated local ports, and non-loopback hosts without an explicit opt-in, and re-applies the same rules to every redirect target. The dashboard path is contained to the examples directory. | +| G-TAM-2 | Tampering-Med | The assisted settings write mutates a user-owned JSONC file that may hold comments, formatting, and unrelated configuration the skill did not create. | Mitigated structurally and now executably. `examples/settings_upsert.py` enforces the contract the prose described: schema-checked keys and types, value-span splicing that never reserializes, a refusal to write when unrelated settings would change or the result would not parse, backup before write, restore on failure, and a redacted audit record. Concurrent VS Code writes remain unpreventable. | +| G-REP-1 | Repudiation-Low | Settings Sync conflict behavior for a write made outside the running VS Code instance was never confirmed. | Open. Disclosed to the user as a caveat before the write rather than asserted in either direction. | +| G-DOS-1 | DoS-Low | Delta-to-cumulative conversion state is held in memory and resets on container restart, producing a bounded gap in converted series. | Accepted. Without the flag the failure mode is worse, because a dropped delta metric can fail an entire batched write. | +| G-DOS-2 | DoS-Low | The shared fleet credential permits unbounded ingestion against a billed backend, so the practical denial of service is financial. | Mitigated by a 5 GB daily cap defaulted in every generated template, and by naming `captureContent` as the dominant volume multiplier wherever cost is discussed. | +| G-EOP-3 | EoP-Med | Generated infrastructure templates provision billable Azure resources and can create a `Monitoring Reader` role assignment when deployed. | Bounded. The agent never holds Azure credentials and never deploys; the role assignment is opt-in through an empty-by-default parameter and is scoped to a single workspace rather than to the resource group or subscription; required inputs have no defaults. | +| G-TLS-2 | Spoofing-Med | The generated fleet receiver now terminates TLS from operator-supplied material, but how Copilot's exporter validates that certificate is exporter behavior the skill does not control. | Open by ownership. The receiver requires TLS material or an explicitly recorded terminating ingress that owns certificate lifecycle and cipher policy. No claim is made about exporter-side validation or custom CA support, because none was verified. | +| G-INF-5 | InfoDisc-Med | A single shared Log Analytics workspace gives every authorized reader visibility across all environments, and a resource attribute cannot restrict that after the fact. | Mitigated by default. One collector endpoint, ingest token, Application Insights component, and workspace per environment is now the shipped topology; environment is a required input with no default in Bicep, Terraform, and the CLI script. Reader assignments are scoped to a single workspace. Attributes are documented as grouping keys, never as isolation controls. This is environment separation, not customer multi-tenancy, and is stated as such. | +| G-INF-6 | InfoDisc-Low | Retention is the only deletion mechanism. Neither the local stack nor the Azure templates purge a specific person's data on request. | Open and operator-owned. Retention is documented as the deletion boundary in both templates and their guidance, and an erasure obligation is presented as an operator procedure against the workspace rather than a template capability. No purge automation is provided. | +| G-SUP-2 | SupplyChain-Low | Terraform state for the fleet templates contains the Application Insights connection string, a fleet-wide write credential, and `sensitive = true` does not remove it from state. | Documented rather than solved. The configuration, its README, and the Azure reference all state that sensitivity hides CLI display only, and recommend an encrypted remote backend with restricted access and treating the state file as a secret. | +| G-EOP-4 | EoP-Med | The prohibition on the agent running `docker compose`, `az deployment`, or `terraform apply` is advisory prose in `SKILL.md` rather than an enforced control. | Open. Stated in the constraints and the stop rules and exercised by the behavior gate. A hook would make it enforced. | +| G-TLS-1 | InfoDisc-Low | OTLP ingest and all service queries use plaintext HTTP with no transport security. | Acceptable on loopback, and that containment is now enforced rather than assumed. Ingest is contained by the receiver's `127.0.0.1` binding. The query path is the helpers, which reach the network only through `open_url`; it previously inherited whatever `HTTP_PROXY` the operator's environment carried, because `build_opener` adds the default proxy handler and `proxy_bypass` does not exempt loopback, so containment depended on `no_proxy` hygiene rather than on the code. `open_url` now pins an empty `ProxyHandler`, asserted by `tests/test_helpers.py::TestProxyNeutralization`. Plaintext remains material the moment `otlpEndpoint` targets a remote collector, which the skill states explicitly. | +| G-INF-13 | InfoDisc-Low | Proxy neutralization lives inside `open_url`, so it protects only code that routes through the shared policy module. The invariant that keeps helpers there is a source substring check for `urlopen(` across four named files. | Open and narrow. A helper that built its own opener, or a fifth helper not added to the list, would escape both the proxy pin and the redirect allow-list without failing the suite. Bounded by the four bundled helpers being the only shipped network callers and by `tests/test_helpers.py::TestPolicyRoutedHelpers`. Closing it means asserting the property against the built opener for every helper rather than against the source text of a fixed list. | +| G-EOP-5 | EoP-Low | Grafana's first-run-only credential semantics survive the volume separation. Changing `COPILOT_OTEL_GRAFANA_PASSWORD` on a stack whose Grafana database already exists has no effect, so the shipped stack has no in-band rotation. | Open and disclosed. Separating the database fixes adoption, not rotation: the variable is still read once, now per project rather than per host. Rotation is an operator procedure β€” remove the `copilot-otel-grafana-db` volume and start the stack again, or use `grafana cli admin reset-admin-password` β€” documented in `examples/README.md`. Detection is `verify.py`, which reports whether the image default authenticates rather than only whether the configured pair does. A secondary residual: the pre-existing `grafana.db` on `copilot-otel-data` is shadowed rather than deleted, so it keeps its old password hash and dashboards until the operator removes that volume. | +| G-INF-7 | InfoDisc-Med | The shipped per-workstation relay listens without authentication, so any local process able to reach loopback can inject spans that the relay then forwards under the fleet credential. Its runtime environment file is readable by anything running as the same operating-system account or able to inspect the container. | Open, and the least-bad of three options: the alternatives are delivering the ingest token through an environment agent-spawned subprocesses inherit, or removing receiver authentication entirely. Loopback binding is necessary and not sufficient for provenance. The relay authenticates the relay, not the workstation and not the developer. The relay verifies the fleet receiver's certificate against an operator-supplied CA bundle and refuses an untrusted certificate or a mismatched hostname, which is proven by `tests/test_collector_carriers.py`. mTLS is the preferred replacement and is blocked on unestablished agent-host client-certificate support. | +| G-DOS-3 | DoS-Med | Managed settings expose one telemetry endpoint to both the extension and the agent host, so pointing it at the relay makes relay health a prerequisite for all of a workstation's Copilot telemetry. A stopped, missing, or unhealthy relay loses that telemetry outright rather than queueing it. | Open by design trade. The alternative topologies either distribute the fleet credential into every agent-spawned subprocess or leave agent-host telemetry dropped fleet-wide. Bounded by `restart: unless-stopped`, a health endpoint, relay-first rollout with positive verification before the managed endpoint changes, a documented rollback to the previous endpoint, and a detection table whose first symptom is total signal absence from one workstation. | +| G-INF-12 | InfoDisc-Low | The content scrub runs with `error_mode: ignore`, so an OTTL statement that errors on one record shape is skipped and that record's carrier reaches the exporter unscrubbed. This makes the scrub fail-open per statement while its sibling allow-list is fail-closed. | Open and load-bearing rather than incidental: a map-valued log body keeps its allow-listed subset precisely because `set(log.body, ...)` does not apply to it. Bounded by digest-pinned images and by `tests/test_collector_carriers.py`, which asserts the resulting carrier map against the pinned runtime, so a change in that behaviour fails the suite rather than passing quietly. Recorded here rather than left implicit in the configuration comment. | +| G-INF-8 | InfoDisc-Med | Span names, metric names, metric descriptions, and metric units reach storage unfiltered. A span name is emitter-composed and can carry request text. | Open by choice, not by oversight. Three dashboard TraceQL queries match on span name, `baseline.py` reads Tempo `rootTraceName` as its injection discriminator, and every shipped Prometheus query selects by metric name; scrubbing any of them breaks shipped functionality. Recorded by `tests/test_collector_carriers.py`, which fails if a scrub rule is later added that targets them. | +| G-INF-9 | InfoDisc-Med | Span link attributes, span link trace state, and metric exemplar filtered attributes reach storage unfiltered. | Open by capability. Verified against the pinned Collector: the `redaction` processor does not traverse links or exemplars, OTTL has no `spanlink` context and refuses to index `links`, and assigning to `datapoint.exemplars` parses without effect. No processor in this distribution reaches them. Whether Copilot populates links or exemplars in practice is unmeasured, so the exposure is recorded unbounded rather than narrowed by an assumption; settling it needs a live capture. | +| G-INF-10 | InfoDisc-Low | The carrier map is established against the OTLP/HTTP receiver. The shipped configuration also publishes OTLP/gRPC, whose content handling is unprobed. | Open. Both receivers feed the same processor chain, so divergence is unlikely rather than excluded. Closing it means extending the harness to the gRPC path. | +| G-INF-11 | InfoDisc-Low | Instrumentation scope name and version and the resource and scope schema URLs reach storage unfiltered. They normally carry library identity, but the receiver is unauthenticated and no processor filters them, so a local process can place arbitrary text there. | Open by low expected content, the weakest of this document's three justifications for an open carrier. Recorded as a gap rather than dismissed by rationale, because "identity, not content" is the same argument the harness disproved for span links. Closing it means extending `transform/scrub` to these fields, which is possible for the schema URLs and would need a consumer check for scope name. | ## References diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/README.md b/.github/skills/experimental/copilot-otel-metrics/examples/README.md index 519f58d2e..91fdac251 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/README.md +++ b/.github/skills/experimental/copilot-otel-metrics/examples/README.md @@ -39,37 +39,53 @@ The helpers use only the Python standard library, so there is nothing to install Run these yourself, from this directory. Nothing here runs automatically, and the skill will not run any of it for you. -```bash -# 1. Choose Grafana credentials. The stack will not start without them, and -# the helpers read the same two variables. Reading the password rather than -# writing it inline keeps it out of the terminal scrollback and out of the -# shell history file. -export COPILOT_OTEL_GRAFANA_USER=admin -read -rs -p 'Grafana password: ' COPILOT_OTEL_GRAFANA_PASSWORD; echo -export COPILOT_OTEL_GRAFANA_PASSWORD - -# 2. Create the volume once, then start the stack. -docker volume create copilot-otel-data -docker compose -f compose.yaml up -d +1. Choose Grafana credentials. The stack will not start without them, and the helpers read the same two variables. Reading the password rather than writing it inline keeps it out of the terminal scrollback and out of the shell history file. -# 3. Optional but recommended if this store has ever held test payloads. -python3 baseline.py capture + ```bash + export COPILOT_OTEL_GRAFANA_USER=admin + read -rs -p 'Grafana password: ' COPILOT_OTEL_GRAFANA_PASSWORD; echo + export COPILOT_OTEL_GRAFANA_PASSWORD + ``` -# 4. Enable export in your VS Code user settings, then reload the window. +2. Create the telemetry volume once, then start the stack. Grafana's own volume is created by Compose, so there is nothing to create for it. -# 5. Confirm signals are actually stored, not merely accepted. -python3 verify.py + ```bash + docker volume create copilot-otel-data + docker compose -f compose.yaml up -d + ``` -# 6. If step 5 is ambiguous, prove the telemetry is genuine. -python3 baseline.py diff +3. Optional, but recommended if this store has ever held test payloads. -# 7. See what your build really emits before trusting any metric name. -python3 inspect_metrics.py + ```bash + python3 baseline.py capture + ``` -# 8. Import a dashboard and check every panel resolves. -python3 validate_dashboard.py # the bundled seed -python3 validate_dashboard.py my-dash.json # a generated dashboard -``` +4. Enable export in your VS Code user settings, then reload the window. + +5. Confirm signals are actually stored, not merely accepted. This also reports whether the Grafana credential you configured is the live one. + + ```bash + python3 verify.py + ``` + +6. If step 5 is ambiguous, prove the telemetry is genuine. + + ```bash + python3 baseline.py diff + ``` + +7. See what your build really emits before trusting any metric name. + + ```bash + python3 inspect_metrics.py + ``` + +8. Import a dashboard and check every panel resolves. + + ```bash + python3 validate_dashboard.py # the bundled seed + python3 validate_dashboard.py my-dash.json # a generated dashboard + ``` ## Notes on the stack definition @@ -77,14 +93,36 @@ python3 validate_dashboard.py my-dash.json # a generated dashboard * **Copilot exports to the Collector, not to LGTM.** LGTM's OTLP ports are not published to the host at all. That is what makes the filter unavoidable: if LGTM listened on the host too, any local process could write straight past it, and loopback binding would not help, because every process on the machine is already on loopback. * **The attribute policy is an allow-list.** `allow_all_keys: false` drops any key the config has not named, so an attribute added by a future extension release is dropped rather than stored. A delete-list would pass it through. Every allowed key is there because a shipped dashboard panel reads it. -* **Grafana requires a credential you supply.** The image otherwise enables anonymous Admin access and hides the login form. The compose file turns that off and requires `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD`; an unset variable fails the `up` with a message rather than starting on a guessable credential. +* **Grafana requires a credential you supply.** The image otherwise enables anonymous Admin access and hides the login form. The compose file turns that off and requires `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD`. The `${VAR:?message}` form fails the `up` when a variable is unset; it does not make Grafana adopt the value. Grafana reads those variables only when it creates its database, so adoption depends on the volume layout below. +* **Grafana's database is on its own volume.** `copilot-otel-grafana-db` is mounted over `/data/grafana/data` and is deliberately not `external`, so Compose creates it per project. Without that, a stack started against a reused `copilot-otel-data` volume would find an existing `grafana.db` and keep whatever password it already carries β€” possibly the image default of `admin`/`admin` β€” while the required variables were supplied and ignored. Grafana users and dashboards live on this volume, so they are not carried over from a pre-existing shared volume. Telemetry history is unaffected; it stays on `copilot-otel-data`. * **Both images are pinned by digest, with the tag in a comment.** A tag is mutable, so a tag-pinned stack can change under a configuration you already reviewed. * Every port publishes to `127.0.0.1` only. The OTLP endpoint accepts writes from anything that can reach it. -* The data volume is declared `external`, so Compose binds the existing `copilot-otel-data` volume instead of creating a project-prefixed duplicate that would orphan your history. +* The telemetry volume is declared `external`, so Compose binds the existing `copilot-otel-data` volume instead of creating a project-prefixed duplicate that would orphan your history. * `PROMETHEUS_EXTRA_ARGS` enables delta-to-cumulative conversion and raises retention to 120 days. Prometheus otherwise drops delta metrics, and a dropped delta metric was observed failing an entire batched write. The default 15-day retention silently truncates monthly token totals. The filter governs what reaches the store. It does not change what the extension emits, so content attributes still leave the editor and still cross the loopback hop in plaintext. +### If you already ran an older version of this stack + +Before `copilot-otel-grafana-db` existed, Grafana's database lived on the shared `copilot-otel-data` volume. Adding the separate volume shadows that database rather than deleting it, so the first `up` after the change gives you a Grafana that has adopted your configured credential and has none of your previous dashboards or Grafana users. The old database is still on `copilot-otel-data` at `grafana/data` if you want anything out of it. + +Check which credential is actually live before you assume the change took effect: + +```bash +python3 verify.py +``` + +`verify.py` reports the configured credential and the Grafana default separately. If it reports that `admin`/`admin` still authenticates, the stack is running on a database that was created before you set a password. Rotating it is the same operation as discarding it, because a fresh database adopts `COPILOT_OTEL_GRAFANA_PASSWORD` on first start: + +```bash +docker compose -f compose.yaml down +docker volume ls --filter name=copilot-otel-grafana-db # confirm the project-prefixed name +docker volume rm +docker compose -f compose.yaml up -d +``` + +That discards Grafana users and dashboards. It does not touch telemetry, which lives on `copilot-otel-data`. Grafana also ships `grafana cli admin reset-admin-password` for an in-place change; its invocation depends on the image's Grafana paths, so check `docker exec copilot-otel-lgtm env | grep GF_PATHS` before using it. + To move to a newer image, resolve the multi-architecture manifest digest for the tag you want from the publisher's own registry listing, verify the publisher's signature or attestation separately, and review the new version for known vulnerabilities. Those are three different checks and none of them implies the others. ## Notes on the helpers diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py index 989c2484f..2b7168148 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/_input_policy.py @@ -13,12 +13,14 @@ from __future__ import annotations +import http.client import ipaddress import os import pathlib import socket import urllib.parse import urllib.request +from typing import IO __all__ = [ "DEFAULT_ALLOWED_PORTS", @@ -199,7 +201,15 @@ def __init__(self, *, allow_remote: bool, allowed_ports: frozenset[int]) -> None self._allow_remote = allow_remote self._allowed_ports = allowed_ports - def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ANN201 + def redirect_request( + self, + req: urllib.request.Request, + fp: IO[bytes], + code: int, + msg: str, + headers: http.client.HTTPMessage, + newurl: str, + ) -> urllib.request.Request | None: check_url(newurl, allow_remote=self._allow_remote, allowed_ports=self._allowed_ports) redirected = super().redirect_request(req, fp, code, msg, headers, newurl) if redirected is None: @@ -227,12 +237,20 @@ def open_url( allow_remote: bool = False, allowed_ports: frozenset[int] = DEFAULT_ALLOWED_PORTS, timeout: int = 25, -): # noqa: ANN201 - """Open a URL after checking it, and re-check every redirect it follows.""" +) -> http.client.HTTPResponse: + """Open a URL after checking it, and re-check every redirect it follows. + + The response is an `HTTPResponse` rather than a wider union because + `check_url` admits only http and https, so no other handler can answer. + """ url = request if isinstance(request, str) else request.full_url check_url(url, allow_remote=allow_remote, allowed_ports=allowed_ports) + # The empty ProxyHandler is explicit because build_opener adds to the default + # chain: without it, the default handler reads HTTP_PROXY and routes even a + # loopback request off-box, since proxy_bypass does not exempt loopback. opener = urllib.request.build_opener( - _PolicyRedirectHandler(allow_remote=allow_remote, allowed_ports=allowed_ports) + _PolicyRedirectHandler(allow_remote=allow_remote, allowed_ports=allowed_ports), + urllib.request.ProxyHandler({}), ) return opener.open(request, timeout=timeout) diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml index 69c3df928..a3017b006 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml +++ b/.github/skills/experimental/copilot-otel-metrics/examples/compose.yaml @@ -6,8 +6,10 @@ # deliberately unmapped so no local process can bypass the filter. # # Images are pinned by manifest digest. Grafana credentials are required with -# no default, and the data volume is external so this file binds the existing -# copilot-otel-data volume rather than creating a project-prefixed duplicate. +# no default, and the telemetry volume is external so this file binds the +# existing copilot-otel-data volume rather than creating a project-prefixed +# duplicate. Grafana's own database is deliberately NOT on that volume: see the +# copilot-otel-grafana-db notes below. # # Prerequisites, credential handling, and first-run steps: # references/local-setup.md. @@ -60,7 +62,11 @@ services: GF_AUTH_ANONYMOUS_ENABLED: "false" GF_AUTH_DISABLE_LOGIN_FORM: "false" GF_AUTH_BASIC_ENABLED: "true" - # Required with no default; the :? form fails the up with this message. + # The :? form fails the `up` when the variable is unset. That is all it + # does. Grafana reads GF_SECURITY_ADMIN_* only when it initializes its + # database, so on a database that already exists the value is supplied + # and then ignored. The separate copilot-otel-grafana-db volume below is + # what makes this pair actually take effect. # validate_dashboard.py reads the same pair. GF_SECURITY_ADMIN_USER: ${COPILOT_OTEL_GRAFANA_USER:?set COPILOT_OTEL_GRAFANA_USER before starting the stack} GF_SECURITY_ADMIN_PASSWORD: ${COPILOT_OTEL_GRAFANA_PASSWORD:?set COPILOT_OTEL_GRAFANA_PASSWORD before starting the stack} @@ -75,6 +81,14 @@ services: # Everything persistent lives under /data: grafana, prometheus, tempo, # loki, pyroscope. - copilot-otel-data:/data + # Grafana's database, mounted over GF_PATHS_DATA so it is not part of the + # reusable telemetry volume. Grafana applies GF_SECURITY_ADMIN_USER and + # GF_SECURITY_ADMIN_PASSWORD only when it creates grafana.db; on a reused + # /data volume it would find an existing database and keep whatever + # password that database already carries, possibly the image default. + # Giving the database its own project-managed volume means a new stack + # initializes it, so the configured credential is the live one. + - copilot-otel-grafana-db:/data/grafana/data healthcheck: test: [ @@ -95,3 +109,7 @@ networks: volumes: copilot-otel-data: external: true + # Deliberately not external. Compose creates it per project, which is what + # produces the Grafana first run that adopts the configured credential. + # Dashboards and Grafana users live here rather than in copilot-otel-data. + copilot-otel-grafana-db: {} diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py index 0d522383f..4435741ae 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/settings_upsert.py @@ -161,10 +161,16 @@ def check_policy( ) -> None: """Apply the rules the schema alone cannot express. - Cross-setting rules are evaluated against the settings that would result, - not against this invocation's arguments. Setting `exporterType` alone - otherwise passes while producing a file that contradicts the `outfile` - already written beside it. + Every rule is evaluated against the settings that would result, not against + this invocation's arguments. Two invocations that each pass alone can + otherwise combine into a document no invocation ever validated: setting + `exporterType` while the file already holds an off-policy `otlpEndpoint` + writes that endpoint's document without ever checking it. + + A pre-existing value that fails policy is refused rather than carried + through, and the refusal says the value was already there. That is the same + posture `outfile` and `maxAttributeSizeChars` already had; `captureContent` + and `otlpEndpoint` were the two rules that looked only at the invocation. """ merged = {**(existing or {}), **values} @@ -174,13 +180,21 @@ def check_policy( "instructions, and tool arguments onto spans. Set it by hand, deliberately." ) + if merged.get("github.copilot.chat.otel.captureContent") is True: + raise SettingsError( + "refusing to write while captureContent is already enabled in this file. This " + "invocation did not set it, but the document it would produce still puts prompt " + "text, responses, system instructions, and tool arguments onto spans. Turn it off, " + "or make this edit by hand." + ) + exporter = merged.get("github.copilot.chat.otel.exporterType") if exporter is not None and exporter not in EXPORTER_TYPES: raise SettingsError( f"exporterType must be one of {', '.join(sorted(EXPORTER_TYPES))}, got '{exporter}'" ) - endpoint = values.get("github.copilot.chat.otel.otlpEndpoint") + endpoint = merged.get("github.copilot.chat.otel.otlpEndpoint") if endpoint: try: check_url( @@ -189,7 +203,13 @@ def check_policy( allowed_ports=DEFAULT_ALLOWED_PORTS, ) except PolicyError as exc: - raise SettingsError(f"otlpEndpoint rejected: {exc}") from exc + if "github.copilot.chat.otel.otlpEndpoint" in values: + raise SettingsError(f"otlpEndpoint rejected: {exc}") from exc + raise SettingsError( + "refusing to write: this file already holds an otlpEndpoint that this " + f"invocation did not set, and it fails policy ({exc}). Correct or remove " + "that value first." + ) from exc outfile = merged.get("github.copilot.chat.otel.outfile") if outfile: diff --git a/.github/skills/experimental/copilot-otel-metrics/examples/verify.py b/.github/skills/experimental/copilot-otel-metrics/examples/verify.py index 78dfa499d..fd171ebe2 100644 --- a/.github/skills/experimental/copilot-otel-metrics/examples/verify.py +++ b/.github/skills/experimental/copilot-otel-metrics/examples/verify.py @@ -17,9 +17,13 @@ from __future__ import annotations +import base64 import json +import os import time +import urllib.error import urllib.parse +import urllib.request from _input_policy import open_url @@ -27,6 +31,12 @@ PROM = "http://localhost:9090" TEMPO = "http://localhost:3200" +# The credential grafana/otel-lgtm ships with. Grafana sets the admin password +# only when it creates its database, so a stack started against a pre-existing +# database keeps this pair even though COPILOT_OTEL_GRAFANA_PASSWORD was set. +GRAFANA_DEFAULT_USER = "admin" +GRAFANA_DEFAULT_PASSWORD = "admin" + COPILOT_SERVICES = ("copilot-chat", "github-copilot", "claude-code") results: list[tuple[str, bool, str]] = [] @@ -68,6 +78,69 @@ def check_delta_flag() -> None: record("delta-to-cumulative enabled", False, str(exc)) +def grafana_accepts(user: str, password: str) -> bool | None: + """Whether Grafana accepts the pair, or None when the probe got no answer. + + A rejected credential and an unreachable Grafana call for different + operator actions, so they are not collapsed into one boolean. + """ + token = base64.b64encode(f"{user}:{password}".encode()).decode("ascii") + request = urllib.request.Request(f"{GRAFANA}/api/org") + request.add_header("Authorization", f"Basic {token}") + try: + with open_url(request, timeout=10) as resp: + return resp.status == 200 + except urllib.error.HTTPError as exc: + if exc.code in (401, 403): + return False + return None + except Exception: # noqa: BLE001 - an unanswered probe is not a rejection + return None + + +def check_grafana_credentials() -> None: + """Report which admin credential is live, not only whether yours works. + + "My password is rejected" is a configuration problem. "The image default + still authenticates" is a disclosure problem, because Grafana holds the + dashboards over stored prompt content. The second is the one worth failing + the run for, and a check that only tried the configured pair would report + it as a mismatch. + """ + user = os.environ.get("COPILOT_OTEL_GRAFANA_USER") + password = os.environ.get("COPILOT_OTEL_GRAFANA_PASSWORD") + if user and password: + accepted = grafana_accepts(user, password) + detail = { + True: "Grafana accepted the configured pair", + False: "Grafana rejected it; its database predates this password", + None: "no answer from Grafana", + }[accepted] + record("configured grafana credential works", accepted is True, detail) + else: + record( + "configured grafana credential works", + False, + "COPILOT_OTEL_GRAFANA_USER and COPILOT_OTEL_GRAFANA_PASSWORD are not both set", + ) + + if (user, password) == (GRAFANA_DEFAULT_USER, GRAFANA_DEFAULT_PASSWORD): + record( + "grafana default credential inactive", + False, + "the configured credential is the image default admin/admin", + ) + return + + live = grafana_accepts(GRAFANA_DEFAULT_USER, GRAFANA_DEFAULT_PASSWORD) + detail = { + True: "admin/admin authenticates; reset it, see examples/README.md", + False: "admin/admin is rejected", + None: "no answer from Grafana", + }[live] + record("grafana default credential inactive", live is False, detail) + + def check_copilot_metrics() -> None: try: names = api(PROM, "/api/v1/label/__name__/values")["data"] @@ -125,6 +198,7 @@ def check_copilot_traces() -> None: def main() -> int: check_health() check_delta_flag() + check_grafana_credentials() check_copilot_metrics() check_copilot_traces() @@ -133,15 +207,28 @@ def main() -> int: for name, ok, detail in results: print(f" [{'PASS' if ok else 'FAIL'}] {name:<{width}} {detail}") - # Health and the delta flag are infrastructure and must pass. Signal presence - # depends on the editor having exported something, which is reported but does - # not by itself indicate a broken stack. - required = [ok for name, ok, _ in results if "reachable" in name or "delta" in name] + # Health, the delta flag, and a dead default credential are infrastructure + # and must pass. A rejected configured credential is reported but does not + # fail the run, because running this without exporting the pair is a + # reasonable thing to do and says nothing about the stack. Signal presence + # depends on the editor having exported something. + required = [ + ok + for name, ok, _ in results + if "reachable" in name or "delta" in name or "default credential" in name + ] signals = [ok for name, ok, _ in results if "copilot" in name] print() if not all(required): - print("RESULT: stack is not healthy. Start it with: docker compose up -d") + # A stopped stack also fails the credential probe, so health is reported + # first rather than sending the operator to the rotation procedure. + if all(ok for name, ok, _ in results if "reachable" in name or "delta" in name): + print("RESULT: the Grafana admin credential you configured is not the live one.") + print(" See 'If you already ran an older version of this stack' in") + print(" examples/README.md.") + else: + print("RESULT: stack is not healthy. Start it with: docker compose up -d") return 1 if not any(signals): print("RESULT: stack healthy, but no Copilot signals stored yet.") diff --git a/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md b/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md index 079f17f43..7ac56fccb 100644 --- a/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md +++ b/.github/skills/experimental/copilot-otel-metrics/references/local-stack.md @@ -54,10 +54,11 @@ Seed from `examples/compose.yaml` and `examples/otel-collector-local.yaml`. Seve * **The Collector is the only OTLP listener published on the host.** LGTM's own `4317` and `4318` are deliberately not mapped. This is the difference between filtering telemetry and merely hoping it is filtered: if LGTM published OTLP itself, any local process could write straight past the allow-list. Loopback binding does not prevent that, because every process on the machine is already on loopback. * **The attribute policy is an allow-list, not a delete-list.** `allow_all_keys: false` means an attribute the configuration has never seen is dropped. The emitter's attribute set is not a stable contract, so a delete-list is only correct until the next extension release adds a content-bearing key. Adding a key to `allowed_keys` is a deliberate decision to store it. -* **Grafana credentials are required with no default.** The image enables anonymous access with the Admin role and hides the login form, so left alone every stored prompt fragment is readable by anything that can reach port 3000. The compose file disables anonymous access, restores the login form, and requires `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD` through the `${VAR:?message}` form, which fails the `up` rather than starting on a guessable credential. +* **Grafana credentials are required with no default.** The image enables anonymous access with the Admin role and hides the login form, so left alone every stored prompt fragment is readable by anything that can reach port 3000. The compose file disables anonymous access, restores the login form, and requires `COPILOT_OTEL_GRAFANA_USER` and `COPILOT_OTEL_GRAFANA_PASSWORD` through the `${VAR:?message}` form. Be precise about what that form buys: it fails the `up` when a variable is unset. It does not make Grafana adopt the value. +* **Grafana's database has its own volume.** `copilot-otel-grafana-db` mounts over `/data/grafana/data`, which is `GF_PATHS_DATA` in this image. Grafana applies `GF_SECURITY_ADMIN_USER` and `GF_SECURITY_ADMIN_PASSWORD` only when it creates `grafana.db`, so on a stack sharing the reusable `copilot-otel-data` volume the required variables would be supplied and then ignored, leaving whatever password that database already carried β€” possibly the image default of `admin`/`admin`. This volume is deliberately not `external`, because the per-project creation is what produces the first run that adopts the credential. The cost is that Grafana users and dashboards do not carry over from an older shared volume; telemetry history does, because it stays on `copilot-otel-data`. * **Both images are pinned by digest.** A tag is mutable, so a tag-pinned stack can change underneath a configuration that was reviewed. The tag is kept in a comment above each digest so the version stays legible. * **Every port binds `127.0.0.1`.** The OTLP endpoint accepts writes from anyone who can reach it. Loopback binding is what makes that acceptable. Publishing `4318` on all interfaces, as some published walkthroughs do, exposes an unauthenticated ingest endpoint on the network; OpenTelemetry's own security guidance flags that pattern under CWE-1327. -* **The data volume is declared `external`.** Compose namespaces volumes by project. A plain declaration creates a second, empty, project-prefixed volume and silently orphans everything already collected. This is why the volume is created by hand first. +* **The telemetry volume is declared `external`.** Compose namespaces volumes by project. A plain declaration creates a second, empty, project-prefixed volume and silently orphans everything already collected. This is why the volume is created by hand first. * **`--enable-feature=otlp-deltatocumulative` is set.** Prometheus drops delta-temporality metrics by default, and a dropped delta metric was observed failing an entire batched write, losing unrelated cumulative metrics that shared the batch. The flag converts instead of dropping and is inert when traffic is already cumulative. Its per-series state lives in memory and resets when the container restarts. * **Retention is raised to 120 days.** Prometheus defaults to 15, which silently truncates any monthly total into a smaller number that still looks plausible. @@ -84,11 +85,11 @@ Two variants, and the difference is the entire history. docker compose -f compose.yaml down # Stop the stack and destroy all history -docker compose -f compose.yaml down +docker compose -f compose.yaml down -v docker volume rm copilot-otel-data ``` -`down` alone leaves the external volume intact, so a later `up -d` restores the stack with its data. Only the explicit `docker volume rm` discards it. Say which one the user asked for before handing over either. +`down` alone leaves both volumes intact, so a later `up -d` restores the stack with its data and its Grafana users. `down -v` removes the project-managed `copilot-otel-grafana-db`, which discards Grafana users and dashboards but not telemetry; only the explicit `docker volume rm copilot-otel-data` discards the telemetry, because that volume is external. Say which one the user asked for before handing over either. ## The local dashboard diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py index e7ad1306c..3b93ab7ff 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py @@ -215,6 +215,49 @@ def test_open_url_refuses_before_opening_anything(self, monkeypatch) -> None: assert opened == [], "a refused URL still reached the opener" +class TestProxyNeutralization: + """A permitted loopback request is not routed through an ambient proxy. + + `build_opener` adds to the default handler chain, so the default + `ProxyHandler` is installed unless an empty one is passed explicitly. + `proxy_bypass` does not exempt loopback, so without that the request goes + to whatever `HTTP_PROXY` names -- carrying the Grafana Basic credential + that `validate_dashboard` attaches. `no_proxy` is cleared here on purpose: + an environment where it already covers loopback would pass either way. + """ + + def _built_opener(self, monkeypatch) -> urllib.request.OpenerDirector: + monkeypatch.setenv("HTTP_PROXY", "http://proxy.invalid:3128") + monkeypatch.setenv("http_proxy", "http://proxy.invalid:3128") + monkeypatch.setenv("NO_PROXY", "") + monkeypatch.setenv("no_proxy", "") + captured: list[urllib.request.OpenerDirector] = [] + monkeypatch.setattr( + urllib.request.OpenerDirector, + "open", + lambda self, *a, **k: captured.append(self), + ) + open_url("http://127.0.0.1:3000/api/health") + return captured[0] + + def test_the_built_opener_carries_no_proxy_configuration(self, monkeypatch) -> None: + opener = self._built_opener(monkeypatch) + configured = { + scheme: target + for handler in opener.handlers + if isinstance(handler, urllib.request.ProxyHandler) + for scheme, target in handler.proxies.items() + } + assert configured == {}, f"open_url would proxy loopback traffic to {configured}" + + def test_the_built_opener_keeps_the_policy_redirect_handler(self, monkeypatch) -> None: + policy_handler = open_url.__globals__["_PolicyRedirectHandler"] + opener = self._built_opener(monkeypatch) + assert any(isinstance(handler, policy_handler) for handler in opener.handlers), ( + "the redirect allow-list re-check and cross-origin credential stripping were dropped" + ) + + class TestPathContainment: """A configurable path cannot escape its root.""" @@ -451,6 +494,91 @@ def _json_response(payload: dict) -> io.BytesIO: } +class TestGrafanaCredentialLiveness: + """`verify.py` reports which admin credential is live, not only whether ours works. + + Grafana applies `GF_SECURITY_ADMIN_*` only when it creates its database, so + a stack on a pre-existing database can be running on `admin`/`admin` while + the configured pair was supplied and ignored. A check that tried only the + configured pair would report that as a mismatch rather than as a live + default credential, which is the condition worth failing the run for. + """ + + @pytest.fixture(autouse=True) + def _isolated_results(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(verify, "results", []) + + def _probe(self, monkeypatch: pytest.MonkeyPatch, answers: dict) -> None: + monkeypatch.setattr( + verify, "grafana_accepts", lambda user, password: answers.get((user, password), False) + ) + + def _outcome(self, name: str) -> tuple[bool, str]: + return next((ok, detail) for recorded, ok, detail in verify.results if recorded == name) + + def _configure(self, monkeypatch: pytest.MonkeyPatch, user: str, password: str) -> None: + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", user) + monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", password) + + def test_an_adopted_credential_passes_both_checks( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._configure(monkeypatch, "operator", "chosen-by-the-user") + self._probe(monkeypatch, {("operator", "chosen-by-the-user"): True}) + verify.check_grafana_credentials() + assert self._outcome("configured grafana credential works")[0] is True + assert self._outcome("grafana default credential inactive")[0] is True + + def test_a_live_default_credential_is_reported_as_such( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._configure(monkeypatch, "operator", "chosen-by-the-user") + self._probe(monkeypatch, {("admin", "admin"): True}) + verify.check_grafana_credentials() + ok, detail = self._outcome("grafana default credential inactive") + assert ok is False + assert "admin/admin authenticates" in detail + + def test_a_rejected_configured_credential_is_reported_separately( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._configure(monkeypatch, "operator", "chosen-by-the-user") + self._probe(monkeypatch, {("admin", "admin"): True}) + verify.check_grafana_credentials() + ok, detail = self._outcome("configured grafana credential works") + assert ok is False + assert "database predates this password" in detail + + def test_an_unanswered_probe_is_not_reported_as_a_rejection( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._configure(monkeypatch, "operator", "chosen-by-the-user") + monkeypatch.setattr(verify, "grafana_accepts", lambda user, password: None) + verify.check_grafana_credentials() + assert "no answer from Grafana" in self._outcome("configured grafana credential works")[1] + + def test_missing_configuration_is_named_rather_than_probed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("COPILOT_OTEL_GRAFANA_USER", raising=False) + monkeypatch.delenv("COPILOT_OTEL_GRAFANA_PASSWORD", raising=False) + self._probe(monkeypatch, {}) + verify.check_grafana_credentials() + ok, detail = self._outcome("configured grafana credential works") + assert ok is False + assert "are not both set" in detail + + def test_configuring_the_default_pair_fails_the_default_check( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + self._configure(monkeypatch, "admin", "admin") + self._probe(monkeypatch, {("admin", "admin"): True}) + verify.check_grafana_credentials() + ok, detail = self._outcome("grafana default credential inactive") + assert ok is False + assert "is the image default" in detail + + class TestGrafanaCredentialScope: """The Grafana credential reaches Grafana and nothing else. diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py index b3076347d..b58716e85 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py @@ -174,6 +174,74 @@ def test_an_existing_unknown_exporter_is_reported(self) -> None: with pytest.raises(SettingsError, match="exporterType"): check_policy({ENABLED: True}, existing={EXPORTER: "otlp-quic"}) + def test_an_existing_off_policy_endpoint_is_refused(self) -> None: + with pytest.raises(SettingsError, match="already holds an otlpEndpoint"): + check_policy( + {EXPORTER: "otlp-http"}, existing={ENDPOINT: "http://evil.example.com:4318"} + ) + + def test_replacing_an_off_policy_endpoint_in_the_same_run_is_allowed(self) -> None: + check_policy( + {ENDPOINT: "http://localhost:4318"}, + existing={ENDPOINT: "http://evil.example.com:4318"}, + ) + + def test_an_already_enabled_content_capture_is_refused(self) -> None: + with pytest.raises(SettingsError, match="already enabled"): + check_policy({ENABLED: True}, existing={CAPTURE: True}) + + def test_turning_an_enabled_content_capture_off_in_the_same_run_is_allowed(self) -> None: + check_policy({CAPTURE: False}, existing={CAPTURE: True}) + + +class TestSplitInvocationWrites: + """What reaches disk is validated, not just the arguments of the run writing it. + + This is the shape a single-invocation test cannot reach. The off-policy + value is already in the file -- hand-edited, or written before this policy + existed -- and the current run's own argument is clean. Checking only the + argument writes the combined document without ever judging it. + + These go through `apply_changes` rather than `check_policy` because the + claim is about the write path, and `apply_changes` is the only caller of + `write_atomically`, which is the only writer of the settings document. + """ + + def test_a_clean_edit_cannot_write_past_an_off_policy_endpoint_already_in_the_file( + self, settings_file: pathlib.Path + ) -> None: + settings_file.write_text( + json.dumps({ENDPOINT: "http://evil.example.com:4318"}, indent=2), encoding="utf-8" + ) + with pytest.raises(SettingsError, match="already holds an otlpEndpoint"): + apply_changes(settings_file, {ENABLED: True}, apply=True) + + def test_the_refused_write_leaves_the_document_byte_identical( + self, settings_file: pathlib.Path + ) -> None: + original = json.dumps({ENDPOINT: "http://evil.example.com:4318"}, indent=2) + settings_file.write_text(original, encoding="utf-8") + with pytest.raises(SettingsError): + apply_changes(settings_file, {ENABLED: True}, apply=True) + assert settings_file.read_text(encoding="utf-8") == original + + def test_a_clean_edit_cannot_write_past_content_capture_already_in_the_file( + self, settings_file: pathlib.Path + ) -> None: + settings_file.write_text(json.dumps({CAPTURE: True}, indent=2), encoding="utf-8") + with pytest.raises(SettingsError, match="already enabled"): + apply_changes(settings_file, {ENABLED: True}, apply=True) + + def test_correcting_the_offending_value_in_the_same_run_is_allowed( + self, settings_file: pathlib.Path + ) -> None: + settings_file.write_text( + json.dumps({ENDPOINT: "http://evil.example.com:4318"}, indent=2), encoding="utf-8" + ) + apply_changes(settings_file, {ENDPOINT: "http://localhost:4318"}, apply=True) + written = json.loads(settings_file.read_text(encoding="utf-8")) + assert written[ENDPOINT] == "http://localhost:4318" + class TestOutfilePolicy: """Captured spans may not land somewhere nobody chose.""" From 7afb1dbe63b6765c29c4c9b956e17b21739d6cc8 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:08:56 -0700 Subject: [PATCH 20/28] refactor(plugins)!: remove hooks support from the plugin manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - delete the fixed hook path, guarded emission, and path validation - report any unsupported committed field as drift instead - generalize plugin, release, and docs fixtures away from the hook 🧹 - Generated by Copilot --- .github/copilot-instructions.md | 8 +- .../__tests__/pluginManifestCards.test.ts | 16 +- .../__tests__/resolvePluginManifest.test.ts | 3 +- .../src/data/pluginManifestCards.ts | 4 +- scripts/plugins/README.md | 8 +- scripts/plugins/Sync-PluginManifest.ps1 | 177 ++++++++++++----- .../plugins/Sync-PluginManifest.Tests.ps1 | 181 +++++++++++------- .../release/Set-RepositoryVersion.Tests.ps1 | 3 +- 8 files changed, 266 insertions(+), 134 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 018283c3d..c00530ce9 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -54,7 +54,6 @@ The project is organized into these main areas: * Documentation (`docs/`) - Getting started guides, templates, RPI workflow documentation, and contribution guidelines. * Scripts (`scripts/`) - Automation for linting, security validation, extension packaging, and development tools. * Skills (`.github/skills/{package-id}/`) - Self-contained skill packages, by convention organized by package. -* Hooks (`.github/hooks/{package-id}/`) - Package-scoped Copilot hook manifests (JSON) that wire lifecycle event commands. * Extension (`extension/`) - VS Code extension source and packaging. * GitHub Configuration (`.github/`) - Workflows, instructions, prompts, agents, composite actions, and issue templates, typically organized into `{package-id}` subdirectories. * Plugin manifest (`plugin.json`) - Deterministic membership and metadata for the sole `hve-core` plugin. @@ -141,12 +140,12 @@ Parent agents reference subagents using glob paths like `.github/agents/**/code- The plugin manifest owns plugin and VSIX composition: -* Root `plugin.json` owns the complete `hve-core` component membership: package-scoped agents, prompts, instructions, and distributable skills discovered from tracked `.github` paths, plus the fixed telemetry hook. `.github/plugin/marketplace.json` contains one relative locator to the repository root and no component recipe. -* After adding, changing, moving, or removing a distributable artifact, run `npm run plugin:sync` to update the manifest. Run `npm run plugin:validate` to check manifest drift, marketplace parity and containment, component coverage, and hooks. +* Root `plugin.json` owns the complete `hve-core` component membership: package-scoped agents, prompts, instructions, and distributable skills discovered from tracked `.github` paths. The manifest declares no hooks. Hooks support was removed with the telemetry hook, so a committed `hooks` field is reported as drift rather than preserved. `.github/plugin/marketplace.json` contains one relative locator to the repository root and no component recipe. +* After adding, changing, moving, or removing a distributable artifact, run `npm run plugin:sync` to update the manifest. Run `npm run plugin:validate` to check manifest drift, marketplace parity and containment, and component coverage. * The installable plugin root is the repository root. Artifact discovery remains limited to package-scoped `.github` paths; do not materialize a copied plugin tree or create a repository-root `plugins/` directory. * Run `npm run extension:prepare` or `npm run extension:prepare:prerelease` to refresh the single `extension/package.json` and `extension/README.md`. Stable and PreRelease contain the same component set. * After adding, changing, moving, or removing a documentable agent, prompt, instruction, or skill, run `npm run docs:generate` and commit the matching page under `docs/reference/`. The generator owns page frontmatter and the prefix through ``; edit only the preserved `When to use it`, applicable `How to use it`, and `Example usage` tail. Do not edit generated regions or catalog indexes by hand. -* Run `npm run plugin:validate` to confirm the manifest, one-entry locator, component coverage, and hooks are correct. +* Run `npm run plugin:validate` to confirm the manifest, one-entry locator, and component coverage are correct. @@ -168,7 +167,6 @@ Commit message scopes map to repository directories: * `(prompts)` = `.github/prompts/` * `(instructions)` = `.github/instructions/` * `(skills)` = `.github/skills/` -* `(hooks)` = `.github/hooks/` * `(templates)` = `.github/ISSUE_TEMPLATE/` * `(workflows)` = `.github/workflows/` * `(extension)` = `extension/` diff --git a/docs/docusaurus/src/data/__tests__/pluginManifestCards.test.ts b/docs/docusaurus/src/data/__tests__/pluginManifestCards.test.ts index 3d8e5ae1f..23d4058e4 100644 --- a/docs/docusaurus/src/data/__tests__/pluginManifestCards.test.ts +++ b/docs/docusaurus/src/data/__tests__/pluginManifestCards.test.ts @@ -32,7 +32,7 @@ const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8")) as PluginMan // Independent re-derivation of the component tally so expected values come from // the manifest itself rather than from the function under test or a magic number. -const componentFields = ["agents", "commands", "rules", "skills", "hooks"]; +const componentFields = ["agents", "commands", "rules", "skills"]; function expectedComponentCount(entry: PluginManifest): number { return componentFields.reduce((total, field) => { const value = entry[field]; @@ -60,7 +60,7 @@ describe("loadPackageCards against the canonical plugin manifest", () => { describe("countPluginComponents", () => { it("counts a string component field as one component", () => { expect( - countPluginComponents({ hooks: "hooks/shared/telemetry.json" }), + countPluginComponents({ skills: ".github/skills/rpi/rpi-plan" }), ).toBe(1); }); @@ -77,9 +77,17 @@ describe("countPluginComponents", () => { commands: ["c.md"], rules: [], skills: ["s"], + }), + ).toBe(4); + }); + + it("ignores a retired field the manifest no longer declares", () => { + expect( + countPluginComponents({ + agents: ["a.md"], hooks: "hooks/shared/telemetry.json", }), - ).toBe(5); + ).toBe(1); }); it("ignores fields that are neither a string nor an array", () => { @@ -96,4 +104,4 @@ describe("countPluginComponents", () => { expect(countPluginComponents(manifest)).toBe(expectedComponentCount(manifest)); expect(countPluginComponents(manifest)).toBeGreaterThan(0); }); -}); \ No newline at end of file +}); diff --git a/docs/docusaurus/src/data/__tests__/resolvePluginManifest.test.ts b/docs/docusaurus/src/data/__tests__/resolvePluginManifest.test.ts index 49adf89aa..b5141706b 100644 --- a/docs/docusaurus/src/data/__tests__/resolvePluginManifest.test.ts +++ b/docs/docusaurus/src/data/__tests__/resolvePluginManifest.test.ts @@ -36,7 +36,6 @@ function manifest(overrides: JsonObject = {}): JsonObject { commands: ['.github/commands/one.prompt.md'], rules: [], skills: ['.github/skills/one'], - hooks: '.github/hooks/shared/telemetry.json', ...overrides, }; } @@ -70,7 +69,7 @@ describe('loadPackageCards plugin manifest resolution', () => { name: 'hve-core', title: 'HVE Core', description: 'HVE Core description', - artifacts: 5, + artifacts: 4, maturity: 'Stable', href: '/docs/plugins/hve-core', }]); diff --git a/docs/docusaurus/src/data/pluginManifestCards.ts b/docs/docusaurus/src/data/pluginManifestCards.ts index 481e5ca79..d522062c7 100644 --- a/docs/docusaurus/src/data/pluginManifestCards.ts +++ b/docs/docusaurus/src/data/pluginManifestCards.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import type { PackageCardData } from './packageCards'; import { labelRegistry } from './labelRegistry'; -const componentFields = ['agents', 'commands', 'rules', 'skills', 'hooks']; +const componentFields = ['agents', 'commands', 'rules', 'skills']; const errorPrefix = '[pluginManifestCards]'; export function countPluginComponents( @@ -119,4 +119,4 @@ export function loadPackageCards(pluginLocatorPath: string): PackageCardData[] { maturity: labelRegistry.stable, href: `/docs/plugins/${name}`, }]; -} \ No newline at end of file +} diff --git a/scripts/plugins/README.md b/scripts/plugins/README.md index d5e26d84d..d32c3059b 100644 --- a/scripts/plugins/README.md +++ b/scripts/plugins/README.md @@ -11,7 +11,7 @@ PowerShell tooling for synchronizing root `plugin.json` with the complete distri |--------------------------------|--------------------------------|---------------------------------------------------| | Sync-PluginManifest.ps1 | `npm run plugin:sync` | Write deterministic membership and locator parity | | Sync-PluginManifest.ps1 -Check | `npm run lint:plugin-manifest` | Fail on manifest or marketplace locator drift | -| Aggregate validation | `npm run plugin:validate` | Check the manifest, locator, and hook declaration | +| Aggregate validation | `npm run plugin:validate` | Check the manifest and locator | ## Prerequisites @@ -20,9 +20,9 @@ PowerShell tooling for synchronizing root `plugin.json` with the complete distri ## Source to Manifest Pipeline -1. Author artifacts in `.github/` (agents, prompts, instructions, skills, hooks) +1. Author artifacts in `.github/` (agents, prompts, instructions, skills) 2. Run `npm run plugin:sync` to derive root `plugin.json` -3. Run `npm run plugin:validate` for non-mutating manifest and hook validation +3. Run `npm run plugin:validate` for non-mutating manifest validation 4. Prepare or package the single extension separately when VSIX output is in scope ## Manifest Output @@ -34,7 +34,7 @@ The script discovers these git-tracked paths beneath `.github` and emits reposit * `instructions//**/*.instructions.md` * `skills///SKILL.md`, unless the top-level license has a noncommercial qualifier -Artifacts without a package segment are excluded from discovery. Paths are unique and ordinal-sorted. Metadata is preserved, the version follows root `package.json`, and `.github/hooks/shared/telemetry.json` remains fixed. Validation also requires tracked, regular, non-empty root `plugin.json`, `README.md`, and `LICENSE` files. +Artifacts without a package segment are excluded from discovery. Paths are unique and ordinal-sorted. Metadata is preserved and the version follows root `package.json`. The manifest declares no `hooks`: support for it was removed with the telemetry hook, so a committed `hooks` field (or any other field the manifest cannot derive) is reported as drift rather than preserved. Validation also requires tracked, regular, non-empty root `plugin.json`, `README.md`, and `LICENSE` files. ## Synchronizing After Artifact Changes diff --git a/scripts/plugins/Sync-PluginManifest.ps1 b/scripts/plugins/Sync-PluginManifest.ps1 index 97848daef..1ce11e5e3 100644 --- a/scripts/plugins/Sync-PluginManifest.ps1 +++ b/scripts/plugins/Sync-PluginManifest.ps1 @@ -19,9 +19,10 @@ carries a noncommercial qualifier Repository-root artifacts, which have no package segment, are excluded. - Paths are unique and ordinal-sorted, manifest metadata is preserved, the - version follows the root package.json, and the hook declaration is emitted - only while the hook manifest exists under the plugin root. + Paths are unique and ordinal-sorted, manifest metadata is preserved, and the + version follows the root package.json. The manifest declares no hooks: hooks + support was removed with the telemetry hook, so a committed hooks field is + reported as drift rather than preserved. Check mode writes nothing. It reports manifest drift and validates the one-entry marketplace catalog: exactly one entry, name and version parity, @@ -58,11 +59,16 @@ $ErrorActionPreference = 'Stop' $script:ArtifactRoot = '.github' $script:PluginManifestFile = 'plugin.json' $script:MarketplaceCatalogFile = '.github/plugin/marketplace.json' -$script:FixedHookPath = '.github/hooks/shared/telemetry.json' $script:RequiredPluginMetadata = @('plugin.json', 'README.md', 'LICENSE') $script:ManifestMetadataKey = @('author', 'homepage', 'repository', 'license', 'keywords') $script:LocatorMetadataKey = @('name', 'description', 'version', 'author', 'homepage', 'repository', 'license', 'keywords') +# Fields the marketplace locator must never declare. 'hooks' stays listed after +# the manifest stopped supporting it, because a retired field is exactly what +# this deny-list exists to reject. $script:RecipeField = @('agents', 'commands', 'rules', 'skills', 'hooks', 'x-hve') +# Every field the derived manifest can produce. A committed field outside this +# set is drift, which is what keeps a removed field from returning silently. +$script:SupportedManifestField = @('name', 'description', 'version') + $script:ManifestMetadataKey + @('agents', 'commands', 'rules', 'skills') #region Discovery @@ -71,12 +77,17 @@ function Get-TrackedPluginIndex { .SYNOPSIS Reads tracked plugin paths and rejects symbolic-link index entries. + .DESCRIPTION + One staged index read serves both component discovery and root metadata + validation. Mode maps every returned repository-relative path, including + the required root files, to its exact git index mode. + .PARAMETER RepoRoot Root directory of the repository. .OUTPUTS - [System.Collections.Specialized.OrderedDictionary] Paths and violation - messages derived from the git index. + [System.Collections.Specialized.OrderedDictionary] Paths, violation + messages, and index modes derived from the git index. #> [CmdletBinding()] [OutputType([System.Collections.Specialized.OrderedDictionary])] @@ -86,7 +97,7 @@ function Get-TrackedPluginIndex { [string]$RepoRoot ) - $output = & git -C $RepoRoot ls-files -s -z --full-name -- $script:ArtifactRoot + $output = & git -C $RepoRoot ls-files -s -z --full-name -- $script:ArtifactRoot $script:RequiredPluginMetadata if ($LASTEXITCODE -ne 0) { throw "git ls-files failed with exit code $LASTEXITCODE in $RepoRoot" } @@ -94,12 +105,14 @@ function Get-TrackedPluginIndex { $prefix = "$script:ArtifactRoot/" $paths = @() $violations = @() + $mode = [System.Collections.Generic.Dictionary[string, string]]::new([System.StringComparer]::Ordinal) foreach ($entry in (($output -join '') -split "`0" | Where-Object { $_ })) { if ($entry -notmatch '^(?\d{6}) [0-9a-f]+ \d+\t(?.+)$') { throw "Unexpected git ls-files entry: $entry" } $path = $Matches['Path'] + $mode[$path] = $Matches['Mode'] if (-not $path.StartsWith($prefix, [System.StringComparison]::Ordinal)) { continue } @@ -115,6 +128,7 @@ function Get-TrackedPluginIndex { return [ordered]@{ Paths = @($paths) Violations = @($violations) + Mode = $mode } } @@ -316,8 +330,8 @@ function New-PluginManifest { .DESCRIPTION Metadata is preserved from the committed manifest, the version follows - the root package manifest, component arrays are derived, and the hook - declaration is emitted only while the hook manifest exists. + the root package manifest, and component arrays are derived. No hooks + declaration is emitted. .PARAMETER RepoRoot Root directory of the repository. @@ -370,9 +384,6 @@ function New-PluginManifest { foreach ($kind in @('agents', 'commands', 'rules', 'skills')) { $manifest[$kind] = @($Component[$kind]) } - if (Test-Path -LiteralPath (Join-Path $RepoRoot $script:FixedHookPath) -PathType Leaf) { - $manifest['hooks'] = $script:FixedHookPath - } return $manifest } @@ -425,7 +436,7 @@ function Compare-PluginManifest { ) $differences = @() - foreach ($field in @('version', 'hooks')) { + foreach ($field in @('version')) { $committedValue = if ($null -ne $Committed -and $Committed.Contains($field)) { [string]$Committed[$field] } else { '' } $expectedValue = if ($Expected.Contains($field)) { [string]$Expected[$field] } else { '' } if ($committedValue -cne $expectedValue) { @@ -433,6 +444,14 @@ function Compare-PluginManifest { } } + if ($null -ne $Committed) { + foreach ($field in @($Committed.Keys)) { + if ($script:SupportedManifestField -cnotcontains $field) { + $differences += "$field is declared but the manifest no longer supports it; remove it" + } + } + } + foreach ($kind in @('agents', 'commands', 'rules', 'skills')) { $committedPaths = @() if ($null -ne $Committed -and $Committed.Contains($kind)) { @@ -466,6 +485,10 @@ function Get-PluginMetadataViolations { .PARAMETER RepoRoot Root directory of the repository. + .PARAMETER TrackedMode + Repository-relative path to git index mode map. Defaults to a fresh + index read so direct callers need no snapshot. + .OUTPUTS [string[]] Violation messages. #> @@ -474,9 +497,16 @@ function Get-PluginMetadataViolations { param( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] - [string]$RepoRoot + [string]$RepoRoot, + + [Parameter(Mandatory = $false)] + [System.Collections.Generic.IDictionary[string, string]]$TrackedMode ) + if (-not $TrackedMode) { + $TrackedMode = (Get-TrackedPluginIndex -RepoRoot $RepoRoot).Mode + } + $violations = @() foreach ($relativePath in $script:RequiredPluginMetadata) { $path = Join-Path $RepoRoot $relativePath @@ -485,17 +515,13 @@ function Get-PluginMetadataViolations { continue } - $indexOutput = & git -C $RepoRoot ls-files -s --full-name -- $relativePath - if ($LASTEXITCODE -ne 0) { - throw "git ls-files failed with exit code $LASTEXITCODE in $RepoRoot" - } - $entry = @($indexOutput | Where-Object { $_ }) - if ($entry.Count -ne 1 -or $entry[0] -notmatch '^(?\d{6}) [0-9a-f]+ \d+\t(?.+)$' -or $Matches['Path'] -cne $relativePath) { + $mode = $null + if (-not $TrackedMode.TryGetValue($relativePath, [ref]$mode)) { $violations += "Required plugin root file is not tracked: $relativePath" continue } - if ($Matches['Mode'] -notin @('100644', '100755')) { - $violations += "Required plugin root file is not a tracked regular file: $relativePath (mode $($Matches['Mode']))" + if ($mode -notin @('100644', '100755')) { + $violations += "Required plugin root file is not a tracked regular file: $relativePath (mode $mode)" continue } if ((Get-Item -LiteralPath $path).Length -eq 0) { @@ -578,6 +604,11 @@ function Get-PluginComponentCoverageViolations { $root = Resolve-PluginPath -Path (Join-Path $RepoRoot $SourcePath) $rootPrefix = $root.TrimEnd([System.IO.Path]::DirectorySeparatorChar) + [System.IO.Path]::DirectorySeparatorChar + # Components share few immediate parents, so resolving each distinct parent + # once avoids repeating the whole segment walk per component. Ordinal keys on + # every platform preserve the script's case-sensitive path identity. + $parentCache = [System.Collections.Generic.Dictionary[string, string]]::new([System.StringComparer]::Ordinal) + foreach ($kind in @('agents', 'commands', 'rules', 'skills')) { $type = if ($kind -eq 'skills') { 'Container' } else { 'Leaf' } foreach ($path in @($Manifest[$kind])) { @@ -590,22 +621,24 @@ function Get-PluginComponentCoverageViolations { $violations += "$kind path does not exist under $SourcePath : $path" continue } - $resolved = Resolve-PluginPath -Path $candidate - if ($resolved -cne $root -and -not $resolved.StartsWith($rootPrefix, [System.StringComparison]::Ordinal)) { - $violations += "$kind path resolves outside the plugin root: $path" + + $fullCandidate = [System.IO.Path]::GetFullPath($candidate) + $parent = [System.IO.Path]::GetDirectoryName($fullCandidate) + $resolvedParent = $null + if (-not $parentCache.TryGetValue($parent, [ref]$resolvedParent)) { + $resolvedParent = Resolve-PluginPath -Path $parent + $parentCache[$parent] = $resolvedParent } - } - } - if ($Manifest.Contains('hooks')) { - $hookPath = Join-Path $root $Manifest['hooks'] - if (-not (Test-Path -LiteralPath $hookPath -PathType Leaf)) { - $violations += "hooks path does not exist under $SourcePath : $($Manifest['hooks'])" - } - else { - $resolvedHook = Resolve-PluginPath -Path $hookPath - if (-not $resolvedHook.StartsWith($rootPrefix, [System.StringComparison]::Ordinal)) { - $violations += "hooks path resolves outside the plugin root: $($Manifest['hooks'])" + $resolved = Join-Path $resolvedParent ([System.IO.Path]::GetFileName($fullCandidate)) + $item = Get-Item -LiteralPath $resolved -Force + if ($item.LinkType) { + $resolved = $item.ResolveLinkTarget($true).FullName + } + $resolved = [System.IO.Path]::GetFullPath($resolved) + + if ($resolved -cne $root -and -not $resolved.StartsWith($rootPrefix, [System.StringComparison]::Ordinal)) { + $violations += "$kind path resolves outside the plugin root: $path" } } } @@ -613,10 +646,16 @@ function Get-PluginComponentCoverageViolations { return $violations } -function Get-PluginCatalogViolations { +function Get-PluginCatalogMetadataViolations { <# .SYNOPSIS - Validates the one-entry marketplace catalog against the plugin manifest. + Validates catalog shape, manifest parity, and the source locator. + + .DESCRIPTION + Owns every catalog check that needs no traversal of declared component + paths. CoverageSource carries the validated relative plugin root when + component coverage can still be meaningful, and $null when an earlier + violation already invalidates that traversal. .PARAMETER RepoRoot Root directory of the repository. @@ -625,10 +664,11 @@ function Get-PluginCatalogViolations { Expected manifest content. .OUTPUTS - [string[]] Violation messages. + [System.Collections.Specialized.OrderedDictionary] Violation messages + and the coverage source, if any. #> [CmdletBinding()] - [OutputType([string[]])] + [OutputType([System.Collections.Specialized.OrderedDictionary])] param( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] @@ -640,7 +680,10 @@ function Get-PluginCatalogViolations { $catalogPath = Join-Path $RepoRoot $script:MarketplaceCatalogFile if (-not (Test-Path -LiteralPath $catalogPath -PathType Leaf)) { - return @("Marketplace catalog not found: $script:MarketplaceCatalogFile") + return [ordered]@{ + Violations = @("Marketplace catalog not found: $script:MarketplaceCatalogFile") + CoverageSource = $null + } } $catalog = Get-Content -LiteralPath $catalogPath -Raw | ConvertFrom-Json -AsHashtable @@ -648,7 +691,10 @@ function Get-PluginCatalogViolations { $entries = @($catalog['plugins']) if ($entries.Count -ne 1) { - return @("Marketplace catalog declares $($entries.Count) entries; exactly one is required") + return [ordered]@{ + Violations = @("Marketplace catalog declares $($entries.Count) entries; exactly one is required") + CoverageSource = $null + } } $entry = $entries[0] @@ -680,19 +726,53 @@ function Get-PluginCatalogViolations { $source = $entry['source'] if ($source -isnot [string]) { $violations += 'Catalog entry source must be a relative path string' - return $violations + return [ordered]@{ Violations = @($violations); CoverageSource = $null } } if ([System.IO.Path]::IsPathRooted($source) -or $source -match '\\' -or $source -match '(^|/)\.\.(/|$)') { $violations += "Catalog entry source escapes the repository: $source" - return $violations + return [ordered]@{ Violations = @($violations); CoverageSource = $null } } if (-not (Test-Path -LiteralPath (Join-Path $RepoRoot $source 'plugin.json') -PathType Leaf)) { $violations += "Catalog entry source has no plugin manifest: $source/plugin.json" - return $violations + return [ordered]@{ Violations = @($violations); CoverageSource = $null } } - return @($violations) + @(Get-PluginComponentCoverageViolations -RepoRoot $RepoRoot -SourcePath $source -Manifest $Manifest) + return [ordered]@{ Violations = @($violations); CoverageSource = $source } +} + +function Get-PluginCatalogViolations { + <# + .SYNOPSIS + Validates the one-entry marketplace catalog against the plugin manifest. + + .PARAMETER RepoRoot + Root directory of the repository. + + .PARAMETER Manifest + Expected manifest content. + + .OUTPUTS + [string[]] Violation messages. + #> + [CmdletBinding()] + [OutputType([string[]])] + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$RepoRoot, + + [Parameter(Mandatory = $true)] + [System.Collections.IDictionary]$Manifest + ) + + $metadata = Get-PluginCatalogMetadataViolations -RepoRoot $RepoRoot -Manifest $Manifest + if ($null -eq $metadata.CoverageSource) { + return @($metadata.Violations) + } + + return @($metadata.Violations) + + @(Get-PluginComponentCoverageViolations -RepoRoot $RepoRoot -SourcePath $metadata.CoverageSource -Manifest $Manifest) } #endregion Catalog @@ -725,7 +805,9 @@ function Invoke-PluginManifestSync { [switch]$Check ) - $metadataViolations = @(Get-PluginMetadataViolations -RepoRoot $RepoRoot) + $trackedIndex = Get-TrackedPluginIndex -RepoRoot $RepoRoot + + $metadataViolations = @(Get-PluginMetadataViolations -RepoRoot $RepoRoot -TrackedMode $trackedIndex.Mode) if ($metadataViolations.Count -gt 0) { return [ordered]@{ Changed = $false @@ -734,7 +816,6 @@ function Invoke-PluginManifestSync { } } - $trackedIndex = Get-TrackedPluginIndex -RepoRoot $RepoRoot $components = Get-PluginComponentSet -RepoRoot $RepoRoot -TrackedPath $trackedIndex.Paths $version = Get-RepositoryVersion -RepoRoot $RepoRoot $manifest = New-PluginManifest -RepoRoot $RepoRoot -Component $components -Version $version diff --git a/scripts/tests/plugins/Sync-PluginManifest.Tests.ps1 b/scripts/tests/plugins/Sync-PluginManifest.Tests.ps1 index 52317664b..634b7c7a8 100644 --- a/scripts/tests/plugins/Sync-PluginManifest.Tests.ps1 +++ b/scripts/tests/plugins/Sync-PluginManifest.Tests.ps1 @@ -56,10 +56,6 @@ BeforeAll { New-SkillFile -Path (Join-Path $github 'skills/beta/noncommercial-skill/SKILL.md') -Name 'noncommercial-skill' -License 'CC-BY-NC-SA-4.0' New-SkillFile -Path (Join-Path $github 'skills/root-only/SKILL.md') -Name 'root-only' -License 'MIT' - $hookPath = Join-Path $github 'hooks/shared/telemetry.json' - New-Item -ItemType Directory -Path (Split-Path $hookPath -Parent) -Force | Out-Null - Set-Content -LiteralPath $hookPath -Value '{ "version": 1, "hooks": {} }' -Encoding UTF8 - Set-Content -LiteralPath (Join-Path $Root 'package.json') ` -Value ("{`n `"name`": `"fixture`",`n `"version`": `"$Version`"`n}`n") -Encoding UTF8 Set-Content -LiteralPath (Join-Path $Root 'README.md') -Value "# Fixture`n" -Encoding UTF8 -NoNewline @@ -78,7 +74,6 @@ BeforeAll { commands = @() rules = @() skills = @() - hooks = '.github/hooks/shared/telemetry.json' } Set-Content -LiteralPath (Join-Path $Root 'plugin.json') ` -Value (ConvertTo-PluginManifestJson -Manifest $manifest) -Encoding UTF8 -NoNewline @@ -279,15 +274,15 @@ Describe 'New-PluginManifest' -Tag 'Unit' { $script:Manifest.license | Should -BeExactly 'MIT' } - It 'Declares the fixed hook manifest' { - $script:Manifest.hooks | Should -BeExactly '.github/hooks/shared/telemetry.json' + It 'Emits no hooks field' { + $script:Manifest.Contains('hooks') | Should -BeFalse } It 'Orders manifest keys deterministically' { @($script:Manifest.Keys) | Should -Be @( 'name', 'description', 'version', 'author', 'homepage', 'repository', 'license', 'keywords', - 'agents', 'commands', 'rules', 'skills', 'hooks' + 'agents', 'commands', 'rules', 'skills' ) } @@ -419,7 +414,22 @@ Describe 'Invoke-PluginManifestSync failure atomicity' -Tag 'Unit' { Describe 'Invoke-PluginManifestSync check mode' -Tag 'Unit' { BeforeAll { $script:CheckRoot = New-PluginFixture -Root (Join-Path $TestDrive 'check') - Invoke-PluginManifestSync -RepoRoot $script:CheckRoot | Out-Null + $script:CheckExpected = (Invoke-PluginManifestSync -RepoRoot $script:CheckRoot).Manifest + $script:CheckManifestPath = Join-Path $script:CheckRoot 'plugin.json' + $script:CheckManifestBytes = [System.IO.File]::ReadAllBytes($script:CheckManifestPath) + + function New-CommittedManifest { + <# + .SYNOPSIS + Copies the derived manifest so a drift case can mutate one field. + #> + param([hashtable]$Override) + + $committed = [ordered]@{} + foreach ($key in $script:CheckExpected.Keys) { $committed[$key] = $script:CheckExpected[$key] } + foreach ($key in $Override.Keys) { $committed[$key] = $Override[$key] } + return $committed + } } It 'Reports no violations for a synchronized manifest' { @@ -430,50 +440,53 @@ Describe 'Invoke-PluginManifestSync check mode' -Tag 'Unit' { Set-FixtureManifest -Root $script:CheckRoot -Transform { param($m) $m['agents'] = @('agents/alpha/one.agent.md'); $m } - $before = (Get-FileHash (Join-Path $script:CheckRoot 'plugin.json') -Algorithm SHA256).Hash + $before = (Get-FileHash $script:CheckManifestPath -Algorithm SHA256).Hash Invoke-PluginManifestSync -RepoRoot $script:CheckRoot -Check | Out-Null - (Get-FileHash (Join-Path $script:CheckRoot 'plugin.json') -Algorithm SHA256).Hash | Should -BeExactly $before + (Get-FileHash $script:CheckManifestPath -Algorithm SHA256).Hash | Should -BeExactly $before } It 'Names components missing from the committed manifest' { - Set-FixtureManifest -Root $script:CheckRoot -Transform { - param($m) $m['agents'] = @('.github/agents/alpha/one.agent.md'); $m - } - $violations = (Invoke-PluginManifestSync -RepoRoot $script:CheckRoot -Check).Violations - $violations -join "`n" | Should -Match 'agents missing 1: \.github/agents/alpha/subagents/two\.agent\.md' + $committed = New-CommittedManifest -Override @{ agents = @('.github/agents/alpha/one.agent.md') } + (Compare-PluginManifest -Committed $committed -Expected $script:CheckExpected) -join "`n" | + Should -Match 'agents missing 1: \.github/agents/alpha/subagents/two\.agent\.md' } It 'Names components the committed manifest adds' { - Set-FixtureManifest -Root $script:CheckRoot -Transform { - param($m) $m['skills'] = @($m['skills']) + 'skills/beta/noncommercial-skill'; $m - } - $violations = (Invoke-PluginManifestSync -RepoRoot $script:CheckRoot -Check).Violations - $violations -join "`n" | Should -Match 'skills unexpected 1: skills/beta/noncommercial-skill' + $committed = New-CommittedManifest -Override @{ skills = @($script:CheckExpected['skills']) + 'skills/beta/noncommercial-skill' } + (Compare-PluginManifest -Committed $committed -Expected $script:CheckExpected) -join "`n" | + Should -Match 'skills unexpected 1: skills/beta/noncommercial-skill' } - It 'Detects stale drift' -ForEach @( - @{ Field = 'version'; Value = '9.9.9'; Pattern = "version differs: committed '9\.9\.9'; expected '1\.2\.3'" } - @{ Field = 'hooks'; Value = '.github/hooks/other.json'; Pattern = "hooks differs: committed '\.github/hooks/other\.json'; expected '\.github/hooks/shared/telemetry\.json'" } - ) { - $field, $value = $Field, $Value - Set-FixtureManifest -Root $script:CheckRoot -Transform { - param($m) $m[$field] = $value; $m - }.GetNewClosure() - $violations = (Invoke-PluginManifestSync -RepoRoot $script:CheckRoot -Check).Violations - $violations -join "`n" | Should -Match $Pattern + It 'Detects stale version drift' { + $committed = New-CommittedManifest -Override @{ version = '9.9.9' } + (Compare-PluginManifest -Committed $committed -Expected $script:CheckExpected) -join "`n" | + Should -Match "version differs: committed '9\.9\.9'; expected '1\.2\.3'" + } + + It 'Names an unsupported committed field rather than reporting drift without detail' { + $committed = New-CommittedManifest -Override @{ unknown = 'value' } + (Compare-PluginManifest -Committed $committed -Expected $script:CheckExpected) -join "`n" | + Should -Match 'unknown is declared but the manifest no longer supports it' } It 'Reports unsupported raw-only drift explicitly' { - Set-FixtureManifest -Root $script:CheckRoot -Transform { param($m) $m['unknown'] = 'value'; $m } - (Invoke-PluginManifestSync -RepoRoot $script:CheckRoot -Check).Violations -join "`n" | Should -Match 'no supported drift detail is available' + # Key order is the drift no field-level comparison can describe. + Set-FixtureManifest -Root $script:CheckRoot -Transform { + param($m) + $reordered = [ordered]@{} + foreach ($key in @($m.Keys) | Sort-Object) { $reordered[$key] = $m[$key] } + $reordered + } + (Invoke-PluginManifestSync -RepoRoot $script:CheckRoot -Check).Violations -join "`n" | + Should -Match 'no supported drift detail is available' } AfterEach { - Invoke-PluginManifestSync -RepoRoot $script:CheckRoot | Out-Null + [System.IO.File]::WriteAllBytes($script:CheckManifestPath, $script:CheckManifestBytes) } } -Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { +Describe 'Plugin catalog validation' -Tag 'Unit' { BeforeAll { $script:CatalogRoot = New-PluginFixture -Root (Join-Path $TestDrive 'catalog') $script:CatalogManifest = (Invoke-PluginManifestSync -RepoRoot $script:CatalogRoot).Manifest @@ -507,7 +520,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) $c['plugins'] = @($c['plugins']) + @([ordered]@{ name = 'extra'; source = '.github'; version = '1.2.3' }); $c } - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match 'declares 2 entries' } @@ -515,7 +528,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) @($c['plugins'])[0]['name'] = 'other'; $c } - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match "field 'name' does not match" } @@ -523,7 +536,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) @($c['plugins'])[0]['version'] = '9.9.9'; $c } - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match "field 'version' does not match" } @@ -531,7 +544,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) $c['metadata']['version'] = '9.9.9'; $c } - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match 'metadata version' Set-FixtureCatalog -Root $script:CatalogRoot -Transform { @@ -552,7 +565,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) @($c['plugins'])[0][$field] = $value; $c }.GetNewClosure() - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match "field '$([regex]::Escape($Field))' does not match" } @@ -569,7 +582,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) @($c['plugins'])[0][$field] = $value; $c }.GetNewClosure() - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match "retired package recipe field '$([regex]::Escape($Field))'" } @@ -577,7 +590,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) @($c['plugins'])[0]['source'] = [ordered]@{ source = 'github'; repo = 'microsoft/hve-core'; path = '.github' }; $c } - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match 'must be a relative path string' } @@ -585,7 +598,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) @($c['plugins'])[0]['source'] = '../elsewhere'; $c } - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match 'escapes the repository' } @@ -593,7 +606,7 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Set-FixtureCatalog -Root $script:CatalogRoot -Transform { param($c) @($c['plugins'])[0]['source'] = 'missing'; $c } - (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest) -join "`n" | + (Get-PluginCatalogMetadataViolations -RepoRoot $script:CatalogRoot -Manifest $script:CatalogManifest).Violations -join "`n" | Should -Match 'has no plugin manifest' } @@ -615,45 +628,79 @@ Describe 'Get-PluginCatalogViolations' -Tag 'Unit' { Should -Match 'skills path escapes the plugin root' } - It 'Reports a declared component that resolves outside the plugin root' { - $outsidePath = Join-Path $TestDrive 'outside.agent.md' - Set-Content -LiteralPath $outsidePath -Value 'outside' -Encoding UTF8 - $linkPath = Join-Path $script:CatalogRoot '.github/agents/alpha/outside.agent.md' - New-Item -ItemType SymbolicLink -Path $linkPath -Target $outsidePath | Out-Null + It 'Reports a declared component whose link resolves outside the plugin root' -ForEach @( + @{ Scenario = 'leaf'; TargetName = 'outside-leaf.agent.md'; TargetIsDirectory = $false; LinkRelative = '.github/agents/alpha/outside.agent.md'; Component = '.github/agents/alpha/outside.agent.md' } + @{ Scenario = 'parent'; TargetName = 'outside-parent'; TargetIsDirectory = $true; LinkRelative = '.github/agents/gamma'; Component = '.github/agents/gamma/nested.agent.md' } + ) { + $target = Join-Path $TestDrive $TargetName + if ($TargetIsDirectory) { + New-Item -ItemType Directory -Path $target -Force | Out-Null + Set-Content -LiteralPath (Join-Path $target 'nested.agent.md') -Value 'outside' -Encoding UTF8 + } + else { + Set-Content -LiteralPath $target -Value 'outside' -Encoding UTF8 + } + New-Item -ItemType SymbolicLink -Path (Join-Path $script:CatalogRoot $LinkRelative) -Target $target | Out-Null $manifest = [ordered]@{} foreach ($key in $script:CatalogManifest.Keys) { $manifest[$key] = $script:CatalogManifest[$key] } - $manifest['agents'] = @('.github/agents/alpha/outside.agent.md') + $manifest['agents'] = @($Component) (Get-PluginCatalogViolations -RepoRoot $script:CatalogRoot -Manifest $manifest) -join "`n" | Should -Match 'agents path resolves outside the plugin root' } +} - It 'Reports a declared hook that does not exist' { - $root = New-PluginFixture -Root (Join-Path $TestDrive 'missing-hook') - $manifest = (Invoke-PluginManifestSync -RepoRoot $root).Manifest - Remove-Item -LiteralPath (Join-Path $root '.github/hooks/shared/telemetry.json') -Force +Describe 'Retired hooks support' -Tag 'Unit' { + It 'Reports a committed hooks declaration as drift rather than preserving it' { + $root = New-PluginFixture -Root (Join-Path $TestDrive 'stray-hooks') + $manifestPath = Join-Path $root 'plugin.json' + $committed = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json -AsHashtable + $committed['hooks'] = '.github/hooks/shared/telemetry.json' + Set-Content -LiteralPath $manifestPath ` + -Value (ConvertTo-PluginManifestJson -Manifest $committed) -Encoding UTF8 -NoNewline - (Get-PluginCatalogViolations -RepoRoot $root -Manifest $manifest) -join "`n" | - Should -Match 'hooks path does not exist' + $result = Invoke-PluginManifestSync -RepoRoot $root -Check + + $result.Violations -join "`n" | Should -Match 'hooks is declared but the manifest no longer supports it' + $result.Manifest.Contains('hooks') | Should -BeFalse } - It 'Reports a declared hook that resolves outside the plugin root' { - $root = New-PluginFixture -Root (Join-Path $TestDrive 'outside-hook') - $manifest = (Invoke-PluginManifestSync -RepoRoot $root).Manifest - $outsidePath = Join-Path $TestDrive 'outside-hook.json' - Set-Content -LiteralPath $outsidePath -Value '{}' -Encoding UTF8 - $hookPath = Join-Path $root '.github/hooks/shared/telemetry.json' - Remove-Item -LiteralPath $hookPath -Force - New-Item -ItemType SymbolicLink -Path $hookPath -Target $outsidePath | Out-Null + It 'Reports any other unsupported committed field as drift' { + $root = New-PluginFixture -Root (Join-Path $TestDrive 'stray-field') + $manifestPath = Join-Path $root 'plugin.json' + $committed = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json -AsHashtable + $committed['x-hve'] = @{ displayName = 'HVE Core' } + Set-Content -LiteralPath $manifestPath ` + -Value (ConvertTo-PluginManifestJson -Manifest $committed) -Encoding UTF8 -NoNewline - (Get-PluginCatalogViolations -RepoRoot $root -Manifest $manifest) -join "`n" | - Should -Match 'hooks path resolves outside the plugin root' + (Invoke-PluginManifestSync -RepoRoot $root -Check).Violations -join "`n" | + Should -Match 'x-hve is declared but the manifest no longer supports it' + } + + It 'Drops a committed hooks declaration on write' { + $root = New-PluginFixture -Root (Join-Path $TestDrive 'drop-hooks') + $manifestPath = Join-Path $root 'plugin.json' + $committed = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json -AsHashtable + $committed['hooks'] = '.github/hooks/shared/telemetry.json' + Set-Content -LiteralPath $manifestPath ` + -Value (ConvertTo-PluginManifestJson -Manifest $committed) -Encoding UTF8 -NoNewline + + Invoke-PluginManifestSync -RepoRoot $root | Out-Null + + $written = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json -AsHashtable + $written.Contains('hooks') | Should -BeFalse } } Describe 'Committed repository manifest' -Tag 'Unit' { - It 'Passes check mode without drift or catalog violations' { + It 'Passes check mode without drift, catalog violations, or retired hooks' { + Test-Path -LiteralPath (Join-Path $script:RepositoryRoot '.github/hooks') | Should -BeFalse + + $committed = Get-Content -LiteralPath (Join-Path $script:RepositoryRoot 'plugin.json') -Raw | + ConvertFrom-Json -AsHashtable + $committed.Contains('hooks') | Should -BeFalse + $result = Invoke-PluginManifestSync -RepoRoot $script:RepositoryRoot -Check $result.Violations -join "`n" | Should -BeExactly '' } diff --git a/scripts/tests/release/Set-RepositoryVersion.Tests.ps1 b/scripts/tests/release/Set-RepositoryVersion.Tests.ps1 index b5756ce1c..92bea301d 100644 --- a/scripts/tests/release/Set-RepositoryVersion.Tests.ps1 +++ b/scripts/tests/release/Set-RepositoryVersion.Tests.ps1 @@ -68,8 +68,7 @@ BeforeAll { "agents": [], "commands": [], "rules": [], - "skills": [], - "hooks": "hooks/shared/telemetry.json" + "skills": [] } "@ '.github/plugin/marketplace.json' = @" From 0dbb07310cf99e141d7dcd8aeee63452e8282fcd Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:09:01 -0700 Subject: [PATCH 21/28] docs: add retirement guidance for existing telemetry hook users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Names the four stranded artifact classes with exact paths and states that removing the hook stops new writes but deletes nothing. πŸ“ - Generated by Copilot --- TRANSPARENCY-NOTE.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/TRANSPARENCY-NOTE.md b/TRANSPARENCY-NOTE.md index bc32f9999..d1b7e14df 100644 --- a/TRANSPARENCY-NOTE.md +++ b/TRANSPARENCY-NOTE.md @@ -59,6 +59,19 @@ HVE Core ships text files and supporting tools. When you load an HVE Core file i HVE Core has no model, no API, and no network calls while you author or install it. It ships no telemetry collector that runs on your machine by default. Optional OpenTelemetry export is a Copilot Chat feature you configure yourself, and the `copilot-otel-metrics` skill only generates the settings, local stack, and infrastructure templates you choose to apply. Nothing is collected until you enable that export and run a receiver you control. Validation tools (linters, frontmatter checks, Pester tests, plugin generation) run in CI on pull requests. Nothing runs on your machine unless you install the VS Code extension or run a packaged script yourself. +#### If you used the retired telemetry hook + +HVE Core previously shipped a session hook that wrote local telemetry on your machine. It has been removed, and the opt-in `copilot-otel-metrics` skill replaces it. Removing the hook stops new writes; it does not delete anything already written, and nothing in HVE Core will delete it for you. Four kinds of artifact are left behind, and you decide what happens to each. + +| What was left behind | Where it is | What to do | +|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Per-project telemetry store | `/.copilot-tracking/telemetry`, or the directory named by `HVE_TELEMETRY_DIR` | Holds `sessions-*.jsonl`, `.stacks/`, and `report.generated.html`. Delete the directory when you no longer want the history. It is gitignored, so it was never published, but it is still on disk. | +| Raw captures | `raw-input.jsonl` in that same directory | This is the unprocessed capture and the most sensitive of the four. Delete it first if you are triaging. | +| Cross-project registry | `~/.hve/telemetry-dirs`, or `$HVE_HOME/telemetry-dirs`; older installs also have `telemetry-dirs.txt` | A list of every project directory the hook wrote to. Read it to find stores you have forgotten, then delete it. | +| Generated launchers | `~/.hve/generate-report.sh`, `generate-report.ps1`, `clean-telemetry.sh`, `clean-telemetry.ps1`, plus `~/.hve/report.generated.html` | The hook regenerated these each session and will no longer do so. They now point at scripts that are gone, so they fail rather than run. Delete them. | + +The cleanup scripts that used to remove these were part of the hook and were removed with it. If you want the registry-driven sweep, take a copy of `.github/hooks/shared/telemetry/` from a tag before the retirement, run it once, and then discard it. Otherwise delete the paths above by hand; they are ordinary files and directories. + Most skills are pure authoring or validation helpers with no independent Responsible AI surface and are not called out individually. A few skills warrant specific mention because they assemble media outputs or depend on external services: * The **Customer Card Render** skill assembles synthetic-persona slides from authored Design Thinking content through a template-driven PowerPoint pipeline; HVE Core has no image-generation model. When concept imagery is needed, the workflow emits prompts the operator runs on an external platform such as M365 Copilot, where the host's Responsible AI layers apply. The cards stay low-fidelity and carry disclosure, redaction, and stereotyping-review controls. See Appendix 5. @@ -134,7 +147,7 @@ For a set of files, "performance" is not a model-accuracy score. It is how well Quality rests on a few things: * **CI checks on every pull request.** Markdown linting, frontmatter validation, model-reference checks, link checking, PowerShell and Python linting, YAML validation, collection-metadata and marketplace validation, dependency-pinning and action-version checks, copyright-header checks, and skill-structure validation all run on each pull request and block merge on failure. -* **Plugin-manifest gate.** The complete root `plugin.json` membership is derived from tracked distributable `.github` source paths; missing or untracked root metadata, drift, invalid locator metadata, escaping or absent component paths, or invalid hooks block merge. +* **Plugin-manifest gate.** The complete root `plugin.json` membership is derived from tracked distributable `.github` source paths; missing or untracked root metadata, drift, invalid locator metadata, or escaping or absent component paths block merge. * **Human review.** Every file change needs human review. Supply-chain and dependency checks surface to reviewers. * **Manifest parity and release review.** Stable and PreRelease ship the same complete plugin manifest. Stable release review happens through promotion of a reviewed PreRelease tree into `release/stable`; channel selection changes release cadence and assurance rather than component membership. * **Feedback channel.** GitHub issues on `microsoft/hve-core` are the main place for bugs, requests, and concerns. @@ -144,7 +157,7 @@ HVE Core does not measure performance against a specific model. If you need repr ### Getting the best results * **Pin to a release tag.** Treat the main branch as a moving target. For anything production-relevant, pin to a release tag and review changes before upgrading. -* **Choose a managed or selective footprint.** The extension and plugin contain the same complete distributable component set. Teams that need fewer repository-owned files can use the included installer skill's complete or custom selection across agents, prompts, instructions, and whole skill directories. The installer preserves repository-relative paths, records schema version 2, and does not copy hooks. +* **Choose a managed or selective footprint.** The extension and plugin contain the same complete distributable component set. Teams that need fewer repository-owned files can use the included installer skill's complete or custom selection across agents, prompts, instructions, and whole skill directories. The installer preserves repository-relative paths and records schema version 2. * **Read an agent's description before loading it.** Each agent file documents its purpose, inputs, outputs, and limits. Skipping this is the most common cause of surprises. * **Treat decision-shaping output as a draft.** Planning agents, code-review agents that gate pull requests, and customer-handoff agents produce drafts. Do not turn a draft into a binding decision without qualified human review. * **Check saved memory before sharing a workspace.** Agents that write to the memory layer carry context across sessions. Inspect and clear it through the host's controls before sharing a workspace, screenshot, or recording. From 41dc50b38b4b0050c9f1e3d3011a157cf4b363ad Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:09:05 -0700 Subject: [PATCH 22/28] fix(scripts): scan only tracked files and parse pip comments and markers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - restrict the walk to git ls-files, failing open outside a git tree - blank comments and PEP 508 markers before matching, keeping offsets - repository compliance moves from 93.49% to 100% πŸ” - Generated by Copilot --- scripts/security/Test-DependencyPinning.ps1 | 112 ++++++++++++++++++ .../security/Test-DependencyPinning.Tests.ps1 | 105 ++++++++++++++++ 2 files changed, 217 insertions(+) diff --git a/scripts/security/Test-DependencyPinning.ps1 b/scripts/security/Test-DependencyPinning.ps1 index af2e12df9..2cd08d4ce 100644 --- a/scripts/security/Test-DependencyPinning.ps1 +++ b/scripts/security/Test-DependencyPinning.ps1 @@ -149,6 +149,9 @@ $DependencyPatterns = @{ 'pip' = @{ FilePatterns = @('**/requirements*.txt', '**/Pipfile', '**/pyproject.toml', '**/setup.py') ExcludePatterns = @('.venv', 'venv', '.tox', '.nox', '__pypackages__') + # '#' starts a comment and ';' starts a PEP 508 environment marker. Both + # carry '==' comparisons that are not package pins. + IgnoreAfter = @('#', ';') VersionPatterns = @( @{ Pattern = '([a-zA-Z0-9._-]+(?:\[[a-zA-Z0-9._,-]+\])?)\s*==\s*([^#\s,";]+)' @@ -526,6 +529,48 @@ function Test-NpmExactVersion { return $Version -match '^\d+\.\d+\.\d+(-[a-zA-Z0-9._-]+)?(\+[a-zA-Z0-9._-]+)?$' } +function Get-TrackedFilePathSet { + <# + .SYNOPSIS + Returns the set of git-tracked file paths under a scan root. + + .DESCRIPTION + CI scans a clean checkout, so an ignored or untracked working-tree file is + content the pipeline never sees. Counting it locally produces compliance + failures that cannot be reproduced in CI. Returns $null when the scan root + is not a usable git working tree, which the caller treats as a reason to + scan every file rather than to scan none. + + .PARAMETER ScanPath + Root directory being scanned. + + .OUTPUTS + [System.Collections.Generic.HashSet[string]] Full paths, or $null. + #> + [CmdletBinding()] + [OutputType([System.Collections.Generic.HashSet[string]])] + param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$ScanPath + ) + + if (-not (Get-Command git -ErrorAction SilentlyContinue)) { return $null } + if (-not (Test-Path -LiteralPath $ScanPath -PathType Container)) { return $null } + + # Paths come back relative to ScanPath, so a subdirectory scan stays correct. + $tracked = & git -C $ScanPath ls-files --cached 2>$null + if ($LASTEXITCODE -ne 0) { return $null } + + $set = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($relative in $tracked) { + if ([string]::IsNullOrWhiteSpace($relative)) { continue } + $full = Join-Path $ScanPath ($relative -replace '/', [System.IO.Path]::DirectorySeparatorChar) + $null = $set.Add([System.IO.Path]::GetFullPath($full)) + } + return $set +} + function Get-FilesToScan { <# .SYNOPSIS @@ -539,6 +584,10 @@ function Get-FilesToScan { ) $allFiles = @() + $trackedPaths = Get-TrackedFilePathSet -ScanPath $ScanPath + if ($null -eq $trackedPaths) { + Write-SecurityLog -CIAnnotation "Scan root is not a git working tree; scanning all matching files." -Level Info + } foreach ($type in $Types) { if ($DependencyPatterns.ContainsKey($type)) { @@ -566,6 +615,10 @@ function Get-FilesToScan { $files = Get-ChildItem -Path $basePath -Filter $leafFilter -Recurse -File -ErrorAction SilentlyContinue + if ($null -ne $trackedPaths) { + $files = $files | Where-Object { $trackedPaths.Contains([System.IO.Path]::GetFullPath($_.FullName)) } + } + # Merge type-specific exclude patterns with caller-provided patterns $mergedExcludes = @() if ($ExcludePatterns) { @@ -618,6 +671,60 @@ function Test-DependencyPinned { return $false } +function Get-MatchableContent { + <# + .SYNOPSIS + Blanks trailing line segments a dependency matcher must not read. + + .DESCRIPTION + A pip requirement carries its environment marker after ';', and both + requirements files and pyproject.toml use '#' for comments. Each segment can + hold an '==' comparison that is not a package pin: `sys_platform == 'linux'` + names a marker variable, not a package. Blanking with spaces rather than + removing keeps every match index, and therefore every reported line number, + identical to the original file. + + .PARAMETER Content + Raw file content. + + .PARAMETER IgnoreAfter + Characters whose first occurrence on a line begins an ignorable segment. + + .OUTPUTS + [string] Content with ignorable segments replaced by spaces. + #> + [CmdletBinding()] + [OutputType([string])] + param( + [Parameter(Mandatory = $true)] + [AllowEmptyString()] + [AllowNull()] + [string]$Content, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [char[]]$IgnoreAfter + ) + + if ([string]::IsNullOrEmpty($Content)) { return $Content } + + $builder = [System.Text.StringBuilder]::new($Content.Length) + $ignoring = $false + foreach ($character in $Content.ToCharArray()) { + if ($character -eq [char]10 -or $character -eq [char]13) { + $ignoring = $false + $null = $builder.Append($character) + continue + } + if (-not $ignoring -and $IgnoreAfter -contains $character) { + $ignoring = $true + } + $null = $builder.Append($(if ($ignoring) { ' ' } else { $character })) + } + + return $builder.ToString() +} + function Get-DependencyViolation { <# .SYNOPSIS @@ -675,6 +782,11 @@ function Get-DependencyViolation { $content = Get-Content -Path $filePath -Raw $lines = Get-Content -Path $filePath + $ignoreAfter = $DependencyPatterns[$fileType].IgnoreAfter + if ($ignoreAfter) { + $content = Get-MatchableContent -Content $content -IgnoreAfter $ignoreAfter + } + $patterns = $DependencyPatterns[$fileType].VersionPatterns $totalCount = 0 diff --git a/scripts/tests/security/Test-DependencyPinning.Tests.ps1 b/scripts/tests/security/Test-DependencyPinning.Tests.ps1 index 19e50ec5d..bce7a2ccf 100644 --- a/scripts/tests/security/Test-DependencyPinning.Tests.ps1 +++ b/scripts/tests/security/Test-DependencyPinning.Tests.ps1 @@ -2,6 +2,10 @@ # Copyright (c) 2026 Microsoft Corporation. All rights reserved. # SPDX-License-Identifier: MIT +# Discovery-time probe: Pester evaluates -Skip: while discovering tests, before +# any BeforeAll runs, so a flag set in BeforeAll reads as $null here. +$script:GitAvailable = [bool](Get-Command git -ErrorAction SilentlyContinue) + BeforeAll { . $PSScriptRoot/../../security/Test-DependencyPinning.ps1 # Re-import CIHelpers so Pester can resolve its commands for mocking; @@ -757,6 +761,107 @@ Describe 'shell-downloads ExcludePatterns' -Tag 'Unit' { } } +Describe 'pip comment and environment-marker handling' -Tag 'Unit' { + BeforeAll { + # `sys_platform == 'linux'` is a PEP 508 marker variable, not a package. + # The matcher looks for `==` anywhere, so a comment or a marker used to + # be reported as an unpinned dependency named after the marker variable. + $script:PipRoot = Join-Path $TestDrive 'pip-markers' + New-Item -Path $script:PipRoot -ItemType Directory -Force | Out-Null + + $pyproject = @' +[project] +dependencies = [ + # sys_platform == 'linux' condition for secretstorage. + "requests==2.32.3", + "cryptography>=50.0.0,<51.0; sys_platform == 'linux'", + "pywinauto>=0.6.8 ; platform_system == 'Windows'", + "httpx==0.27.0 ; python_version < '3.13'", +] +'@ + Set-Content -Path (Join-Path $script:PipRoot 'pyproject.toml') -Value $pyproject + + $script:PipResult = Get-DependencyViolation -FileInfo @{ + Path = (Join-Path $script:PipRoot 'pyproject.toml') + Type = 'pip' + RelativePath = 'pyproject.toml' + } + } + + It 'Reports no violation for a marker variable or a commented comparison' { + @($script:PipResult.Violations).Name | Should -Not -Contain 'sys_platform' + @($script:PipResult.Violations).Name | Should -Not -Contain 'platform_system' + } + + It 'Reports no violations at all for this file' { + @($script:PipResult.Violations) | Should -HaveCount 0 + } + + It 'Still counts a pinned dependency that carries an environment marker' { + # Blanking the marker must not blank the requirement in front of it. + $script:PipResult.TotalCount | Should -Be 2 + } + + It 'Still flags an unpinned dependency outside a comment or marker' { + $root = Join-Path $TestDrive 'pip-unpinned' + New-Item -Path $root -ItemType Directory -Force | Out-Null + Set-Content -Path (Join-Path $root 'requirements.txt') -Value 'requests==latest' + + $result = Get-DependencyViolation -FileInfo @{ + Path = (Join-Path $root 'requirements.txt') + Type = 'pip' + RelativePath = 'requirements.txt' + } + + @($result.Violations).Name | Should -Contain 'requests' + } +} + +Describe 'git-tracked scan scope' -Tag 'Unit' { BeforeAll { + # CI scans a clean checkout, so an ignored or untracked working-tree file + # is content the pipeline never sees. Counting it locally produces a + # compliance failure no pipeline run can reproduce. + $script:TrackedRoot = Join-Path $TestDrive 'tracked-scope' + New-Item -Path $script:TrackedRoot -ItemType Directory -Force | Out-Null + + if (Get-Command git -ErrorAction SilentlyContinue) { + & git -C $script:TrackedRoot init --quiet + & git -C $script:TrackedRoot config user.email 'test@example.com' + & git -C $script:TrackedRoot config user.name 'Test' + + Set-Content -Path (Join-Path $script:TrackedRoot '.gitignore') -Value 'sandbox/' + Set-Content -Path (Join-Path $script:TrackedRoot 'package.json') -Value '{ "dependencies": { "left-pad": "1.3.0" } }' + & git -C $script:TrackedRoot add .gitignore package.json + & git -C $script:TrackedRoot commit --quiet -m 'tracked fixture' + + # Ignored, and untracked-but-not-ignored: neither reaches CI. + $sandbox = Join-Path $script:TrackedRoot 'sandbox' + New-Item -Path $sandbox -ItemType Directory -Force | Out-Null + Set-Content -Path (Join-Path $sandbox 'package.json') -Value '{ "dependencies": { "left-pad": "^1.3.0" } }' + Set-Content -Path (Join-Path $script:TrackedRoot 'untracked-package.json') -Value '{ }' + } + + $script:PlainRoot = Join-Path $TestDrive 'plain-scope' + New-Item -Path $script:PlainRoot -ItemType Directory -Force | Out-Null + Set-Content -Path (Join-Path $script:PlainRoot 'package.json') -Value '{ "dependencies": { "left-pad": "1.3.0" } }' + } + + It 'Excludes a gitignored file from the scan' -Skip:(-not $script:GitAvailable) { + $files = @(Get-FilesToScan -ScanPath $script:TrackedRoot -Types 'npm') + @($files.RelativePath) | Should -Not -Contain (Join-Path 'sandbox' 'package.json') + } + + It 'Includes a tracked file in the scan' -Skip:(-not $script:GitAvailable) { + $files = @(Get-FilesToScan -ScanPath $script:TrackedRoot -Types 'npm') + @($files.RelativePath) | Should -Contain 'package.json' + } + + It 'Scans every matching file when the root is not a git working tree' { + $files = @(Get-FilesToScan -ScanPath $script:PlainRoot -Types 'npm') + @($files.RelativePath) | Should -Contain 'package.json' + } +} + Describe 'github-actions composite action discovery' -Tag 'Unit' { BeforeAll { $ghaTestRoot = Join-Path $TestDrive 'gha-composite-test' From 2c8c199043ec4e0f4ecbb01555d82906a2848d99 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:09:06 -0700 Subject: [PATCH 23/28] test(skills): guard the bash-dependent installer copy assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - split the PowerShell assertion out so the prefix rule always runs - reject the WSL bash stub, which cannot run a Windows-path script πŸ§ͺ - Generated by Copilot --- .../tests/component-copy.Tests.ps1 | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/skills/installer/hve-core-installer/tests/component-copy.Tests.ps1 b/.github/skills/installer/hve-core-installer/tests/component-copy.Tests.ps1 index f7c74c31c..18b8b39bf 100644 --- a/.github/skills/installer/hve-core-installer/tests/component-copy.Tests.ps1 +++ b/.github/skills/installer/hve-core-installer/tests/component-copy.Tests.ps1 @@ -3,8 +3,15 @@ # SPDX-License-Identifier: MIT # Discovery-time capability probe: the Bash parity fixtures execute the real -# component-copy.sh, which needs both an interpreter and jq. -$script:BashAvailable = [bool](Get-Command bash -ErrorAction SilentlyContinue) -and [bool](Get-Command jq -ErrorAction SilentlyContinue) +# component-copy.sh, which needs both an interpreter and jq. On Windows `bash` +# resolves to the WSL launcher in System32, which cannot execute a script named +# by a Windows path, so it does not count as an available interpreter. +$script:BashCommand = Get-Command bash -ErrorAction SilentlyContinue +if ($IsWindows -and $script:BashCommand -and + $script:BashCommand.Source -eq (Join-Path $env:SystemRoot 'System32/bash.exe')) { + $script:BashCommand = $null +} +$script:BashAvailable = [bool]$script:BashCommand -and [bool](Get-Command jq -ErrorAction SilentlyContinue) BeforeAll { $script:PowerShellScript = (Resolve-Path (Join-Path $PSScriptRoot '../scripts/component-copy.ps1')).Path @@ -71,7 +78,6 @@ BeforeAll { commands = @('.github/prompts/hve-core/rpi.prompt.md') rules = @('.github/instructions/hve-core/copilot-tracking.instructions.md') skills = @('.github/skills/rpi/rpi-plan') - hooks = '.github/hooks/shared/telemetry.json' } $pluginManifestPath = Join-Path $source 'plugin.json' $manifest | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $pluginManifestPath -NoNewline @@ -408,26 +414,32 @@ Describe 'component-copy preflight rejection' -Tag 'Unit' { } It 'Rejects a plugin manifest that declares no installable component' { - '{ "name": "hve-core", "hooks": ".github/hooks/shared/telemetry.json" }' | + '{ "name": "hve-core", "version": "1.0.0" }' | Set-Content -LiteralPath (Join-Path $script:fixture.Source 'plugin.json') -NoNewline { Invoke-ComponentCopy -Fixture $script:fixture -Component @('agents/hve-core/rpi-agent.md') } | Should -Throw -ExpectedMessage '*declares no installable components*' } - It 'Rejects installable manifest entries outside .github in both implementations' { - $powerShellFixture = New-ComponentCopyFixture - $bashFixture = New-ComponentCopyFixture - foreach ($fixture in @($powerShellFixture, $bashFixture)) { - $manifestPath = Join-Path $fixture.Source 'plugin.json' - $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json - $manifest.agents = @('agents/hve-core/rpi-agent.agent.md') - $manifest | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $manifestPath -NoNewline - } + It 'Rejects installable manifest entries outside .github' { + $manifestPath = Join-Path $script:fixture.Source 'plugin.json' + $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json + $manifest.agents = @('agents/hve-core/rpi-agent.agent.md') + $manifest | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $manifestPath -NoNewline - { Invoke-ComponentCopy -Fixture $powerShellFixture -Component @('agents/hve-core/rpi-agent.md') } | + { Invoke-ComponentCopy -Fixture $script:fixture -Component @('agents/hve-core/rpi-agent.md') } | Should -Throw -ExpectedMessage "*must start with '.github/'*" + } + + It 'Rejects installable manifest entries outside .github in the Bash implementation' -Skip:(-not $script:BashAvailable) { + $bashFixture = New-ComponentCopyFixture + $manifestPath = Join-Path $bashFixture.Source 'plugin.json' + $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json + $manifest.agents = @('agents/hve-core/rpi-agent.agent.md') + $manifest | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $manifestPath -NoNewline + $bashOutput = Invoke-BashComponentCopy -Fixture $bashFixture -Component @('agents/hve-core/rpi-agent.md') + $LASTEXITCODE | Should -Not -Be 0 $bashOutput | Should -Match "must start with '\.github/'" } From e479034a2dd6b5e0ab5bb1f7babb5947c9f0e9fc Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:09:07 -0700 Subject: [PATCH 24/28] build: pin the devcontainer base image by manifest digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolved from the publisher's registry metadata rather than a local pull, preserving the BASE_IMAGE override for mirrored builds. πŸ“Œ - Generated by Copilot --- .devcontainer/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e0bfbc7da..8fe734ca2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,9 @@ ARG BASE_IMAGE= -FROM ${BASE_IMAGE:-mcr.microsoft.com/devcontainers/base:2-jammy} +# mcr.microsoft.com/devcontainers/base:2-jammy, pinned by multi-architecture +# manifest digest (linux/amd64, linux/arm64) so the tag cannot move underneath +# a reviewed configuration. Resolve a replacement from the publisher's own +# registry metadata, not from a local pull. +FROM ${BASE_IMAGE:-mcr.microsoft.com/devcontainers/base@sha256:55126de3b3e113096201055951b4cb717900bafb379c59dcbada65c49cd7ce22} ARG NPM_CONFIG_REGISTRY= ENV NPM_CONFIG_REGISTRY=${NPM_CONFIG_REGISTRY:-https://registry.npmjs.org/} From 848ed7f8080276d85236278f0e3b8eff752a3cb7 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:09:40 -0700 Subject: [PATCH 25/28] test(skills): adopt BDD naming and Arrange/Act/Assert across the suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rename 283 tests to test_given_when_then across six modules - add Arrange/Act/Assert sections to every test body - prove statement structure identical by AST comparison - hold counts at 314 fast, 134 slow, per module and per lane ♻️ - Generated by Copilot --- .../tests/fuzz_harness.py | 27 +- .../tests/test_azure_templates.py | 345 +++++++++-- .../tests/test_collector_carriers.py | 170 ++++- .../tests/test_helpers.py | 442 +++++++++++-- .../tests/test_local_config.py | 275 +++++++-- .../tests/test_settings_upsert.py | 581 +++++++++++++++--- 6 files changed, 1557 insertions(+), 283 deletions(-) diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py b/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py index 022dbc16b..a4a43d78a 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/fuzz_harness.py @@ -68,21 +68,35 @@ def fuzz_local_config(data: bytes) -> None: class TestLocalConfigFuzzHarness: """Property tests mirroring the fuzz target.""" - def test_loader_raises_config_error_or_returns_mapping(self) -> None: + def test_given_assorted_yaml_text_when_the_loader_runs_then_a_mapping_or_config_error_results( + self, + ) -> None: + # Act & Assert for text in ("", "a: 1", "- list", "a: [\n", "null"): with suppress(ConfigError): assert isinstance(load_yaml_text(text), dict) - def test_redaction_never_emits_a_key_outside_the_allow_list(self) -> None: + def test_given_arbitrary_attribute_keys_when_redaction_runs_then_only_allow_listed_keys_remain( + self, + ) -> None: + # Act & Assert for key in ("service.name", "prompt.text", "", "0" * 200, "service.name.extra"): kept = simulate_redaction({key: "value"}, ALLOW, BLOCKED) assert set(kept) <= ALLOW - def test_a_blocked_value_is_masked_rather_than_leaked(self) -> None: + def test_given_an_allow_listed_key_with_a_blocked_value_when_redaction_runs_then_it_is_masked( + self, + ) -> None: + # Act kept = simulate_redaction({"http.url": "b" * 40}, ALLOW, (re.compile(r"b{40}"),)) + + # Assert assert kept == {"http.url": MASK} - def test_check_url_never_returns_a_non_http_scheme(self) -> None: + def test_given_malformed_url_candidates_when_check_url_runs_then_only_http_schemes_return( + self, + ) -> None: + # Act & Assert for candidate in ("", "file:///etc/passwd", "http://", "://", "http://[", "data:,"): try: parsed = check_url(candidate) @@ -90,7 +104,10 @@ def test_check_url_never_returns_a_non_http_scheme(self) -> None: continue assert parsed.scheme in ("http", "https") - def test_strip_jsonc_preserves_length_and_parses_what_it_can(self) -> None: + def test_given_jsonc_text_when_strip_jsonc_runs_then_length_is_preserved_and_json_parses( + self, + ) -> None: + # Act & Assert for text in ("", "{}", '{"a": 1} // trailing', "/* unterminated", '{"a": "//"}'): stripped = strip_jsonc(text) assert len(stripped) == len(text) diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py index e74c891bb..9e631f42c 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_azure_templates.py @@ -115,25 +115,49 @@ class TestReceiverAuthentication: """The fleet receiver authenticates senders, in configuration and not in prose.""" @pytest.mark.parametrize("protocol", ["http", "grpc"]) - def test_each_protocol_declares_an_authenticator(self, collector: dict, protocol: str) -> None: + def test_given_fleet_collector_when_reading_otlp_protocol_then_authenticator_is_bearertoken( + self, + collector: dict, + protocol: str, + ) -> None: + # Arrange config = collector["receivers"]["otlp"]["protocols"][protocol] + + # Assert assert config["auth"]["authenticator"] == "bearertokenauth" - def test_the_authenticator_extension_is_active(self, collector: dict) -> None: + def test_given_fleet_collector_when_reading_extensions_then_bearertokenauth_is_active( + self, + collector: dict, + ) -> None: + # Act & Assert assert "bearertokenauth" in collector["extensions"] assert "bearertokenauth" in collector["service"]["extensions"] - def test_the_token_is_an_environment_reference(self, collector: dict) -> None: + def test_given_fleet_bearertokenauth_when_reading_token_then_value_is_env_reference( + self, + collector: dict, + ) -> None: + # Arrange token = collector["extensions"]["bearertokenauth"]["token"] + + # Assert assert token.startswith("${env:") - def test_authentication_is_not_left_commented_out(self) -> None: + def test_given_fleet_collector_text_when_scanning_comments_then_no_commented_bearertokenauth( + self, + ) -> None: + # Arrange text = COLLECTOR_PATH.read_text(encoding="utf-8") + + # Act commented = [ line for line in text.splitlines() if line.strip().startswith("#") and "bearertokenauth:" in line ] + + # Assert assert commented == [], "authentication is present only as a comment" @@ -141,8 +165,15 @@ class TestReceiverTls: """TLS is configured, and its material comes from the environment.""" @pytest.mark.parametrize("protocol", ["http", "grpc"]) - def test_each_protocol_configures_tls(self, collector: dict, protocol: str) -> None: + def test_given_fleet_collector_when_reading_protocol_tls_then_cert_and_key_are_env_refs( + self, + collector: dict, + protocol: str, + ) -> None: + # Arrange tls = collector["receivers"]["otlp"]["protocols"][protocol]["tls"] + + # Assert assert tls["cert_file"].startswith("${env:") assert tls["key_file"].startswith("${env:") @@ -150,33 +181,55 @@ def test_each_protocol_configures_tls(self, collector: dict, protocol: str) -> N class TestEnvironmentSeparation: """Separation comes from separate deployments, not from an attribute.""" - def test_bicep_requires_an_environment_parameter(self, bicep: str) -> None: + def test_given_bicep_when_reading_environment_param_then_it_is_declared_without_default( + self, + bicep: str, + ) -> None: + # Act & Assert assert re.search(r"^param environment string$", bicep, re.MULTILINE) assert "param environment string = " not in bicep, "environment must have no default" - def test_terraform_requires_an_environment_variable(self, terraform: str) -> None: + def test_given_terraform_when_reading_environment_variable_then_it_has_no_default( + self, + terraform: str, + ) -> None: + # Act & Assert assert 'variable "environment"' in terraform block = terraform.split('variable "environment"', 1)[1].split("\nvariable ", 1)[0] assert "default" not in block, "environment must have no default" - def test_bicep_resource_names_include_the_environment(self, bicep: str) -> None: + def test_given_bicep_when_reading_resource_name_vars_then_each_interpolates_environment( + self, + bicep: str, + ) -> None: + # Act & Assert for name in ("workspaceName", "appInsightsName", "dashboardName"): line = next(line for line in bicep.splitlines() if line.startswith(f"var {name}")) assert "${environment}" in line - def test_terraform_resource_names_include_the_environment(self, terraform: str) -> None: + def test_given_terraform_when_reading_resource_prefix_then_names_interpolate_environment( + self, + terraform: str, + ) -> None: + # Act & Assert assert 'resource_prefix = "${var.name_prefix}-${var.environment}"' in terraform assert "${local.resource_prefix}-copilot-logs" in terraform assert "${local.resource_prefix}-copilot-insights" in terraform - def test_the_collector_environment_attribute_is_an_environment_reference( + def test_given_fleet_resource_processor_when_reading_environment_attribute_then_value_is_env( self, collector: dict ) -> None: + # Arrange attributes = collector["processors"]["resource/fleet"]["attributes"] by_key = {entry["key"]: entry for entry in attributes} + + # Assert assert by_key["deployment.environment.name"]["value"].startswith("${env:") - def test_no_file_calls_a_resource_attribute_an_isolation_control(self) -> None: + def test_given_all_templates_when_scanning_text_then_no_isolation_claim_for_attributes( + self, + ) -> None: + # Act & Assert for path in TEMPLATE_PATHS: text = path.read_text(encoding="utf-8").lower() for claim in ("namespace provides isolation", "isolated by service.namespace"): @@ -186,27 +239,44 @@ def test_no_file_calls_a_resource_attribute_an_isolation_control(self) -> None: class TestRetentionAndAccess: """Retention, caps, and reader scope agree across both templates.""" - def test_retention_defaults_match(self, bicep: str, terraform: str) -> None: + def test_given_bicep_and_terraform_when_reading_retention_defaults_then_both_are_ninety_days( + self, + bicep: str, + terraform: str, + ) -> None: + # Act & Assert assert "param retentionInDays int = 90" in bicep assert re.search(r'variable "retention_in_days"[\s\S]*?default\s*=\s*90', terraform) - def test_daily_cap_defaults_match(self, bicep: str, terraform: str) -> None: + def test_given_bicep_and_terraform_when_reading_daily_cap_defaults_then_both_are_five_gb( + self, + bicep: str, + terraform: str, + ) -> None: + # Act & Assert assert "param dailyQuotaGb int = 5" in bicep assert re.search(r'variable "daily_quota_gb"[\s\S]*?default\s*=\s*5', terraform) - def test_the_reader_assignment_is_opt_in(self, bicep: str, terraform: str) -> None: + def test_given_bicep_and_terraform_when_reading_reader_assignment_then_it_is_conditional( + self, + bicep: str, + terraform: str, + ) -> None: + # Act & Assert assert "if (!empty(readerPrincipalId))" in bicep assert 'count = var.reader_principal_id == "" ? 0 : 1' in terraform - def test_the_reader_assignment_is_scoped_to_the_workspace( + def test_given_bicep_and_terraform_when_reading_reader_scope_then_it_targets_the_workspace( self, bicep: str, terraform: str ) -> None: + # Act & Assert assert "scope: workspace" in bicep assert "scope = azurerm_log_analytics_workspace.this.id" in terraform - def test_retention_is_described_as_the_deletion_boundary( + def test_given_bicep_and_terraform_when_scanning_text_then_retention_is_a_deletion_policy( self, bicep: str, terraform: str ) -> None: + # Act & Assert assert "deletion policy" in bicep.lower() assert "deletion policy" in terraform.lower() @@ -221,46 +291,61 @@ class TestFleetPolicyParity: rather than restated, so drift in either file fails. """ - def test_the_fleet_allow_list_matches_the_local_allow_list( + def test_given_fleet_collector_when_comparing_allowed_keys_to_local_then_they_match( self, collector: dict, local_collector: dict ) -> None: + # Act & Assert assert ( redaction_policy(collector)["allowed_keys"] == redaction_policy(local_collector)["allowed_keys"] ) - def test_the_fleet_blocked_values_match_the_local_blocked_values( + def test_given_fleet_collector_when_comparing_blocked_values_to_local_then_they_match( self, collector: dict, local_collector: dict ) -> None: + # Act & Assert assert ( redaction_policy(collector)["blocked_values"] == redaction_policy(local_collector)["blocked_values"] ) - def test_the_fleet_policy_is_fail_closed_and_silent(self, collector: dict) -> None: + def test_given_fleet_collector_when_reading_redaction_policy_then_it_is_fail_closed_and_silent( + self, + collector: dict, + ) -> None: + # Act policy = redaction_policy(collector) + + # Assert assert policy["allow_all_keys"] is False assert policy["summary"] == "silent" - def test_the_fleet_scrub_statements_match_the_local_scrub_statements( + def test_given_fleet_collector_when_comparing_scrub_statements_to_local_then_they_match( self, collector: dict, local_collector: dict ) -> None: + # Act & Assert assert scrub_statements(collector) == scrub_statements(local_collector) @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) - def test_every_signal_applies_both_privacy_processors( + def test_given_fleet_signal_pipeline_when_reading_processors_then_both_privacy_steps_present( self, collector: dict, signal: str ) -> None: + # Arrange processors = collector["service"]["pipelines"][signal]["processors"] + + # Assert assert "redaction" in processors, f"{signal} reaches storage unfiltered" assert "transform/scrub" in processors, f"{signal} keeps ungoverned carriers" @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) - def test_the_privacy_processors_keep_their_local_relative_order( + def test_given_fleet_and_local_pipelines_when_comparing_privacy_order_then_it_is_unchanged( self, collector: dict, local_collector: dict, signal: str ) -> None: """The runtime carrier probe only substitutes definitions, not ordering.""" + # Arrange privacy = ("redaction", "transform/scrub") + + # Act fleet_order = [ step for step in collector["service"]["pipelines"][signal]["processors"] @@ -271,19 +356,28 @@ def test_the_privacy_processors_keep_their_local_relative_order( for step in local_collector["service"]["pipelines"][signal]["processors"] if step in privacy ] + + # Assert assert fleet_order == local_order @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) - def test_filtering_precedes_export_and_operator_identity( + def test_given_fleet_signal_pipeline_when_ordering_processors_then_filtering_precedes_export( self, collector: dict, signal: str ) -> None: + # Arrange processors = collector["service"]["pipelines"][signal]["processors"] + + # Assert assert processors.index("memory_limiter") < processors.index("redaction") assert processors.index("transform/scrub") < processors.index("resource/fleet") assert processors.index("resource/fleet") < processors.index("batch") - def test_the_superseded_delete_list_is_gone(self, collector: dict) -> None: + def test_given_fleet_collector_when_searching_for_strip_content_then_the_delete_list_is_absent( + self, + collector: dict, + ) -> None: """A delete-list cannot fail closed on an attribute it has never seen.""" + # Act & Assert assert "attributes/strip-content" not in collector["processors"] for pipeline in collector["service"]["pipelines"].values(): assert "attributes/strip-content" not in pipeline["processors"] @@ -292,27 +386,53 @@ def test_the_superseded_delete_list_is_gone(self, collector: dict) -> None: class TestRelayTopology: """The relay is reachable from this workstation and from nothing else.""" - def test_the_relay_declares_only_otlp_http(self, relay: dict) -> None: + def test_given_relay_collector_when_reading_otlp_protocols_then_only_http_is_declared( + self, + relay: dict, + ) -> None: + # Arrange protocols = relay["receivers"]["otlp"]["protocols"] + + # Assert assert set(protocols) == {"http"}, "a second ingress protocol would be unexercised" - def test_the_relay_receiver_takes_the_container_interface(self, relay: dict) -> None: + def test_given_relay_collector_when_reading_http_endpoint_then_it_binds_all_interfaces( + self, + relay: dict, + ) -> None: """Host exposure is the port mapping's decision, exactly as the local stack does it.""" + # Act & Assert assert relay["receivers"]["otlp"]["protocols"]["http"]["endpoint"] == "0.0.0.0:4318" - def test_the_relay_receiver_requires_no_credential(self, relay: dict) -> None: + def test_given_relay_collector_when_reading_http_receiver_then_no_auth_is_required( + self, + relay: dict, + ) -> None: """Both emitters export without one; requiring it here would drop everything.""" + # Act & Assert assert "auth" not in relay["receivers"]["otlp"]["protocols"]["http"] - def test_every_published_port_is_loopback(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_reading_published_ports_then_all_bind_loopback( + self, + relay_compose: dict, + ) -> None: + # Arrange ports = [str(entry) for entry in relay_compose["services"]["otel-relay"]["ports"]] + + # Assert assert ports, "the relay publishes nothing" for entry in ports: assert entry.startswith("127.0.0.1:"), f"{entry} is reachable off this workstation" assert "::" not in entry - def test_the_only_otlp_mapping_is_http_on_4318(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_reading_port_mappings_then_only_4318_http_exists( + self, + relay_compose: dict, + ) -> None: + # Arrange ports = [str(entry) for entry in relay_compose["services"]["otel-relay"]["ports"]] + + # Assert assert "127.0.0.1:4318:4318" in ports assert not [entry for entry in ports if ":4317:" in entry], "gRPC is unsupported" @@ -320,22 +440,42 @@ def test_the_only_otlp_mapping_is_http_on_4318(self, relay_compose: dict) -> Non class TestRelayUpstream: """The upstream hop is authenticated and its certificate is verified.""" - def test_the_endpoint_is_an_environment_reference(self, relay: dict) -> None: + def test_given_relay_fleet_exporter_when_reading_endpoint_then_it_is_an_env_reference( + self, + relay: dict, + ) -> None: + # Arrange exporter = relay["exporters"]["otlp_http/fleet"] + + # Assert assert exporter["endpoint"] == "${env:COPILOT_OTEL_FLEET_ENDPOINT}" - def test_the_exporter_authenticates(self, relay: dict) -> None: + def test_given_relay_fleet_exporter_when_reading_auth_then_bearertokenauth_uses_env_token( + self, + relay: dict, + ) -> None: + # Act & Assert assert relay["exporters"]["otlp_http/fleet"]["auth"]["authenticator"] == "bearertokenauth" assert relay["extensions"]["bearertokenauth"]["token"].startswith("${env:") assert "bearertokenauth" in relay["service"]["extensions"] - def test_the_exporter_verifies_against_the_mounted_bundle(self, relay: dict) -> None: + def test_given_relay_fleet_exporter_when_reading_tls_then_it_verifies_the_mounted_ca( + self, + relay: dict, + ) -> None: + # Arrange tls = relay["exporters"]["otlp_http/fleet"]["tls"] + + # Assert assert tls["ca_file"] == RELAY_CA_MOUNT assert "insecure" not in tls, "the upstream hop must not opt out of verification" assert "insecure_skip_verify" not in tls - def test_no_pipeline_exports_anywhere_else(self, relay: dict) -> None: + def test_given_relay_pipelines_when_reading_exporters_then_only_the_fleet_exporter_is_used( + self, + relay: dict, + ) -> None: + # Act & Assert for name, pipeline in relay["service"]["pipelines"].items(): assert pipeline["exporters"] == ["otlp_http/fleet"], f"{name} has a second destination" @@ -343,32 +483,38 @@ def test_no_pipeline_exports_anywhere_else(self, relay: dict) -> None: class TestRelayPolicyParity: """Telemetry is minimized on the workstation that produced it.""" - def test_the_relay_allow_list_matches_the_local_allow_list( + def test_given_relay_collector_when_comparing_allowed_keys_to_local_then_they_match( self, relay: dict, local_collector: dict ) -> None: + # Act & Assert assert ( redaction_policy(relay)["allowed_keys"] == redaction_policy(local_collector)["allowed_keys"] ) - def test_the_relay_blocked_values_match_the_local_blocked_values( + def test_given_relay_collector_when_comparing_blocked_values_to_local_then_they_match( self, relay: dict, local_collector: dict ) -> None: + # Act & Assert assert ( redaction_policy(relay)["blocked_values"] == redaction_policy(local_collector)["blocked_values"] ) - def test_the_relay_scrub_statements_match_the_local_scrub_statements( + def test_given_relay_collector_when_comparing_scrub_statements_to_local_then_they_match( self, relay: dict, local_collector: dict ) -> None: + # Act & Assert assert scrub_statements(relay) == scrub_statements(local_collector) @pytest.mark.parametrize("signal", ["traces", "metrics", "logs"]) - def test_every_signal_is_filtered_before_it_leaves_the_workstation( + def test_given_relay_signal_pipeline_when_ordering_processors_then_filtering_precedes_batch( self, relay: dict, signal: str ) -> None: + # Arrange processors = relay["service"]["pipelines"][signal]["processors"] + + # Assert assert processors.index("redaction") < processors.index("batch") assert processors.index("transform/scrub") < processors.index("batch") @@ -376,44 +522,86 @@ def test_every_signal_is_filtered_before_it_leaves_the_workstation( class TestRelayRuntime: """The relay outlives VS Code, holds no committed credential, and is bounded.""" - def test_the_image_is_digest_pinned(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_reading_the_image_then_it_is_digest_pinned( + self, + relay_compose: dict, + ) -> None: + # Act & Assert assert DIGEST_REFERENCE.match(relay_compose["services"]["otel-relay"]["image"]) - def test_the_relay_restarts_independently_of_the_editor(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_reading_restart_policy_then_it_is_unless_stopped( + self, + relay_compose: dict, + ) -> None: + # Act & Assert assert relay_compose["services"]["otel-relay"]["restart"] == "unless-stopped" - def test_the_container_is_hardened(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_reading_service_hardening_then_limits_are_applied( + self, + relay_compose: dict, + ) -> None: + # Arrange service = relay_compose["services"]["otel-relay"] + + # Assert assert service["read_only"] is True assert service["cap_drop"] == ["ALL"] assert "no-new-privileges:true" in service["security_opt"] assert service["mem_limit"] == "512m" @pytest.mark.parametrize("variable", sorted(RELAY_SENTINELS)) - def test_every_runtime_input_is_required(self, variable: str) -> None: + def test_given_relay_compose_text_when_reading_a_runtime_variable_then_it_fails_without_value( + self, + variable: str, + ) -> None: """A defaulted input would start a relay that silently discards everything.""" + # Arrange text = RELAY_COMPOSE_PATH.read_text(encoding="utf-8") + + # Act matches = [match for match in COMPOSE_VARIABLE.finditer(text) if match["name"] == variable] + + # Assert assert matches, f"{variable} is not read by the relay Compose document" for match in matches: assert match["message"], f"{variable} has no failure message and may default to empty" - def test_the_config_and_ca_bundle_are_mounted_read_only(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_reading_volumes_then_config_and_ca_are_read_only( + self, + relay_compose: dict, + ) -> None: + # Arrange volumes = [str(entry) for entry in relay_compose["services"]["otel-relay"]["volumes"]] + + # Act & Assert config_mount = next(entry for entry in volumes if entry.endswith("config.yaml:ro")) assert config_mount.startswith("./otel-collector-config.yaml:") ca_mount = next(entry for entry in volumes if RELAY_CA_MOUNT in entry) assert ca_mount == f"{RELAY_SENTINELS['COPILOT_OTEL_FLEET_CA_BUNDLE']}:{RELAY_CA_MOUNT}:ro" - def test_the_ca_bundle_comes_from_outside_this_repository(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_reading_the_ca_mount_then_its_source_is_absolute( + self, + relay_compose: dict, + ) -> None: """The skill ships no certificate authority, and could not ship a useful one.""" + # Arrange volumes = [str(entry) for entry in relay_compose["services"]["otel-relay"]["volumes"]] + + # Act ca_mount = next(entry for entry in volumes if RELAY_CA_MOUNT in entry) + + # Assert assert ca_mount.startswith("/"), "the bundle path must be absolute" assert not ca_mount.startswith("./") - def test_only_the_relay_service_receives_the_upstream_values(self, relay_compose: dict) -> None: + def test_given_rendered_relay_compose_when_scanning_services_then_only_relay_has_the_token( + self, + relay_compose: dict, + ) -> None: + # Arrange rendered = yaml.safe_dump(relay_compose) + + # Assert for service, definition in relay_compose["services"].items(): environment = definition.get("environment") or {} if service != "otel-relay": @@ -429,29 +617,58 @@ class TestManagedSettingsMatchTheRelay: managed settings change. """ - def test_the_managed_endpoint_is_the_relay_listener(self, relay_compose: dict) -> None: + def test_given_org_distribution_doc_when_reading_managed_endpoint_then_it_matches_relay_port( + self, + relay_compose: dict, + ) -> None: + # Arrange text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8") + + # Act & Assert assert '"endpoint": "http://127.0.0.1:4318"' in text ports = [str(entry) for entry in relay_compose["services"]["otel-relay"]["ports"]] assert "127.0.0.1:4318:4318" in ports - def test_the_managed_protocol_is_the_one_the_relay_serves(self, relay: dict) -> None: + def test_given_org_distribution_doc_when_reading_managed_protocol_then_it_matches_relay_http( + self, + relay: dict, + ) -> None: + # Arrange text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8") + + # Assert assert '"protocol": "otlp-http"' in text assert set(relay["receivers"]["otlp"]["protocols"]) == {"http"} - def test_managed_settings_carry_no_fleet_credential(self) -> None: + def test_given_org_distribution_json_block_when_reading_it_then_no_fleet_credential_appears( + self, + ) -> None: + # Arrange text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8") + + # Act block = text.split("```json", 1)[1].split("```", 1)[0] + + # Assert assert "headers" not in block, "the managed block must not distribute the fleet token" assert "Bearer" not in block - def test_grpc_is_named_unsupported_rather_than_untested(self) -> None: + def test_given_org_distribution_doc_when_searching_grpc_wording_then_it_is_named_unsupported( + self, + ) -> None: + # Arrange text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8").lower() + + # Assert assert "no grpc listener" in text or "there is no grpc listener" in text - def test_the_relay_prerequisite_precedes_the_managed_change(self) -> None: + def test_given_org_distribution_doc_when_reading_rollout_prose_then_relay_comes_first( + self, + ) -> None: + # Arrange text = ORG_DISTRIBUTION_PATH.read_text(encoding="utf-8").lower() + + # Assert assert "distributing this block" in text assert "sends no telemetry at all" in text assert "rolling back" in text or "rollback" in text @@ -461,21 +678,43 @@ class TestStateAndSecrets: """No template carries a credential, and state sensitivity is stated.""" @pytest.mark.parametrize("path", TEMPLATE_PATHS, ids=lambda p: p.name) - def test_no_literal_secret_is_committed(self, path: pathlib.Path) -> None: + def test_given_a_template_file_when_scanning_for_secret_patterns_then_none_match( + self, + path: pathlib.Path, + ) -> None: + # Arrange text = path.read_text(encoding="utf-8") + + # Assert for pattern in SECRET_PATTERNS: assert not pattern.search(text), f"{path.name} matches {pattern.pattern}" - def test_the_connection_string_output_is_marked_sensitive(self) -> None: + def test_given_terraform_outputs_when_reading_connection_string_then_it_is_marked_sensitive( + self, + ) -> None: + # Arrange text = OUTPUTS_TF_PATH.read_text(encoding="utf-8") + + # Act block = text.split('output "connection_string"', 1)[1] + + # Assert assert "sensitive = true" in block - def test_state_sensitivity_is_documented_rather_than_implied(self) -> None: + def test_given_main_terraform_when_reading_state_prose_then_sensitivity_is_documented( + self, + ) -> None: + # Arrange text = MAIN_TF_PATH.read_text(encoding="utf-8").lower() + + # Assert assert "state file" in text assert "does not remove it from state" in text - def test_bicep_emits_a_command_rather_than_the_connection_string(self, bicep: str) -> None: + def test_given_bicep_outputs_when_reading_them_then_a_command_replaces_the_connection_string( + self, + bicep: str, + ) -> None: + # Act & Assert assert "output connectionStringCommand string" in bicep assert "output connectionString string =" not in bicep diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py index aefeba112..21640ff72 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_collector_carriers.py @@ -841,64 +841,113 @@ def control_output(runtime: ContainerRuntime) -> str: class TestMarkerInventory: """The markers must be distinguishable before any classification means anything.""" - def test_every_carrier_has_a_unique_marker(self) -> None: + def test_given_the_carrier_map_when_markers_are_collected_then_each_is_unique(self) -> None: + # Act markers = [carrier.marker for carrier in CARRIERS] + + # Assert assert len(markers) == len(set(markers)) - def test_no_marker_is_a_substring_of_another(self) -> None: + def test_given_the_carrier_map_when_markers_are_compared_pairwise_then_none_is_a_substring( + self, + ) -> None: + # Act overlapping = [ (one.name, other.name) for one in CARRIERS for other in CARRIERS if one is not other and one.marker in other.marker ] + + # Assert assert overlapping == [] - def test_every_carrier_declares_a_known_classification(self) -> None: + def test_given_the_carrier_map_when_expectations_are_checked_then_each_class_is_known( + self, + ) -> None: + # Arrange allowed = {GOVERNED, MASKED, PASSED_THROUGH} + + # Act unknown = [c.name for c in CARRIERS if c.expected not in allowed] + + # Assert assert unknown == [], ( "An unobservable or unclassified carrier is a gap, not a baseline. " "Record it in SECURITY.md rather than encoding it as an expectation." ) - def test_a_masked_expectation_declares_what_masking_looks_like(self) -> None: + def test_given_a_masked_carrier_when_its_entry_is_read_then_a_masked_signature_is_declared( + self, + ) -> None: + # Act undeclared = [c.name for c in CARRIERS if c.expected == MASKED and not c.masked_signature] + + # Assert assert undeclared == [] - def test_a_passed_through_carrier_declares_no_mechanism(self) -> None: + def test_given_a_passed_through_carrier_when_its_entry_is_read_then_no_mechanism_is_claimed( + self, + ) -> None: """A carrier cannot claim a control and pass through at the same time.""" + # Act contradictory = [ c.name for c in CARRIERS if c.expected == PASSED_THROUGH and c.mechanism != NO_MECHANISM ] + + # Assert assert contradictory == [] - def test_a_handled_carrier_names_the_mechanism_that_handles_it(self) -> None: + def test_given_a_governed_or_masked_carrier_when_its_entry_is_read_then_a_mechanism_is_named( + self, + ) -> None: + # Act unattributed = [ c.name for c in CARRIERS if c.expected in {GOVERNED, MASKED} and c.mechanism == NO_MECHANISM ] + + # Assert assert unattributed == [] class TestProbeConfigDerivation: """The configuration under test must be the shipped one, minus stated changes.""" - def test_the_probe_preserves_the_shipped_processor_order(self) -> None: + def test_given_the_shipped_config_when_a_probe_config_is_derived_then_processor_order_is_kept( + self, + ) -> None: + # Arrange shipped = shipped_collector_config() + + # Act derived = derive_probe_config(remove_content_processors=False) + + # Assert for name, pipeline in derived["service"]["pipelines"].items(): assert pipeline["processors"] == shipped["service"]["pipelines"][name]["processors"] - def test_the_probe_preserves_the_shipped_redaction_policy(self) -> None: + def test_given_the_shipped_config_when_a_probe_config_is_derived_then_redaction_policy_is_kept( + self, + ) -> None: + # Arrange shipped = shipped_collector_config() + + # Act derived = derive_probe_config(remove_content_processors=False) + + # Assert assert derived["processors"]["redaction"] == shipped["processors"]["redaction"] - def test_the_control_removes_only_the_content_processors(self) -> None: + def test_given_a_control_config_when_it_is_derived_then_only_content_processors_are_removed( + self, + ) -> None: + # Act policy = derive_probe_config(remove_content_processors=False) control = derive_probe_config(remove_content_processors=True) + + # Assert assert set(policy["processors"]) - set(control["processors"]) == set(CONTENT_PROCESSORS) for name, pipeline in control["service"]["pipelines"].items(): expected = [ @@ -908,19 +957,34 @@ def test_the_control_removes_only_the_content_processors(self) -> None: ] assert pipeline["processors"] == expected - def test_the_probe_runs_the_image_the_stack_pins(self) -> None: + def test_given_the_compose_file_when_the_probe_image_is_resolved_then_it_is_digest_pinned( + self, + ) -> None: + # Act & Assert assert "@sha256:" in pinned_collector_image() - def test_a_fleet_run_uses_the_fleet_content_processors(self) -> None: + def test_given_fleet_policy_when_a_probe_config_is_derived_then_fleet_content_processors_apply( + self, + ) -> None: + # Arrange fleet = policy_document(FLEET_POLICY) + + # Act derived = derive_probe_config(remove_content_processors=False, policy_source=FLEET_POLICY) + + # Assert for name in CONTENT_PROCESSORS: assert derived["processors"][name] == fleet["processors"][name] - def test_a_fleet_run_substitutes_nothing_but_the_content_processors(self) -> None: + def test_given_fleet_policy_when_a_probe_config_is_derived_then_transport_blocks_are_unchanged( + self, + ) -> None: """Swapping transport as well would test the fleet's network, not its policy.""" + # Act local = derive_probe_config(remove_content_processors=False) fleet = derive_probe_config(remove_content_processors=False, policy_source=FLEET_POLICY) + + # Assert assert local["receivers"] == fleet["receivers"] assert local["exporters"] == fleet["exporters"] assert local["service"] == fleet["service"] @@ -956,32 +1020,48 @@ class TestStrictRuntimeMode: ({STRICT_RUNTIME_ENV: ""}, False), ], ) - def test_strictness_follows_the_environment(self, env: dict[str, str], expected: bool) -> None: + def test_given_an_environment_mapping_when_strictness_is_resolved_then_the_flag_decides( + self, + env: dict[str, str], + expected: bool, + ) -> None: + # Act & Assert assert strict_runtime_required(env) is expected - def test_a_missing_runtime_fails_when_strictness_is_on( + def test_given_no_container_runtime_when_strictness_is_on_then_resolution_fails_loudly( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange monkeypatch.setitem(globals(), "detect_runtime", lambda: None) monkeypatch.setenv(STRICT_RUNTIME_ENV, "1") + + # Act with pytest.raises(pytest.fail.Exception) as raised: resolve_runtime() + + # Assert assert STRICT_RUNTIME_ENV in str(raised.value) - def test_a_missing_runtime_skips_when_strictness_is_off( + def test_given_no_container_runtime_when_strictness_is_off_then_resolution_skips( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange monkeypatch.setitem(globals(), "detect_runtime", lambda: None) monkeypatch.setenv(STRICT_RUNTIME_ENV, "0") + + # Act & Assert with pytest.raises(pytest.skip.Exception): resolve_runtime() - def test_a_reachable_runtime_is_returned_under_strictness( + def test_given_a_reachable_runtime_when_strictness_is_on_then_that_runtime_is_returned( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange stub = ContainerRuntime(["docker"], "stub runtime") monkeypatch.setitem(globals(), "detect_runtime", lambda: stub) monkeypatch.setenv(STRICT_RUNTIME_ENV, "1") + + # Act & Assert assert resolve_runtime() is stub @@ -993,8 +1073,14 @@ class TestProcessorChainPosition: """ @pytest.mark.parametrize("processor", CONTENT_PROCESSORS) - def test_the_processor_sits_between_the_limiter_and_the_batcher(self, processor: str) -> None: + def test_given_a_content_processor_when_the_chain_is_read_then_it_follows_the_limiter( + self, + processor: str, + ) -> None: + # Act pipelines = shipped_collector_config()["service"]["pipelines"] + + # Assert for name, pipeline in pipelines.items(): chain = pipeline["processors"] assert processor in chain, f"{name} does not apply {processor}" @@ -1007,9 +1093,10 @@ class TestNegativeControl: """Validate the instrument before trusting anything it reports.""" @pytest.mark.parametrize("carrier", CARRIERS, ids=lambda c: c.name) - def test_the_control_without_content_processors_renders_every_marker( + def test_given_the_control_run_when_a_carrier_is_probed_then_its_marker_is_rendered( self, carrier: Carrier, control_output: str ) -> None: + # Assert assert carrier.marker in control_output, ( f"{carrier.name} is not rendered even with the content processors " f"removed, so its absence under policy proves nothing. Classify it " @@ -1021,10 +1108,13 @@ class TestCarrierMap: """The recorded map is the regression boundary.""" @pytest.mark.parametrize("carrier", CARRIERS, ids=lambda c: c.name) - def test_each_carrier_matches_its_recorded_classification( + def test_given_paired_policy_and_control_runs_when_a_carrier_is_classified_then_the_map_holds( self, carrier: Carrier, policy_output: str, control_output: str ) -> None: + # Act observed = classify(carrier, policy_output, control_output) + + # Assert assert observed == carrier.expected, ( f"{carrier.name} is now {observed}, recorded as {carrier.expected}. " "Rebaselining this entry is a policy decision and needs its own " @@ -1043,15 +1133,21 @@ class TestRestoredDetectionAttributes: """ @pytest.mark.parametrize("key", sorted(RESTORED_DETECTION_KEYS)) - def test_the_shipped_policy_keeps_the_attribute(self, key: str, policy_output: str) -> None: + def test_given_a_restored_detection_key_when_the_shipped_policy_runs_then_its_marker_survives( + self, + key: str, + policy_output: str, + ) -> None: + # Assert assert RESTORED_DETECTION_KEYS[key] in policy_output, ( f"{key} is in the allow-list but was dropped by the running Collector" ) @pytest.mark.parametrize("key", sorted(RESTORED_DETECTION_KEYS)) - def test_the_attribute_is_rendered_under_its_own_key( + def test_given_a_restored_detection_key_when_the_exporter_renders_it_then_its_own_key_is_used( self, key: str, policy_output: str ) -> None: + # Assert assert f"-> {key}: Str({RESTORED_DETECTION_KEYS[key]})" in policy_output @@ -1063,10 +1159,13 @@ class TestShippedConfigurationStartsClean: runs the file exactly as an operator runs it. """ - def test_the_shipped_configuration_logs_no_deprecation_warning( + def test_given_the_shipped_configuration_when_the_collector_starts_then_no_deprecation_logs( self, runtime: ContainerRuntime ) -> None: + # Arrange container = f"copilot-otel-alias-check-{_free_port()}" + + # Act started = runtime.docker( "run", "-d", @@ -1091,6 +1190,8 @@ def test_the_shipped_configuration_logs_no_deprecation_warning( if "Everything is ready" in output: break time.sleep(1) + + # Assert assert "Everything is ready" in output, f"Collector never finished starting:\n{output}" assert "deprecated" not in output.lower(), ( f"the shipped configuration still triggers a deprecation warning:\n{output}" @@ -1436,38 +1537,53 @@ class TestAgentHostRelay: found by a fleet already sending telemetry somewhere it should not. """ - def test_a_trusted_receiver_with_the_right_credential_receives_the_span( + def test_given_a_trusted_receiver_and_credential_when_the_relay_forwards_then_the_span_arrives( self, relay_results: dict[str, tuple[str, str]] ) -> None: + # Arrange receiver_output, _ = relay_results["trusted"] + + # Assert assert "AGENTHOSTMARKER" in receiver_output, ( "the relay did not forward agent-host telemetry over verified TLS" ) - def test_the_forwarded_span_keeps_its_agent_host_identity( + def test_given_a_trusted_relay_run_when_the_span_arrives_then_agent_host_identity_is_kept( self, relay_results: dict[str, tuple[str, str]] ) -> None: + # Arrange receiver_output, _ = relay_results["trusted"] + + # Assert assert "copilot-agent-host" in receiver_output @pytest.mark.parametrize("case", [entry.name for entry in RELAY_CASES if not entry.forwards]) - def test_a_rejected_transport_forwards_nothing( + def test_given_a_rejected_transport_case_when_the_relay_runs_then_nothing_reaches_the_receiver( self, relay_results: dict[str, tuple[str, str]], case: str ) -> None: + # Arrange receiver_output, _ = relay_results[case] + + # Assert assert "AGENTHOSTMARKER" not in receiver_output, ( f"the {case} case reached the fleet receiver anyway" ) - def test_a_wrong_bearer_is_refused_by_the_receiver( + def test_given_a_wrong_bearer_token_when_the_relay_forwards_then_the_receiver_answers_401( self, relay_results: dict[str, tuple[str, str]] ) -> None: + # Arrange _, relay_output = relay_results["wrong-bearer"] + + # Assert assert "401" in relay_output or "unauthorized" in relay_output.lower() @pytest.mark.parametrize("case", ["untrusted-ca", "hostname-mismatch"]) - def test_a_certificate_failure_is_reported_as_one( + def test_given_a_bad_certificate_case_when_the_relay_dials_then_a_tls_failure_is_logged( self, relay_results: dict[str, tuple[str, str]], case: str ) -> None: + # Arrange _, relay_output = relay_results[case] + + # Assert assert "certificate" in relay_output.lower() or "tls" in relay_output.lower() diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py index 3b93ab7ff..7db272105 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_helpers.py @@ -80,11 +80,18 @@ class TestScheme: "//localhost:3000/api", ], ) - def test_a_non_http_scheme_is_refused(self, url: str) -> None: + def test_given_a_non_http_scheme_url_when_check_url_runs_then_it_raises_policy_error( + self, + url: str, + ) -> None: + # Act & Assert with pytest.raises(PolicyError): check_url(url) - def test_loopback_http_is_allowed(self) -> None: + def test_given_a_loopback_http_url_when_check_url_runs_then_the_hostname_is_returned( + self, + ) -> None: + # Act & Assert assert check_url("http://localhost:3000/api/health").hostname == "localhost" @@ -99,15 +106,25 @@ class TestAuthority: "http://:pass@localhost:3000/", ], ) - def test_userinfo_is_refused(self, url: str) -> None: + def test_given_a_url_with_userinfo_when_check_url_runs_then_it_refuses_credentials( + self, + url: str, + ) -> None: + # Act & Assert with pytest.raises(PolicyError, match="credentials"): check_url(url) - def test_a_missing_host_is_refused(self) -> None: + def test_given_a_url_without_a_host_when_check_url_runs_then_it_raises_policy_error( + self, + ) -> None: + # Act & Assert with pytest.raises(PolicyError): check_url("http:///api/health") - def test_a_malformed_port_is_refused(self) -> None: + def test_given_a_url_with_a_malformed_port_when_check_url_runs_then_it_raises_policy_error( + self, + ) -> None: + # Act & Assert with pytest.raises(PolicyError): check_url("http://localhost:notaport/") @@ -116,11 +133,19 @@ class TestPorts: """Local requests stay on the stack's own ports.""" @pytest.mark.parametrize("port", sorted(DEFAULT_ALLOWED_PORTS)) - def test_each_stack_port_is_allowed(self, port: int) -> None: + def test_given_a_default_allowed_stack_port_when_check_url_runs_then_the_port_is_preserved( + self, + port: int, + ) -> None: + # Act & Assert assert check_url(f"http://127.0.0.1:{port}/").port == port @pytest.mark.parametrize("port", [22, 80, 443, 8080, 5432]) - def test_an_unrelated_local_port_is_refused(self, port: int) -> None: + def test_given_a_loopback_url_on_another_port_when_check_url_runs_then_it_is_refused( + self, + port: int, + ) -> None: + # Act & Assert with pytest.raises(PolicyError, match="local port"): check_url(f"http://127.0.0.1:{port}/") @@ -128,25 +153,46 @@ def test_an_unrelated_local_port_is_refused(self, port: int) -> None: class TestRemoteOptIn: """A remote target needs an explicit opt-in, TLS, and a globally routable address.""" - def test_a_remote_host_is_refused_without_opt_in(self, resolves) -> None: + def test_given_a_remote_host_without_opt_in_when_check_url_runs_then_it_refuses_as_non_loopback( + self, + resolves, + ) -> None: + # Arrange resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) + + # Act & Assert with pytest.raises(PolicyError, match="non-loopback"): check_url("https://grafana.example.com/") - def test_a_remote_host_is_allowed_with_opt_in(self, resolves) -> None: + def test_given_a_globally_routable_host_with_opt_in_when_check_url_runs_then_https_is_accepted( + self, + resolves, + ) -> None: + # Arrange resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) + + # Act & Assert assert check_url("https://grafana.example.com/", allow_remote=True).scheme == "https" - def test_plaintext_remote_is_refused_even_with_opt_in(self, resolves) -> None: + def test_given_an_http_remote_host_with_opt_in_when_check_url_runs_then_it_refuses_plaintext( + self, + resolves, + ) -> None: + # Arrange resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) + + # Act & Assert with pytest.raises(PolicyError, match="plaintext"): check_url("http://grafana.example.com/", allow_remote=True) - def test_an_opted_in_remote_name_resolving_inside_the_network_is_refused( + def test_given_an_opted_in_private_name_when_check_url_runs_then_non_global_is_refused( self, resolves ) -> None: """The opt-in permits a remote target, not a route back into the network.""" + # Arrange resolves({"grafana.example.com": [PRIVATE_ADDRESS]}) + + # Act & Assert with pytest.raises(PolicyError, match="non-global"): check_url("https://grafana.example.com/", allow_remote=True) @@ -154,26 +200,48 @@ def test_an_opted_in_remote_name_resolving_inside_the_network_is_refused( class TestHostResolution: """A name is classified by where it resolves, not by how it is spelled.""" - def test_a_local_looking_name_that_resolves_off_machine_is_not_local(self, resolves) -> None: + def test_given_localhost_resolving_globally_when_it_is_classified_then_it_is_not_loopback( + self, + resolves, + ) -> None: + # Arrange resolves({"localhost": [GLOBAL_ADDRESS]}) + + # Act & Assert assert is_loopback_host("localhost") is False with pytest.raises(PolicyError, match="non-loopback"): check_url("http://localhost:3000/") - def test_a_name_resolving_to_both_is_refused(self, resolves) -> None: + def test_given_a_name_resolving_to_loopback_and_global_when_it_is_checked_then_it_is_refused( + self, + resolves, + ) -> None: """Treating a mixed answer as local would permit a connection off machine.""" + # Arrange resolves({"split.example": ["127.0.0.1", GLOBAL_ADDRESS]}) + + # Act & Assert assert is_loopback_host("split.example") is False with pytest.raises(PolicyError, match="loopback and non-loopback"): check_url("http://split.example:3000/") - def test_a_name_that_does_not_resolve_is_refused(self, resolves) -> None: + def test_given_a_name_that_does_not_resolve_when_check_url_runs_then_it_is_refused( + self, + resolves, + ) -> None: + # Arrange resolves({}) + + # Act & Assert with pytest.raises(PolicyError, match="does not resolve"): check_url("http://nowhere.invalid:3000/") @pytest.mark.parametrize("literal", ["127.0.0.1", "[::1]"]) - def test_an_address_literal_needs_no_resolver(self, literal: str) -> None: + def test_given_a_loopback_address_literal_when_it_is_classified_then_no_resolver_is_needed( + self, + literal: str, + ) -> None: + # Act & Assert assert is_loopback_host(literal) is True @@ -184,34 +252,57 @@ def _handler(self) -> urllib.request.HTTPRedirectHandler: opener = open_url.__globals__["_PolicyRedirectHandler"] return opener(allow_remote=False, allowed_ports=DEFAULT_ALLOWED_PORTS) - def test_a_redirect_off_loopback_is_refused(self) -> None: + def test_given_a_loopback_request_when_it_redirects_off_loopback_then_it_is_refused( + self, + ) -> None: + # Arrange request = urllib.request.Request("http://localhost:3000/api") + + # Act & Assert with pytest.raises(PolicyError, match="non-loopback"): self._handler().redirect_request( request, None, 302, "Found", {}, f"http://{GLOBAL_ADDRESS}/" ) - def test_a_redirect_to_a_file_url_is_refused(self) -> None: + def test_given_a_loopback_request_when_it_redirects_to_a_file_url_then_the_scheme_is_refused( + self, + ) -> None: + # Arrange request = urllib.request.Request("http://localhost:3000/api") + + # Act & Assert with pytest.raises(PolicyError, match="scheme"): self._handler().redirect_request(request, None, 302, "Found", {}, "file:///etc/passwd") - def test_a_redirect_to_an_unrelated_local_port_is_refused(self) -> None: + def test_given_a_loopback_request_when_it_redirects_to_port_22_then_the_local_port_is_refused( + self, + ) -> None: + # Arrange request = urllib.request.Request("http://localhost:3000/api") + + # Act & Assert with pytest.raises(PolicyError, match="local port"): self._handler().redirect_request( request, None, 302, "Found", {}, "http://localhost:22/" ) - def test_open_url_refuses_before_opening_anything(self, monkeypatch) -> None: + def test_given_a_disallowed_url_when_open_url_runs_then_it_refuses_before_calling_the_opener( + self, + monkeypatch, + ) -> None: + # Arrange opened: list[str] = [] monkeypatch.setattr( urllib.request.OpenerDirector, "open", lambda self, *a, **k: opened.append("opened"), ) + + # Act with pytest.raises(PolicyError): open_url("http://evil.example.com/") + + # Assert assert opened == [], "a refused URL still reached the opener" @@ -240,8 +331,14 @@ def _built_opener(self, monkeypatch) -> urllib.request.OpenerDirector: open_url("http://127.0.0.1:3000/api/health") return captured[0] - def test_the_built_opener_carries_no_proxy_configuration(self, monkeypatch) -> None: + def test_given_proxy_env_variables_when_open_url_builds_an_opener_then_no_proxy_is_configured( + self, + monkeypatch, + ) -> None: + # Act opener = self._built_opener(monkeypatch) + + # Assert configured = { scheme: target for handler in opener.handlers @@ -250,9 +347,17 @@ def test_the_built_opener_carries_no_proxy_configuration(self, monkeypatch) -> N } assert configured == {}, f"open_url would proxy loopback traffic to {configured}" - def test_the_built_opener_keeps_the_policy_redirect_handler(self, monkeypatch) -> None: + def test_given_open_url_when_it_builds_an_opener_then_the_policy_redirect_handler_is_kept( + self, + monkeypatch, + ) -> None: + # Arrange policy_handler = open_url.__globals__["_PolicyRedirectHandler"] + + # Act opener = self._built_opener(monkeypatch) + + # Assert assert any(isinstance(handler, policy_handler) for handler in opener.handlers), ( "the redirect allow-list re-check and cross-origin credential stripping were dropped" ) @@ -261,19 +366,37 @@ def test_the_built_opener_keeps_the_policy_redirect_handler(self, monkeypatch) - class TestPathContainment: """A configurable path cannot escape its root.""" - def test_a_contained_relative_path_resolves(self, tmp_path: pathlib.Path) -> None: + def test_given_a_relative_path_when_contain_path_runs_then_it_resolves_inside_the_root( + self, + tmp_path: pathlib.Path, + ) -> None: + # Act & Assert assert contain_path("snapshot.json", tmp_path) == (tmp_path / "snapshot.json").resolve() - def test_traversal_is_refused(self, tmp_path: pathlib.Path) -> None: + def test_given_a_traversal_path_when_contain_path_runs_then_it_is_refused( + self, + tmp_path: pathlib.Path, + ) -> None: + # Act & Assert with pytest.raises(PolicyError): contain_path("../../etc/passwd", tmp_path) - def test_an_absolute_path_outside_the_root_is_refused(self, tmp_path: pathlib.Path) -> None: + def test_given_an_absolute_path_outside_the_root_when_contain_path_runs_then_it_is_refused( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange outside = tmp_path.parent / "outside.json" + + # Act & Assert with pytest.raises(PolicyError): contain_path(outside, tmp_path) - def test_a_symlink_pointing_outside_the_root_is_refused(self, tmp_path: pathlib.Path) -> None: + def test_given_a_symlink_pointing_outside_the_root_when_contain_path_runs_then_it_is_refused( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange root = tmp_path / "root" root.mkdir() target = tmp_path / "outside" @@ -283,39 +406,71 @@ def test_a_symlink_pointing_outside_the_root_is_refused(self, tmp_path: pathlib. link.symlink_to(target, target_is_directory=True) except (OSError, NotImplementedError): pytest.skip("symlink creation is not permitted in this environment") + + # Act & Assert with pytest.raises(PolicyError): contain_path(link / "snapshot.json", root) - def test_the_root_itself_is_allowed(self, tmp_path: pathlib.Path) -> None: + def test_given_the_root_itself_when_contain_path_runs_then_the_resolved_root_is_returned( + self, + tmp_path: pathlib.Path, + ) -> None: + # Act & Assert assert contain_path(tmp_path, tmp_path) == tmp_path.resolve() class TestCredentials: """Missing configuration is reported as configuration, before any request.""" - def test_both_missing_variables_are_named(self, monkeypatch) -> None: + def test_given_both_credential_env_vars_missing_when_they_are_required_then_both_are_named( + self, + monkeypatch, + ) -> None: + # Arrange monkeypatch.delenv("COPILOT_OTEL_GRAFANA_USER", raising=False) monkeypatch.delenv("COPILOT_OTEL_GRAFANA_PASSWORD", raising=False) + + # Act with pytest.raises(PolicyError) as excinfo: require_credentials("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD") + + # Assert assert "COPILOT_OTEL_GRAFANA_USER" in str(excinfo.value) assert "COPILOT_OTEL_GRAFANA_PASSWORD" in str(excinfo.value) - def test_a_single_missing_variable_is_named(self, monkeypatch) -> None: + def test_given_only_the_password_var_missing_when_they_are_required_then_it_is_named( + self, + monkeypatch, + ) -> None: + # Arrange monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", "admin") monkeypatch.delenv("COPILOT_OTEL_GRAFANA_PASSWORD", raising=False) + + # Act & Assert with pytest.raises(PolicyError, match="COPILOT_OTEL_GRAFANA_PASSWORD"): require_credentials("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD") - def test_an_empty_value_counts_as_missing(self, monkeypatch) -> None: + def test_given_an_empty_password_value_when_they_are_required_then_it_counts_as_missing( + self, + monkeypatch, + ) -> None: + # Arrange monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", "admin") monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", "") + + # Act & Assert with pytest.raises(PolicyError, match="COPILOT_OTEL_GRAFANA_PASSWORD"): require_credentials("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD") - def test_supplied_credentials_are_returned_in_order(self, monkeypatch) -> None: + def test_given_both_variables_set_when_they_are_required_then_the_pair_is_returned_in_order( + self, + monkeypatch, + ) -> None: + # Arrange monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", "operator") monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", "chosen-by-the-user") + + # Act & Assert assert require_credentials( "COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD" ) == ("operator", "chosen-by-the-user") @@ -351,27 +506,41 @@ class TestHelperWiring: policy tests above green while restoring the exact write this change closed. """ - def test_baseline_refuses_a_snapshot_path_outside_the_cache_root( + def test_given_an_escaping_snapshot_path_when_baseline_captures_then_nothing_is_written( self, tmp_path: pathlib.Path ) -> None: + # Arrange target = tmp_path / "escaped-snapshot.json" + + # Act result = run_helper("baseline.py", ["capture"], {"COPILOT_OTEL_BASELINE": str(target)}) + + # Assert assert result.returncode != 0 assert "refusing a path outside" in (result.stderr + result.stdout) assert not target.exists(), "a refused snapshot path was still written" - def test_baseline_refuses_a_traversal_snapshot_path(self, tmp_path: pathlib.Path) -> None: + def test_given_a_traversal_snapshot_path_when_baseline_captures_then_it_exits_nonzero( + self, + tmp_path: pathlib.Path, + ) -> None: + # Act result = run_helper( "baseline.py", ["capture"], {"COPILOT_OTEL_BASELINE": "../../escaped.json"} ) + + # Assert assert result.returncode != 0 assert "refusing a path outside" in (result.stderr + result.stdout) - def test_validate_dashboard_reports_a_malformed_dashboard_without_a_traceback( + def test_given_a_malformed_dashboard_when_the_helper_runs_then_it_reports_without_a_traceback( self, tmp_path: pathlib.Path ) -> None: + # Arrange broken = tmp_path / "generated-dashboard.json" broken.write_text("{not json", encoding="utf-8") + + # Act result = run_helper( "validate_dashboard.py", [str(broken)], @@ -380,12 +549,17 @@ def test_validate_dashboard_reports_a_malformed_dashboard_without_a_traceback( "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", }, ) + + # Assert assert result.returncode == 1 combined = result.stderr + result.stdout assert "not valid JSON" in combined assert "Traceback" not in combined - def test_validate_dashboard_refuses_a_non_loopback_endpoint_without_opt_in(self) -> None: + def test_given_a_non_loopback_grafana_endpoint_when_the_helper_runs_then_it_is_refused( + self, + ) -> None: + # Act result = run_helper( "validate_dashboard.py", [], @@ -395,11 +569,16 @@ def test_validate_dashboard_refuses_a_non_loopback_endpoint_without_opt_in(self) "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", }, ) + + # Assert assert result.returncode != 0 assert "non-loopback" in (result.stderr + result.stdout) - def test_validate_dashboard_refuses_an_unguarded_query_endpoint(self) -> None: + def test_given_a_non_loopback_prometheus_endpoint_when_the_helper_runs_then_it_is_refused( + self, + ) -> None: """The Prometheus override was previously unchecked entirely.""" + # Act result = run_helper( "validate_dashboard.py", [], @@ -409,15 +588,22 @@ def test_validate_dashboard_refuses_an_unguarded_query_endpoint(self) -> None: "COPILOT_OTEL_GRAFANA_PASSWORD": "chosen-by-the-user", }, ) + + # Assert assert result.returncode != 0 assert "non-loopback" in (result.stderr + result.stdout) - def test_validate_dashboard_reports_missing_credentials_as_configuration(self) -> None: + def test_given_empty_grafana_credentials_when_the_helper_runs_then_both_variables_are_named( + self, + ) -> None: + # Act result = run_helper( "validate_dashboard.py", [], {"COPILOT_OTEL_GRAFANA_USER": "", "COPILOT_OTEL_GRAFANA_PASSWORD": ""}, ) + + # Assert assert result.returncode != 0 combined = result.stderr + result.stdout assert "COPILOT_OTEL_GRAFANA_USER" in combined @@ -436,43 +622,69 @@ class TestDashboardSelection: def _dashboard(self) -> dict: return {"panels": [{"title": "one", "type": "row"}]} - def test_a_generated_dashboard_outside_the_skill_is_accepted( + def test_given_a_dashboard_outside_the_skill_when_load_dashboard_runs_then_it_is_loaded( self, tmp_path: pathlib.Path ) -> None: + # Arrange generated = tmp_path / "generated.json" generated.write_text(json.dumps(self._dashboard()), encoding="utf-8") + + # Act & Assert assert validate_dashboard.load_dashboard(str(generated)) == self._dashboard() - def test_a_relative_path_resolves_against_the_current_directory( + def test_given_a_relative_path_when_load_dashboard_runs_then_it_resolves_against_the_cwd( self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange (tmp_path / "generated.json").write_text(json.dumps(self._dashboard()), encoding="utf-8") monkeypatch.chdir(tmp_path) + + # Act & Assert assert validate_dashboard.load_dashboard("generated.json") == self._dashboard() - def test_the_bundled_dashboard_is_the_default(self) -> None: + def test_given_no_dashboard_argument_when_load_dashboard_runs_then_the_bundled_one_is_loaded( + self, + ) -> None: + # Act & Assert assert validate_dashboard.load_dashboard(None)["panels"] - def test_a_missing_file_is_reported_as_a_missing_file(self, tmp_path: pathlib.Path) -> None: + def test_given_a_path_to_no_file_when_load_dashboard_runs_then_it_reports_a_missing_dashboard( + self, + tmp_path: pathlib.Path, + ) -> None: + # Act & Assert with pytest.raises(PolicyError, match="no dashboard at"): validate_dashboard.load_dashboard(str(tmp_path / "absent.json")) - def test_a_directory_is_reported_as_a_directory(self, tmp_path: pathlib.Path) -> None: + def test_given_a_directory_path_when_load_dashboard_runs_then_it_reports_a_directory( + self, + tmp_path: pathlib.Path, + ) -> None: + # Act & Assert with pytest.raises(PolicyError, match="not a dashboard file"): validate_dashboard.load_dashboard(str(tmp_path)) - def test_invalid_json_is_reported_as_invalid_json(self, tmp_path: pathlib.Path) -> None: + def test_given_a_file_of_invalid_json_when_load_dashboard_runs_then_it_reports_invalid_json( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange broken = tmp_path / "broken.json" broken.write_text("{not json", encoding="utf-8") + + # Act & Assert with pytest.raises(PolicyError, match="not valid JSON"): validate_dashboard.load_dashboard(str(broken)) @pytest.mark.parametrize("content", ["[]", '{"title": "no panels"}', '{"panels": {}}']) - def test_json_that_is_not_a_dashboard_is_reported_as_such( + def test_given_json_without_a_panels_array_when_load_dashboard_runs_then_it_reports_no_panels( self, tmp_path: pathlib.Path, content: str ) -> None: + # Arrange candidate = tmp_path / "not-a-dashboard.json" candidate.write_text(content, encoding="utf-8") + + # Act & Assert with pytest.raises(PolicyError, match="no panels array"): validate_dashboard.load_dashboard(str(candidate)) @@ -520,60 +732,90 @@ def _configure(self, monkeypatch: pytest.MonkeyPatch, user: str, password: str) monkeypatch.setenv("COPILOT_OTEL_GRAFANA_USER", user) monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", password) - def test_an_adopted_credential_passes_both_checks( + def test_given_only_the_configured_credential_works_when_grafana_is_checked_then_both_pass( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange self._configure(monkeypatch, "operator", "chosen-by-the-user") self._probe(monkeypatch, {("operator", "chosen-by-the-user"): True}) + + # Act verify.check_grafana_credentials() + + # Assert assert self._outcome("configured grafana credential works")[0] is True assert self._outcome("grafana default credential inactive")[0] is True - def test_a_live_default_credential_is_reported_as_such( + def test_given_admin_admin_works_when_grafana_is_checked_then_the_default_check_fails( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange self._configure(monkeypatch, "operator", "chosen-by-the-user") self._probe(monkeypatch, {("admin", "admin"): True}) + + # Act verify.check_grafana_credentials() + + # Assert ok, detail = self._outcome("grafana default credential inactive") assert ok is False assert "admin/admin authenticates" in detail - def test_a_rejected_configured_credential_is_reported_separately( + def test_given_admin_admin_works_when_grafana_is_checked_then_the_configured_check_fails( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange self._configure(monkeypatch, "operator", "chosen-by-the-user") self._probe(monkeypatch, {("admin", "admin"): True}) + + # Act verify.check_grafana_credentials() + + # Assert ok, detail = self._outcome("configured grafana credential works") assert ok is False assert "database predates this password" in detail - def test_an_unanswered_probe_is_not_reported_as_a_rejection( + def test_given_an_unanswered_probe_when_grafana_is_checked_then_no_answer_is_reported( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange self._configure(monkeypatch, "operator", "chosen-by-the-user") monkeypatch.setattr(verify, "grafana_accepts", lambda user, password: None) + + # Act verify.check_grafana_credentials() + + # Assert assert "no answer from Grafana" in self._outcome("configured grafana credential works")[1] - def test_missing_configuration_is_named_rather_than_probed( + def test_given_unset_grafana_variables_when_grafana_is_checked_then_they_are_named_not_probed( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange monkeypatch.delenv("COPILOT_OTEL_GRAFANA_USER", raising=False) monkeypatch.delenv("COPILOT_OTEL_GRAFANA_PASSWORD", raising=False) self._probe(monkeypatch, {}) + + # Act verify.check_grafana_credentials() + + # Assert ok, detail = self._outcome("configured grafana credential works") assert ok is False assert "are not both set" in detail - def test_configuring_the_default_pair_fails_the_default_check( + def test_given_the_default_pair_is_configured_when_grafana_is_checked_then_it_fails( self, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange self._configure(monkeypatch, "admin", "admin") self._probe(monkeypatch, {("admin", "admin"): True}) + + # Act verify.check_grafana_credentials() + + # Assert ok, detail = self._outcome("grafana default credential inactive") assert ok is False assert "is the image default" in detail @@ -588,20 +830,28 @@ class TestGrafanaCredentialScope: replay handed a third-party store an admin credential for a fourth. """ - def test_a_request_built_without_a_credential_carries_no_authorization(self) -> None: + def test_given_no_authorization_when_build_request_runs_then_the_header_is_absent(self) -> None: + # Act request = validate_dashboard.build_request("http://localhost:9090/api/v1/query") + + # Assert assert "Authorization" not in request.headers - def test_a_request_built_with_a_credential_carries_it(self) -> None: + def test_given_an_authorization_value_when_build_request_runs_then_the_header_carries_it( + self, + ) -> None: + # Act request = validate_dashboard.build_request( "http://localhost:3000/api/dashboards/db", b"{}", "POST", authorization="Basic supplied-at-the-call-site", ) + + # Assert assert request.headers["Authorization"] == "Basic supplied-at-the-call-site" - def test_only_the_grafana_request_carries_the_credential_end_to_end( + def test_given_a_full_dashboard_walk_when_the_tool_runs_then_only_grafana_is_authorized( self, monkeypatch: pytest.MonkeyPatch ) -> None: """Drive the real tool and inspect every request it actually builds. @@ -610,6 +860,7 @@ def test_only_the_grafana_request_carries_the_credential_end_to_end( representative one: the shipped dashboard is walked, every panel query is issued, and each resulting request object is inspected. """ + # Arrange sent: list[urllib.request.Request] = [] def fake_open_url(request, **kwargs): # noqa: ANN001, ANN202 @@ -621,8 +872,10 @@ def fake_open_url(request, **kwargs): # noqa: ANN001, ANN202 monkeypatch.setenv("COPILOT_OTEL_GRAFANA_PASSWORD", "chosen-by-the-user") monkeypatch.delenv("COPILOT_OTEL_ALLOW_REMOTE", raising=False) + # Act assert validate_dashboard.main([]) == 0 + # Assert authorized = [r for r in sent if "Authorization" in r.headers] assert len(sent) > 1, "the dashboard walk issued no store queries" assert len(authorized) == 1, ( @@ -657,55 +910,97 @@ def _redirect( ) @pytest.mark.parametrize("header", ["Authorization", "Proxy-Authorization", "Cookie"]) - def test_a_redirect_to_another_local_port_drops_each_credential_header( + def test_given_a_credential_header_when_a_redirect_changes_the_local_port_then_it_is_dropped( self, header: str ) -> None: + # Act redirected = self._redirect("http://localhost:3000/api", "http://localhost:9090/api") + + # Assert assert not any(name.lower() == header.lower() for name in redirected.headers) - def test_a_same_origin_redirect_keeps_the_credential(self) -> None: + def test_given_a_credential_header_when_a_redirect_stays_on_the_same_origin_then_it_is_kept( + self, + ) -> None: + # Act redirected = self._redirect("http://localhost:3000/api", "http://localhost:3000/other") + + # Assert assert redirected.headers["Authorization"] == "Basic issued-for-the-original-origin" - def test_an_explicit_default_port_is_the_same_origin(self, resolves) -> None: + def test_given_a_redirect_adding_the_default_https_port_when_it_is_handled_then_it_is_kept( + self, + resolves, + ) -> None: """`https://h/x` and `https://h:443/y` are one origin, not two.""" + # Arrange resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) + + # Act redirected = self._redirect( "https://grafana.example.com/api", "https://grafana.example.com:443/other", allow_remote=True, ) + + # Assert assert redirected.headers["Authorization"] == "Basic issued-for-the-original-origin" - def test_a_different_remote_host_drops_the_credential(self, resolves) -> None: + def test_given_a_redirect_to_a_different_remote_host_when_it_is_handled_then_it_is_dropped( + self, + resolves, + ) -> None: + # Arrange resolves( { "grafana.example.com": [GLOBAL_ADDRESS], "someone-else.example.com": [GLOBAL_ADDRESS], } ) + + # Act redirected = self._redirect( "https://grafana.example.com/api", "https://someone-else.example.com/api", allow_remote=True, ) + + # Assert assert "Authorization" not in redirected.headers - def test_a_different_remote_port_drops_the_credential(self, resolves) -> None: + def test_given_a_redirect_to_a_different_remote_port_when_it_is_handled_then_it_is_dropped( + self, + resolves, + ) -> None: + # Arrange resolves({"grafana.example.com": [GLOBAL_ADDRESS]}) + + # Act redirected = self._redirect( "https://grafana.example.com/api", "https://grafana.example.com:9090/api", allow_remote=True, ) + + # Assert assert "Authorization" not in redirected.headers - def test_a_non_credential_header_survives_an_origin_change(self) -> None: + def test_given_an_accept_header_when_a_redirect_changes_the_origin_then_it_survives( + self, + ) -> None: + # Act redirected = self._redirect("http://localhost:3000/api", "http://localhost:9090/api") + + # Assert assert redirected.headers["Accept"] == "application/json" - def test_the_credential_headers_are_dropped_together(self) -> None: + def test_given_the_credential_headers_when_a_redirect_changes_the_origin_then_none_remain( + self, + ) -> None: + # Act redirected = self._redirect("http://localhost:3000/api", "http://localhost:9090/api") + + # Assert remaining = {name.lower() for name in redirected.headers} assert remaining.isdisjoint({"authorization", "proxy-authorization", "cookie"}) @@ -721,7 +1016,12 @@ class TestOriginNormalization: ("http://H/a", "http://h/b"), ], ) - def test_equivalent_urls_share_an_origin(self, one: str, other: str) -> None: + def test_given_urls_differing_only_by_default_port_or_case_when_origin_of_runs_then_they_match( + self, + one: str, + other: str, + ) -> None: + # Act & Assert assert origin_of(one) == origin_of(other) @pytest.mark.parametrize( @@ -732,7 +1032,12 @@ def test_equivalent_urls_share_an_origin(self, one: str, other: str) -> None: ("http://h:3000/a", "http://h:9090/a"), ], ) - def test_different_urls_have_different_origins(self, one: str, other: str) -> None: + def test_given_urls_differing_by_scheme_host_or_port_when_origin_of_runs_then_they_differ( + self, + one: str, + other: str, + ) -> None: + # Act & Assert assert origin_of(one) != origin_of(other) @@ -745,9 +1050,10 @@ class TestPolicyRoutedHelpers: """ @pytest.mark.parametrize("module", [verify, inspect_metrics], ids=lambda m: m.__name__) - def test_the_helper_reaches_http_through_the_policy_boundary( + def test_given_a_patched_open_url_when_the_helper_calls_api_then_it_goes_through_policy( self, module, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange opened: list[str] = [] def fake_open_url(url, **kwargs): # noqa: ANN001, ANN202 @@ -755,14 +1061,24 @@ def fake_open_url(url, **kwargs): # noqa: ANN001, ANN202 return _json_response({"data": []}) monkeypatch.setattr(module, "open_url", fake_open_url) + + # Act module.api(module.PROM, "/api/v1/label/__name__/values") + + # Assert assert opened == ["http://localhost:9090/api/v1/label/__name__/values"] @pytest.mark.parametrize( "name", ["verify.py", "inspect_metrics.py", "validate_dashboard.py", "baseline.py"] ) - def test_no_bundled_helper_opens_a_url_outside_the_policy_module(self, name: str) -> None: + def test_given_a_bundled_helper_source_when_it_is_scanned_then_no_direct_urlopen_appears( + self, + name: str, + ) -> None: + # Arrange source = (EXAMPLES_DIR / name).read_text(encoding="utf-8") + + # Act & Assert assert "urlopen(" not in source, ( f"{name} opens a URL directly; every helper must go through open_url " "so scheme, authority, port, redirect, and remote-opt-in rules apply" diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py index aaf23b9b6..8dc13f75a 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_local_config.py @@ -52,15 +52,21 @@ def collector() -> dict[str, Any]: class TestConfigurationLoading: """The shipped documents parse and expose the structures later tests read.""" - def test_compose_declares_services(self, compose: dict[str, Any]) -> None: + def test_given_the_committed_compose_when_it_is_parsed_then_services_is_a_non_empty_mapping( + self, + compose: dict[str, Any], + ) -> None: + # Act & Assert assert isinstance(compose.get("services"), dict) assert compose["services"], "compose declares no services" - def test_yaml_loader_rejects_a_non_mapping_document(self) -> None: + def test_given_a_yaml_list_document_when_it_is_loaded_then_config_error_is_raised(self) -> None: + # Act & Assert with pytest.raises(ConfigError): load_yaml_text("- just\n- a list\n") - def test_yaml_loader_rejects_malformed_yaml(self) -> None: + def test_given_malformed_yaml_text_when_it_is_loaded_then_config_error_is_raised(self) -> None: + # Act & Assert with pytest.raises(ConfigError): load_yaml_text("services: [unclosed\n") @@ -68,76 +74,135 @@ def test_yaml_loader_rejects_malformed_yaml(self) -> None: class TestRedactionSimulation: """The policy simulator is fail-closed for keys and masking for values.""" - def test_unknown_keys_are_dropped(self) -> None: + def test_given_a_key_outside_the_allow_list_when_redaction_runs_then_it_is_dropped( + self, + ) -> None: + # Arrange allow = frozenset({"service.name"}) + + # Act result = simulate_redaction({"service.name": "copilot", "prompt.text": "secret"}, allow) + + # Assert assert result == {"service.name": "copilot"} - def test_allowed_values_matching_a_blocked_pattern_are_masked(self) -> None: + def test_given_an_allowed_key_when_its_value_matches_a_blocked_pattern_then_it_is_masked( + self, + ) -> None: + # Arrange allow = frozenset({"http.url"}) blocked = (re.compile(r"[0-9a-f]{40}"),) + + # Act result = simulate_redaction({"http.url": "a" * 40}, allow, blocked) + + # Assert assert result == {"http.url": MASK} - def test_an_empty_allow_list_drops_everything(self) -> None: + def test_given_an_empty_allow_list_when_redaction_runs_then_every_attribute_is_dropped( + self, + ) -> None: + # Act & Assert assert simulate_redaction({"any.key": "value"}, frozenset()) == {} class TestTopology: """The Collector is the only OTLP ingress and LGTM is a private backend.""" - def test_the_collector_publishes_otlp_on_loopback_only(self, compose: dict[str, Any]) -> None: + def test_given_the_collector_when_its_published_ports_are_read_then_otlp_binds_loopback_only( + self, + compose: dict[str, Any], + ) -> None: + # Act ports = published_ports(compose["services"]["otel-collector"]) + + # Assert assert "127.0.0.1:4317:4317" in ports assert "127.0.0.1:4318:4318" in ports assert all(entry.startswith("127.0.0.1:") for entry in ports) - def test_lgtm_publishes_no_otlp_port_to_the_host(self, compose: dict[str, Any]) -> None: + def test_given_the_lgtm_service_when_its_ports_are_read_then_no_otlp_port_is_exposed( + self, + compose: dict[str, Any], + ) -> None: + # Act ports = published_ports(compose["services"]["lgtm"]) + + # Assert assert not [entry for entry in ports if ":4317:" in entry or ":4318:" in entry] - def test_every_lgtm_host_mapping_is_loopback(self, compose: dict[str, Any]) -> None: + def test_given_the_lgtm_service_when_its_host_mappings_are_read_then_each_binds_loopback( + self, + compose: dict[str, Any], + ) -> None: + # Act & Assert assert all( entry.startswith("127.0.0.1:") for entry in published_ports(compose["services"]["lgtm"]) ) - def test_both_services_share_one_declared_network(self, compose: dict[str, Any]) -> None: + def test_given_declared_compose_networks_when_services_are_checked_then_both_join_them( + self, + compose: dict[str, Any], + ) -> None: + # Arrange declared = set(compose.get("networks") or {}) + + # Act & Assert assert declared, "compose declares no explicit network" for name in ("otel-collector", "lgtm"): assert declared.issuperset(compose["services"][name]["networks"]) - def test_the_collector_exports_to_the_lgtm_service_only( + def test_given_the_collector_exporters_when_endpoints_are_read_then_only_lgtm_4317_is_used( self, collector: dict[str, Any] ) -> None: + # Arrange exporters = collector["exporters"] + + # Act endpoints = { config.get("endpoint") for config in exporters.values() if isinstance(config, dict) } + + # Assert assert endpoints == {"lgtm:4317"} class TestRedactionPolicy: """The declared policy is fail-closed and reaches every pipeline.""" - def test_the_policy_is_fail_closed(self, collector: dict[str, Any]) -> None: + def test_given_the_redaction_processor_when_its_policy_is_read_then_allow_all_keys_is_false( + self, + collector: dict[str, Any], + ) -> None: + # Act & Assert assert redaction_policy(collector).get("allow_all_keys") is False - def test_every_pipeline_applies_redaction(self, collector: dict[str, Any]) -> None: + def test_given_the_collector_pipelines_when_each_is_read_then_all_three_include_redaction( + self, + collector: dict[str, Any], + ) -> None: + # Arrange pipelines = collector["service"]["pipelines"] + + # Act & Assert assert set(pipelines) == {"traces", "metrics", "logs"} for name, pipeline in pipelines.items(): assert "redaction" in pipeline["processors"], f"{name} pipeline skips redaction" - def test_the_allow_list_covers_what_the_dashboards_read( + def test_given_shipped_consumers_when_the_allow_list_is_compared_then_no_read_key_is_dropped( self, collector: dict[str, Any] ) -> None: + # Arrange consumers = shipped_consumers() allow = allowed_keys(collector) + + # Act missing = sorted(consumers.protected_attribute_keys() - allow) + + # Assert assert missing == [], f"the dashboard reads attributes the allow-list drops: {missing}" - def test_every_prometheus_label_the_dashboard_reads_has_a_source_key( + def test_given_dashboard_labels_when_allow_list_keys_are_normalized_then_none_is_uncovered( self, collector: dict[str, Any] ) -> None: """Compare in the OTLP-to-Prometheus direction, never the reverse. @@ -147,39 +212,62 @@ def test_every_prometheus_label_the_dashboard_reads_has_a_source_key( at least two plausible sources, so this asserts that some allow-listed key normalizes to each derived label rather than reconstructing one. """ + # Arrange consumers = shipped_consumers() covered = {prometheus_label_for(key) for key in allowed_keys(collector)} + + # Act uncovered = sorted(consumers.prometheus_labels - covered) + + # Assert assert uncovered == [], ( f"the dashboard groups by labels no allow-listed key produces: {uncovered}. " "Add the source attribute or record the label as a known-empty gap; " "do not remove it from the derivation." ) - def test_the_derivation_finds_the_metric_names_the_dashboard_selects(self) -> None: + def test_given_shipped_consumers_when_metric_names_are_derived_then_token_usage_is_included( + self, + ) -> None: """The derivation has to see metric names, not only attributes. Without this, a scrub rule could be licensed against metric names on the grounds that no consumer was named for them. """ + # Act consumers = shipped_consumers() + + # Assert assert len(consumers.metric_names) > 15 assert "gen_ai_client_token_usage_sum" in consumers.metric_names - def test_the_derivation_finds_the_span_name_matchers(self) -> None: + def test_given_shipped_consumers_when_span_matchers_are_derived_then_all_begin_invoke_agent( + self, + ) -> None: + # Act consumers = shipped_consumers() + + # Assert assert consumers.span_name_matchers, "no TraceQL span-name matcher was derived" assert all(matcher.startswith("invoke_agent") for matcher in consumers.span_name_matchers) - def test_the_derivation_finds_the_baseline_trace_field(self) -> None: + def test_given_shipped_consumers_when_trace_fields_are_derived_then_root_trace_name_appears( + self, + ) -> None: + # Act & Assert assert "rootTraceName" in shipped_consumers().trace_fields - def test_observed_content_attributes_are_not_allowed(self, collector: dict[str, Any]) -> None: + def test_given_the_observed_content_attributes_when_the_allow_list_is_read_then_none_appear( + self, + collector: dict[str, Any], + ) -> None: + # Act & Assert assert not (OBSERVED_CONTENT_ATTRIBUTES & allowed_keys(collector)) - def test_a_novel_attribute_is_dropped_rather_than_stored( + def test_given_unknown_future_attributes_when_redaction_runs_then_only_allowed_keys_survive( self, collector: dict[str, Any] ) -> None: + # Arrange allow = allowed_keys(collector) emitted = { "service.name": "copilot-chat", @@ -188,41 +276,64 @@ def test_a_novel_attribute_is_dropped_rather_than_stored( "gen_ai.future.prompt_echo": "the user's entire prompt", "copilot_chat.unreleased_content": "a tool result", } + + # Act kept = simulate_redaction(emitted, allow, blocked_values(collector)) + + # Assert assert set(kept) == {"service.name", "gen_ai.request.model"} - def test_known_content_attributes_are_dropped(self, collector: dict[str, Any]) -> None: + def test_given_known_content_attributes_when_redaction_runs_then_nothing_is_kept( + self, + collector: dict[str, Any], + ) -> None: + # Arrange emitted = dict.fromkeys(OBSERVED_CONTENT_ATTRIBUTES, "plaintext content") + + # Act & Assert assert simulate_redaction(emitted, allowed_keys(collector), blocked_values(collector)) == {} - def test_a_credential_in_an_allowed_value_is_masked(self, collector: dict[str, Any]) -> None: + def test_given_an_allowed_key_holding_a_github_token_when_redaction_runs_then_it_is_masked( + self, + collector: dict[str, Any], + ) -> None: + # Act kept = simulate_redaction( {"gen_ai.request.model": "ghp_" + "a" * 36}, allowed_keys(collector), blocked_values(collector), ) + + # Assert assert kept == {"gen_ai.request.model": MASK} - def test_the_processor_does_not_annotate_telemetry_with_removed_keys( + def test_given_the_redaction_processor_when_its_summary_is_read_then_it_is_silent( self, collector: dict[str, Any] ) -> None: + # Act & Assert assert redaction_policy(collector).get("summary") == "silent" class TestGrafanaAccess: """Grafana requires a credential the operator supplies.""" - def test_anonymous_access_is_disabled_and_the_login_form_is_restored( + def test_given_lgtm_auth_settings_when_they_are_read_then_anonymous_is_off_and_login_form_on( self, compose: dict[str, Any] ) -> None: + # Arrange environment = compose["services"]["lgtm"]["environment"] + + # Act & Assert assert str(environment["GF_AUTH_ANONYMOUS_ENABLED"]).lower() == "false" assert str(environment["GF_AUTH_DISABLE_LOGIN_FORM"]).lower() == "false" - def test_admin_credentials_are_required_environment_variables( + def test_given_lgtm_admin_credentials_when_compose_is_read_then_each_is_a_required_variable( self, compose: dict[str, Any] ) -> None: + # Arrange environment = compose["services"]["lgtm"]["environment"] + + # Act & Assert for key, variable in ( ("GF_SECURITY_ADMIN_USER", "COPILOT_OTEL_GRAFANA_USER"), ("GF_SECURITY_ADMIN_PASSWORD", "COPILOT_OTEL_GRAFANA_PASSWORD"), @@ -230,13 +341,23 @@ def test_admin_credentials_are_required_environment_variables( value = str(environment[key]) assert value.startswith(f"${{{variable}:?"), f"{key} is not a required variable" - def test_no_password_literal_is_committed(self) -> None: + def test_given_the_compose_file_text_when_it_is_scanned_then_no_default_password_is_present( + self, + ) -> None: + # Arrange text = COMPOSE_PATH.read_text(encoding="utf-8") + + # Act & Assert assert "GF_SECURITY_ADMIN_PASSWORD: admin" not in text assert "admin:admin" not in text - def test_the_dashboard_helper_reads_the_same_credential_pair(self) -> None: + def test_given_the_dashboard_helper_when_its_source_is_read_then_both_variables_lack_defaults( + self, + ) -> None: + # Arrange text = (EXAMPLES_DIR / "validate_dashboard.py").read_text(encoding="utf-8") + + # Act & Assert for variable in ("COPILOT_OTEL_GRAFANA_USER", "COPILOT_OTEL_GRAFANA_PASSWORD"): assert variable in text, f"{variable} is not read by the helper" # A default would make the helper usable against a credential the @@ -255,16 +376,32 @@ class TestContainerBounds: developer's machine, in front of an ingress any local process can reach. """ - def test_the_collector_declares_a_cgroup_memory_limit(self, compose: dict[str, Any]) -> None: + def test_given_the_collector_service_when_compose_is_read_then_a_memory_limit_is_declared( + self, + compose: dict[str, Any], + ) -> None: + # Act & Assert assert compose["services"]["otel-collector"]["mem_limit"] == "512m" - def test_the_collector_drops_every_capability(self, compose: dict[str, Any]) -> None: + def test_given_the_collector_service_when_compose_is_read_then_all_capabilities_are_dropped( + self, + compose: dict[str, Any], + ) -> None: + # Act & Assert assert compose["services"]["otel-collector"]["cap_drop"] == ["ALL"] - def test_the_collector_cannot_gain_privileges(self, compose: dict[str, Any]) -> None: + def test_given_the_collector_service_when_compose_is_read_then_no_new_privileges_is_set( + self, + compose: dict[str, Any], + ) -> None: + # Act & Assert assert "no-new-privileges:true" in compose["services"]["otel-collector"]["security_opt"] - def test_the_collector_root_filesystem_is_read_only(self, compose: dict[str, Any]) -> None: + def test_given_the_collector_service_when_compose_is_read_then_read_only_root_is_declared( + self, + compose: dict[str, Any], + ) -> None: + # Act & Assert assert compose["services"]["otel-collector"]["read_only"] is True @@ -292,32 +429,42 @@ class TestCredentialShapeCoverage: "google-api-key", ], ) - def test_a_recognizable_credential_shape_is_masked( + def test_given_credential_shaped_values_when_redaction_runs_then_each_shape_is_masked( self, collector: dict[str, Any], value: str ) -> None: + # Act kept = simulate_redaction( {"gen_ai.request.model": value}, allowed_keys(collector), blocked_values(collector), ) + + # Assert assert kept == {"gen_ai.request.model": MASK} - def test_an_ordinary_value_is_not_masked(self, collector: dict[str, Any]) -> None: + def test_given_an_ordinary_model_value_when_redaction_runs_then_it_is_left_unmasked( + self, + collector: dict[str, Any], + ) -> None: """Without this, a pattern broad enough to mask everything would pass.""" + # Act kept = simulate_redaction( {"gen_ai.request.model": "gpt-5"}, allowed_keys(collector), blocked_values(collector), ) + + # Assert assert kept == {"gen_ai.request.model": "gpt-5"} class TestScrubFailureDirection: """The scrub is fail-open per statement, and that is recorded where it lives.""" - def test_the_scrub_error_mode_is_documented_as_deliberate( + def test_given_the_scrub_processor_when_its_error_mode_is_read_then_fail_open_is_documented( self, collector: dict[str, Any] ) -> None: + # Act & Assert assert collector["processors"]["transform/scrub"]["error_mode"] == "ignore" text = COLLECTOR_PATH.read_text(encoding="utf-8") assert "fail-open per statement" in text, ( @@ -329,7 +476,12 @@ class TestImagePinning: """Both images are pinned by immutable digest.""" @pytest.mark.parametrize("service", ["otel-collector", "lgtm"]) - def test_the_image_is_digest_pinned(self, compose: dict[str, Any], service: str) -> None: + def test_given_each_compose_service_when_its_image_is_read_then_it_is_digest_pinned( + self, + compose: dict[str, Any], + service: str, + ) -> None: + # Act & Assert assert DIGEST_REFERENCE.match(compose["services"][service]["image"]) @@ -343,36 +495,69 @@ class TestScrubStaysInsideTheInventory: still contain their strings cannot fail. """ - def test_the_scrub_processor_declares_statements(self, collector: dict[str, Any]) -> None: + def test_given_the_collector_config_when_scrub_statements_are_read_then_they_are_not_empty( + self, + collector: dict[str, Any], + ) -> None: + # Act & Assert assert scrub_statements(collector), "the content scrub declares no statements" - def test_no_scrub_statement_targets_a_shipped_consumer(self, collector: dict[str, Any]) -> None: + def test_given_the_shipped_scrub_statements_when_they_are_checked_then_no_overreach_is_found( + self, + collector: dict[str, Any], + ) -> None: + # Act findings = overreaching_statements(scrub_statements(collector), shipped_consumers()) + + # Assert assert findings == [], f"a scrub rule targets telemetry a shipped panel reads: {findings}" - def test_the_check_catches_a_rule_that_normalizes_span_names(self) -> None: + def test_given_a_statement_that_rewrites_span_name_when_it_is_checked_then_it_is_reported( + self, + ) -> None: """The negative case. Without it this check could be vacuous.""" + # Arrange overreach = ['set(span.name, "[redacted]")'] + + # Act & Assert assert overreaching_statements(overreach, shipped_consumers()) == [ ('set(span.name, "[redacted]")', "span.name") ] - def test_the_check_catches_a_rule_that_deletes_a_read_attribute(self) -> None: + def test_given_a_statement_deleting_a_read_attribute_when_it_is_checked_then_it_is_reported( + self, + ) -> None: + # Arrange overreach = ['delete_key(span.attributes["gen_ai.request.model"], "x")'] + + # Act findings = overreaching_statements(overreach, shipped_consumers()) + + # Assert assert findings and findings[0][1] == "gen_ai.request.model" - def test_a_condition_that_reads_a_protected_field_is_not_an_overreach(self) -> None: + def test_given_a_where_clause_reading_a_protected_field_when_it_is_checked_then_it_is_allowed( + self, + ) -> None: """A `where` clause reads; it does not write.""" + # Arrange reading = ['set(span.status.message, "[redacted]") where span.name != ""'] + + # Act & Assert assert overreaching_statements(reading, shipped_consumers()) == [] class TestExporterAlias: """The exporter uses the alias the pinned Collector asks for.""" - def test_no_pipeline_references_the_deprecated_alias(self, collector: dict[str, Any]) -> None: + def test_given_the_collector_exporters_when_pipelines_are_read_then_none_uses_the_otlp_alias( + self, + collector: dict[str, Any], + ) -> None: + # Arrange exporters = set(collector["exporters"]) + + # Act & Assert assert not any(name == "otlp" or name.startswith("otlp/") for name in exporters), ( "the pinned Collector logs a deprecation warning once per signal for " "the `otlp` exporter alias; use `otlp_grpc`" @@ -380,7 +565,13 @@ def test_no_pipeline_references_the_deprecated_alias(self, collector: dict[str, for name, pipeline in collector["service"]["pipelines"].items(): assert set(pipeline["exporters"]) <= exporters, f"{name} names an undeclared exporter" - def test_the_internal_hop_is_unchanged_by_the_rename(self, collector: dict[str, Any]) -> None: + def test_given_the_renamed_otlp_grpc_exporter_when_it_is_read_then_the_lgtm_hop_is_unchanged( + self, + collector: dict[str, Any], + ) -> None: + # Arrange exporter = collector["exporters"]["otlp_grpc/lgtm"] + + # Act & Assert assert exporter["endpoint"] == "lgtm:4317" assert exporter["tls"]["insecure"] is True diff --git a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py index b58716e85..2c7db9618 100644 --- a/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py +++ b/.github/skills/experimental/copilot-otel-metrics/tests/test_settings_upsert.py @@ -62,11 +62,13 @@ def settings_file(tmp_path: pathlib.Path) -> pathlib.Path: class TestSchemaProvenance: """The schema records where it came from and matches the verified build.""" - def test_provenance_names_a_version_and_artifact(self) -> None: + def test_given_schema_source_when_provenance_fields_read_then_each_is_populated(self) -> None: + # Act & Assert for field in ("artifact", "extension", "version", "retrieved_from", "verified"): assert SCHEMA_SOURCE[field] - def test_the_schema_is_the_verified_seven_keys(self) -> None: + def test_given_the_schema_when_keys_counted_then_seven_otel_prefixed_keys_exist(self) -> None: + # Act & Assert assert len(SCHEMA) == 7 assert all(key.startswith("github.copilot.chat.otel.") for key in SCHEMA) @@ -74,41 +76,68 @@ def test_the_schema_is_the_verified_seven_keys(self) -> None: class TestAssignmentParsing: """Keys and types are checked before anything else happens.""" - def test_an_unknown_key_is_refused(self) -> None: + def test_given_an_unknown_setting_key_when_parsing_assignment_then_unknown_key_is_raised( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="unknown key"): parse_assignment("github.copilot.chat.otel.protocol=grpc") - def test_a_key_without_a_value_is_refused(self) -> None: + def test_given_an_assignment_without_an_equals_when_parsing_then_key_value_is_raised( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="key=value"): parse_assignment(ENABLED) - def test_a_boolean_key_refuses_a_non_boolean(self) -> None: + def test_given_a_boolean_key_when_parsing_a_non_boolean_value_then_boolean_error_is_raised( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="boolean"): parse_assignment(f"{ENABLED}=yes") - def test_an_integer_key_refuses_a_non_integer(self) -> None: + def test_given_an_integer_key_when_parsing_a_non_integer_value_then_integer_error_is_raised( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="integer"): parse_assignment(f"{MAXSIZE}=lots") @pytest.mark.parametrize(("raw", "expected"), [("true", True), ("false", False)]) - def test_booleans_are_coerced(self, raw: str, expected: bool) -> None: + def test_given_a_boolean_key_when_parsing_true_or_false_then_a_python_bool_is_returned( + self, + raw: str, + expected: bool, + ) -> None: + # Act & Assert assert parse_assignment(f"{ENABLED}={raw}") == (ENABLED, expected) - def test_integers_are_coerced(self) -> None: + def test_given_an_integer_key_when_parsing_a_numeric_value_then_an_int_is_returned( + self, + ) -> None: + # Act & Assert assert parse_assignment(f"{MAXSIZE}=2048") == (MAXSIZE, 2048) class TestPolicy: """Value combinations the schema cannot express are refused.""" - def test_enabling_content_capture_is_refused(self) -> None: + def test_given_capture_content_true_when_checking_policy_then_capture_content_is_refused( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="captureContent"): check_policy({CAPTURE: True}) - def test_disabling_content_capture_is_allowed(self) -> None: + def test_given_capture_content_false_when_checking_policy_then_no_error_is_raised(self) -> None: + # Act & Assert check_policy({CAPTURE: False}) - def test_an_unknown_exporter_type_is_refused(self) -> None: + def test_given_an_unknown_exporter_type_when_checking_policy_then_exporter_type_is_refused( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="exporterType"): check_policy({EXPORTER: "otlp-quic"}) @@ -121,22 +150,42 @@ def test_an_unknown_exporter_type_is_refused(self) -> None: "http://localhost:22", ], ) - def test_an_unsafe_endpoint_is_refused(self, endpoint: str) -> None: + def test_given_an_unsafe_endpoint_when_checking_policy_then_otlp_endpoint_is_refused( + self, + endpoint: str, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="otlpEndpoint"): check_policy({ENDPOINT: endpoint}) - def test_the_local_collector_endpoint_is_allowed(self) -> None: + def test_given_the_localhost_collector_endpoint_when_checking_policy_then_it_is_allowed( + self, + ) -> None: + # Act & Assert check_policy({ENDPOINT: "http://localhost:4318"}) - def test_outfile_with_an_otlp_exporter_is_refused(self, tmp_path: pathlib.Path) -> None: + def test_given_an_outfile_and_otlp_exporter_when_checking_policy_then_outfile_is_refused( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange outfile = str(tmp_path / "spans.jsonl") + + # Act & Assert with pytest.raises(SettingsError, match="outfile"): check_policy({OUTFILE: outfile, EXPORTER: "otlp-http"}) - def test_outfile_with_the_file_exporter_is_allowed(self, tmp_path: pathlib.Path) -> None: + def test_given_an_outfile_and_file_exporter_when_checking_policy_then_it_is_allowed( + self, + tmp_path: pathlib.Path, + ) -> None: + # Act & Assert check_policy({OUTFILE: str(tmp_path / "spans.jsonl"), EXPORTER: "file"}) - def test_a_negative_truncation_limit_is_refused(self) -> None: + def test_given_a_negative_max_attribute_size_when_checking_policy_then_negative_is_refused( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="negative"): check_policy({MAXSIZE: -1}) @@ -149,48 +198,64 @@ class TestMergedStatePolicy: passes while the resulting file says both. """ - def test_an_exporter_that_contradicts_an_existing_outfile_is_refused( + def test_given_an_existing_outfile_when_checking_a_new_otlp_exporter_then_outfile_is_refused( self, tmp_path: pathlib.Path ) -> None: + # Arrange existing = {OUTFILE: str(tmp_path / "spans.jsonl")} + + # Act & Assert with pytest.raises(SettingsError, match="outfile"): check_policy({EXPORTER: "otlp-http"}, existing=existing) - def test_an_outfile_that_contradicts_an_existing_exporter_is_refused( + def test_given_an_existing_otlp_exporter_when_checking_a_new_outfile_then_outfile_is_refused( self, tmp_path: pathlib.Path ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="outfile"): check_policy({OUTFILE: str(tmp_path / "spans.jsonl")}, existing={EXPORTER: "otlp-http"}) - def test_replacing_the_conflicting_setting_in_the_same_run_is_allowed( + def test_given_an_existing_otlp_exporter_when_outfile_and_file_exporter_are_set_then_allowed( self, tmp_path: pathlib.Path ) -> None: + # Act & Assert check_policy( {OUTFILE: str(tmp_path / "spans.jsonl"), EXPORTER: "file"}, existing={EXPORTER: "otlp-http"}, ) - def test_an_existing_unknown_exporter_is_reported(self) -> None: + def test_given_an_existing_unknown_exporter_when_checking_an_unrelated_change_then_refused( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="exporterType"): check_policy({ENABLED: True}, existing={EXPORTER: "otlp-quic"}) - def test_an_existing_off_policy_endpoint_is_refused(self) -> None: + def test_given_an_existing_off_policy_endpoint_when_setting_an_exporter_then_refused( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="already holds an otlpEndpoint"): check_policy( {EXPORTER: "otlp-http"}, existing={ENDPOINT: "http://evil.example.com:4318"} ) - def test_replacing_an_off_policy_endpoint_in_the_same_run_is_allowed(self) -> None: + def test_given_an_existing_off_policy_endpoint_when_a_local_endpoint_replaces_it_then_allowed( + self, + ) -> None: + # Act & Assert check_policy( {ENDPOINT: "http://localhost:4318"}, existing={ENDPOINT: "http://evil.example.com:4318"}, ) - def test_an_already_enabled_content_capture_is_refused(self) -> None: + def test_given_capture_content_already_enabled_when_enabling_otel_then_refused(self) -> None: + # Act & Assert with pytest.raises(SettingsError, match="already enabled"): check_policy({ENABLED: True}, existing={CAPTURE: True}) - def test_turning_an_enabled_content_capture_off_in_the_same_run_is_allowed(self) -> None: + def test_given_capture_content_already_enabled_when_it_is_turned_off_then_allowed(self) -> None: + # Act & Assert check_policy({CAPTURE: False}, existing={CAPTURE: True}) @@ -207,38 +272,54 @@ class TestSplitInvocationWrites: `write_atomically`, which is the only writer of the settings document. """ - def test_a_clean_edit_cannot_write_past_an_off_policy_endpoint_already_in_the_file( + def test_given_a_file_holding_an_off_policy_endpoint_when_applying_a_clean_edit_then_refused( self, settings_file: pathlib.Path ) -> None: + # Arrange settings_file.write_text( json.dumps({ENDPOINT: "http://evil.example.com:4318"}, indent=2), encoding="utf-8" ) + + # Act & Assert with pytest.raises(SettingsError, match="already holds an otlpEndpoint"): apply_changes(settings_file, {ENABLED: True}, apply=True) - def test_the_refused_write_leaves_the_document_byte_identical( + def test_given_a_refused_apply_when_the_document_is_reread_then_it_is_byte_identical( self, settings_file: pathlib.Path ) -> None: + # Arrange original = json.dumps({ENDPOINT: "http://evil.example.com:4318"}, indent=2) settings_file.write_text(original, encoding="utf-8") + + # Act with pytest.raises(SettingsError): apply_changes(settings_file, {ENABLED: True}, apply=True) + + # Assert assert settings_file.read_text(encoding="utf-8") == original - def test_a_clean_edit_cannot_write_past_content_capture_already_in_the_file( + def test_given_a_file_holding_enabled_capture_content_when_applying_a_clean_edit_then_refused( self, settings_file: pathlib.Path ) -> None: + # Arrange settings_file.write_text(json.dumps({CAPTURE: True}, indent=2), encoding="utf-8") + + # Act & Assert with pytest.raises(SettingsError, match="already enabled"): apply_changes(settings_file, {ENABLED: True}, apply=True) - def test_correcting_the_offending_value_in_the_same_run_is_allowed( + def test_given_a_stored_off_policy_endpoint_when_applying_a_local_endpoint_then_it_is_written( self, settings_file: pathlib.Path ) -> None: + # Arrange settings_file.write_text( json.dumps({ENDPOINT: "http://evil.example.com:4318"}, indent=2), encoding="utf-8" ) + + # Act apply_changes(settings_file, {ENDPOINT: "http://localhost:4318"}, apply=True) + + # Assert written = json.loads(settings_file.read_text(encoding="utf-8")) assert written[ENDPOINT] == "http://localhost:4318" @@ -246,31 +327,49 @@ def test_correcting_the_offending_value_in_the_same_run_is_allowed( class TestOutfilePolicy: """Captured spans may not land somewhere nobody chose.""" - def test_a_relative_outfile_is_refused(self) -> None: + def test_given_a_relative_outfile_path_when_checking_policy_then_absolute_is_required( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="absolute"): check_policy({OUTFILE: "spans.jsonl", EXPORTER: "file"}) - def test_an_outfile_inside_this_repository_is_refused(self) -> None: + def test_given_an_outfile_inside_this_repository_when_checking_policy_then_it_is_refused( + self, + ) -> None: + # Arrange inside = str(pathlib.Path(__file__).resolve().parent / "spans.jsonl") + + # Act & Assert with pytest.raises(SettingsError, match="inside this repository"): check_policy({OUTFILE: inside, EXPORTER: "file"}) - def test_a_foreign_platform_absolute_path_is_accepted(self) -> None: + def test_given_a_foreign_platform_absolute_outfile_when_checking_policy_then_it_is_accepted( + self, + ) -> None: """A settings file is routinely authored on one platform for another.""" + # Act & Assert check_policy({OUTFILE: "/var/log/copilot/spans.jsonl", EXPORTER: "file"}) class TestRemoteEndpointOptIn: """A remote endpoint is a deliberate choice, not a default.""" - def test_a_remote_endpoint_is_refused_by_default(self) -> None: + def test_given_a_remote_endpoint_when_checking_policy_without_the_opt_in_then_it_is_refused( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="otlpEndpoint"): check_policy({ENDPOINT: "https://93.184.216.34:4318"}) - def test_a_remote_endpoint_is_allowed_with_the_opt_in(self) -> None: + def test_given_a_remote_https_endpoint_when_the_opt_in_is_set_then_it_is_allowed(self) -> None: + # Act & Assert check_policy({ENDPOINT: "https://93.184.216.34:4318"}, allow_remote_endpoint=True) - def test_the_opt_in_does_not_permit_plaintext(self) -> None: + def test_given_a_plaintext_remote_endpoint_when_the_opt_in_is_set_then_it_is_still_refused( + self, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="otlpEndpoint"): check_policy({ENDPOINT: "http://93.184.216.34:4318"}, allow_remote_endpoint=True) @@ -278,47 +377,80 @@ def test_the_opt_in_does_not_permit_plaintext(self) -> None: class TestJsoncHandling: """Comments survive, offsets line up, and only the target value moves.""" - def test_comments_are_blanked_without_shifting_offsets(self) -> None: + def test_given_a_document_with_comments_when_stripping_jsonc_then_length_is_unchanged( + self, + ) -> None: + # Act stripped = strip_jsonc(EXISTING) + + # Assert assert len(stripped) == len(EXISTING) assert "//" not in stripped assert "block comment" not in stripped assert json.loads(stripped)["editor.fontSize"] == 13 - def test_a_comment_inside_a_string_is_preserved(self) -> None: + def test_given_a_url_inside_a_string_when_stripping_jsonc_then_the_string_survives( + self, + ) -> None: + # Arrange text = '{"a": "http://example.com/x"}' + + # Act & Assert assert json.loads(strip_jsonc(text))["a"] == "http://example.com/x" - def test_top_level_spans_find_each_key(self) -> None: + def test_given_a_jsonc_document_when_scanning_top_level_spans_then_each_key_is_located( + self, + ) -> None: + # Act spans = top_level_spans(EXISTING) + + # Assert assert set(spans) >= {"editor.fontSize", ENABLED, "workbench.colorTheme"} start, end = spans[ENABLED] assert EXISTING[start:end] == "false" - def test_upsert_replaces_only_the_target_value(self) -> None: + def test_given_a_commented_document_when_upserting_a_key_then_only_that_value_changes( + self, + ) -> None: + # Act updated = upsert(EXISTING, {ENABLED: True}) + + # Assert assert "// A comment the edit must not disturb." in updated assert "/* block comment */" in updated assert json.loads(strip_jsonc(updated))[ENABLED] is True assert json.loads(strip_jsonc(updated))["editor.fontSize"] == 13 - def test_upsert_appends_a_missing_key(self) -> None: + def test_given_a_document_without_the_key_when_upserting_then_the_key_is_appended(self) -> None: + # Act updated = upsert(EXISTING, {ENDPOINT: "http://localhost:4318"}) + + # Assert assert json.loads(strip_jsonc(updated))[ENDPOINT] == "http://localhost:4318" assert "// A comment the edit must not disturb." in updated - def test_upsert_handles_an_empty_object(self) -> None: + def test_given_an_empty_json_object_when_upserting_a_key_then_the_key_is_the_only_entry( + self, + ) -> None: + # Act updated = upsert("{}\n", {ENABLED: True}) + + # Assert assert json.loads(strip_jsonc(updated)) == {ENABLED: True} - def test_a_nested_key_of_the_same_name_is_not_targeted(self) -> None: + def test_given_a_language_scoped_override_when_upserting_then_only_the_top_level_key_changes( + self, + ) -> None: """A language-scoped override must not absorb the edit. The span scanner is hand-written, so this is the case most likely to break silently under a later change: the write would land in a `"[python]"` block and the user's real setting would stay unchanged. """ + # Arrange nested = '{\n "[python]": { "' + ENABLED + '": false },\n "' + ENABLED + '": false\n}\n' + + # Act & Assert assert set(top_level_spans(nested)) == {"[python]", ENABLED} parsed = json.loads(strip_jsonc(upsert(nested, {ENABLED: True}))) @@ -329,7 +461,11 @@ def test_a_nested_key_of_the_same_name_is_not_targeted(self) -> None: class TestCommandLine: """Argument handling refuses ambiguous input before touching a file.""" - def test_a_duplicate_key_is_refused(self, settings_file: pathlib.Path) -> None: + def test_given_two_set_flags_for_one_key_when_running_main_then_it_exits_refused( + self, + settings_file: pathlib.Path, + ) -> None: + # Act code = main( [ "--settings", @@ -341,10 +477,16 @@ def test_a_duplicate_key_is_refused(self, settings_file: pathlib.Path) -> None: "--apply", ] ) + + # Assert assert code == EXIT_REFUSED assert settings_file.read_text(encoding="utf-8") == EXISTING - def test_no_assignment_is_refused(self, settings_file: pathlib.Path) -> None: + def test_given_no_set_flags_when_running_main_then_it_exits_refused( + self, + settings_file: pathlib.Path, + ) -> None: + # Act & Assert assert main(["--settings", str(settings_file)]) == EXIT_REFUSED assert settings_file.read_text(encoding="utf-8") == EXISTING @@ -352,38 +494,63 @@ def test_no_assignment_is_refused(self, settings_file: pathlib.Path) -> None: class TestExitConventions: """The exit code distinguishes a refusal from a crash and from an interrupt.""" - def test_a_successful_dry_run_exits_zero(self, settings_file: pathlib.Path) -> None: + def test_given_a_valid_assignment_when_running_main_without_apply_then_it_exits_ok( + self, + settings_file: pathlib.Path, + ) -> None: + # Act & Assert assert main(["--settings", str(settings_file), "--set", f"{ENABLED}=true"]) == EXIT_OK - def test_a_refusal_exits_two(self, settings_file: pathlib.Path) -> None: + def test_given_a_policy_violating_assignment_when_running_main_then_it_exits_refused( + self, + settings_file: pathlib.Path, + ) -> None: + # Act code = main(["--settings", str(settings_file), "--set", f"{CAPTURE}=true", "--apply"]) + + # Assert assert code == EXIT_REFUSED - def test_an_interrupt_exits_one_hundred_thirty( + def test_given_a_keyboard_interrupt_during_apply_when_running_main_then_it_exits_interrupted( self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange def interrupt(*args: object, **kwargs: object) -> str: raise KeyboardInterrupt monkeypatch.setattr("settings_upsert.apply_changes", interrupt) + + # Act code = main(["--settings", str(settings_file), "--set", f"{ENABLED}=true"]) + + # Assert assert code == EXIT_INTERRUPTED - def test_the_entry_point_passes_the_code_to_sys_exit(self) -> None: + def test_given_the_example_module_source_when_inspecting_the_entry_point_then_sys_exit_main( + self, + ) -> None: """`raise SystemExit(main())` and `sys.exit(main())` differ to a reader.""" + # Act source = ( pathlib.Path(__file__).resolve().parents[1] / "examples" / "settings_upsert.py" ).read_text(encoding="utf-8") + + # Assert assert "sys.exit(main())" in source - def test_diagnostics_use_a_module_logger(self) -> None: + def test_given_the_example_module_source_when_inspecting_diagnostics_then_a_module_logger( + self, + ) -> None: + # Act source = ( pathlib.Path(__file__).resolve().parents[1] / "examples" / "settings_upsert.py" ).read_text(encoding="utf-8") + + # Assert assert 'LOGGER = logging.getLogger("settings_upsert")' in source assert "def configure_logging(" in source - def test_a_closed_reader_is_not_an_error( + def test_given_print_raising_broken_pipe_when_running_main_then_it_is_silenced_and_exits_ok( self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: """A quit pager is the reader leaving, not the edit failing. @@ -392,6 +559,7 @@ def test_a_closed_reader_is_not_an_error( would take the test runner's capture with it, so the handler's effect is recorded rather than performed. """ + # Arrange silenced: list[str] = [] def broken(*args: object, **kwargs: object) -> None: @@ -401,6 +569,8 @@ def broken(*args: object, **kwargs: object) -> None: monkeypatch.setattr( "settings_upsert.silence_broken_pipe", lambda: silenced.append("silenced") ) + + # Act & Assert assert main(["--settings", str(settings_file), "--set", f"{ENABLED}=true"]) == EXIT_OK assert silenced == ["silenced"] @@ -408,92 +578,173 @@ def broken(*args: object, **kwargs: object) -> None: class TestApply: """Dry runs write nothing; applied edits are backed up and reversible.""" - def test_a_dry_run_leaves_the_file_untouched(self, settings_file: pathlib.Path) -> None: + def test_given_a_settings_file_when_applying_changes_with_apply_false_then_it_is_unchanged( + self, + settings_file: pathlib.Path, + ) -> None: + # Act apply_changes(settings_file, {ENABLED: True}, apply=False) + + # Assert assert settings_file.read_text(encoding="utf-8") == EXISTING - def test_an_applied_edit_writes_and_backs_up(self, settings_file: pathlib.Path) -> None: + def test_given_a_settings_file_when_applying_a_change_then_it_is_written_and_backed_up( + self, + settings_file: pathlib.Path, + ) -> None: + # Act apply_changes(settings_file, {ENABLED: True}, apply=True) + + # Assert assert json.loads(strip_jsonc(settings_file.read_text(encoding="utf-8")))[ENABLED] is True backups = list(settings_file.parent.glob(f"{settings_file.name}.*.bak")) assert len(backups) == 1 assert backups[0].read_text(encoding="utf-8") == EXISTING - def test_unrelated_settings_survive(self, settings_file: pathlib.Path) -> None: + def test_given_other_settings_in_the_file_when_applying_a_change_then_they_are_preserved( + self, + settings_file: pathlib.Path, + ) -> None: + # Act apply_changes(settings_file, {ENABLED: True}, apply=True) + + # Assert parsed = json.loads(strip_jsonc(settings_file.read_text(encoding="utf-8"))) assert parsed["editor.fontSize"] == 13 assert parsed["workbench.colorTheme"] == "Default Dark+" - def test_a_refused_edit_writes_nothing(self, settings_file: pathlib.Path) -> None: + def test_given_a_policy_violating_change_when_applying_then_no_write_or_backup_occurs( + self, + settings_file: pathlib.Path, + ) -> None: + # Act with pytest.raises(SettingsError): apply_changes(settings_file, {CAPTURE: True}, apply=True) + + # Assert assert settings_file.read_text(encoding="utf-8") == EXISTING assert list(settings_file.parent.glob(f"{settings_file.name}.*.bak")) == [] - def test_a_malformed_existing_file_is_refused(self, tmp_path: pathlib.Path) -> None: + def test_given_a_malformed_jsonc_file_when_applying_a_change_then_not_valid_jsonc_is_raised( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange broken = tmp_path / "settings.json" broken.write_text('{"a": [1, 2', encoding="utf-8") + + # Act with pytest.raises(SettingsError, match="not valid JSONC"): apply_changes(broken, {ENABLED: True}, apply=True) + + # Assert assert broken.read_text(encoding="utf-8") == '{"a": [1, 2' - def test_a_missing_file_starts_from_an_empty_object(self, tmp_path: pathlib.Path) -> None: + def test_given_a_missing_settings_file_when_applying_a_change_then_only_that_key_is_written( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange target = tmp_path / "settings.json" + + # Act apply_changes(target, {ENABLED: True}, apply=True) + + # Assert assert json.loads(strip_jsonc(target.read_text(encoding="utf-8"))) == {ENABLED: True} class TestAudit: """Audit evidence is durable and carries no sensitive value verbatim.""" - def test_an_audit_record_is_appended(self, settings_file: pathlib.Path) -> None: + def test_given_an_audit_path_when_applying_a_change_then_a_record_with_the_diff_is_appended( + self, + settings_file: pathlib.Path, + ) -> None: + # Arrange audit = settings_file.parent / "audit.jsonl" + + # Act apply_changes(settings_file, {ENABLED: True}, apply=True, audit_path=audit) + + # Assert record = json.loads(audit.read_text(encoding="utf-8").strip()) assert record["changes"][ENABLED] == {"from": False, "to": True} assert record["schema_version"] == SCHEMA_SOURCE["version"] - def test_audit_records_accumulate(self, settings_file: pathlib.Path) -> None: + def test_given_an_audit_path_when_applying_two_changes_then_two_records_are_written( + self, + settings_file: pathlib.Path, + ) -> None: + # Arrange audit = settings_file.parent / "audit.jsonl" + + # Act apply_changes(settings_file, {ENABLED: True}, apply=True, audit_path=audit) apply_changes(settings_file, {ENABLED: False}, apply=True, audit_path=audit) + + # Assert assert len(audit.read_text(encoding="utf-8").strip().splitlines()) == 2 - def test_a_sensitive_value_is_summarized_not_copied(self) -> None: + def test_given_an_outfile_path_when_summarizing_then_only_a_length_is_recorded(self) -> None: + # Act summarized = summarize(OUTFILE, "/home/someone/private/spans.jsonl") + + # Assert assert "private" not in str(summarized) assert "length" in str(summarized) - def test_an_ordinary_value_is_recorded_directly(self) -> None: + def test_given_an_ordinary_boolean_when_summarizing_then_the_value_is_returned_unchanged( + self, + ) -> None: + # Act & Assert assert summarize(ENABLED, True) is True - def test_an_endpoint_keeps_its_host_and_loses_everything_else(self) -> None: + def test_given_an_endpoint_with_a_path_and_query_when_summarizing_then_only_the_origin_remains( + self, + ) -> None: """A path or query on an OTLP endpoint is operator-supplied. It is where a tenant identifier or a token would sit, and it is not needed to answer the question the record is kept for. """ + # Act summarized = summarize(ENDPOINT, "https://ingest.example.com:4318/v1/traces?token=abc123") + + # Assert assert summarized == "https://ingest.example.com:4318" assert "token" not in str(summarized) assert "v1/traces" not in str(summarized) - def test_endpoint_userinfo_is_not_copied_into_the_record(self) -> None: + def test_given_an_endpoint_with_userinfo_when_summarizing_then_the_credential_is_dropped( + self, + ) -> None: """The netloc carries the credential; the host and port do not.""" + # Act summarized = summarize(ENDPOINT, "https://someone:s3cret@ingest.example.com:4318/v1") + + # Assert assert summarized == "https://ingest.example.com:4318" assert "s3cret" not in str(summarized) assert "someone" not in str(summarized) - def test_a_loopback_endpoint_is_still_recognizable(self) -> None: + def test_given_a_loopback_endpoint_with_a_path_when_summarizing_then_the_origin_remains( + self, + ) -> None: + # Act & Assert assert summarize(ENDPOINT, "http://localhost:4318/v1/traces") == "http://localhost:4318" - def test_an_unparseable_endpoint_is_reduced_rather_than_copied(self) -> None: + def test_given_an_unparseable_endpoint_when_summarizing_then_the_secret_path_is_not_kept( + self, + ) -> None: + # Act summarized = summarize(ENDPOINT, "not-a-url-at-all/secret-path") + + # Assert assert "secret-path" not in str(summarized) - def test_an_absent_value_records_as_absent(self) -> None: + def test_given_an_absent_endpoint_when_summarizing_then_none_is_returned(self) -> None: + # Act & Assert assert summarize(ENDPOINT, None) is None @@ -506,36 +757,71 @@ class TestDocumentShapes: """ @pytest.mark.parametrize("content", ["", " \n\n", "// nothing set yet\n"]) - def test_an_effectively_empty_document_is_editable( + def test_given_an_effectively_empty_document_when_applying_a_change_then_the_key_is_written( self, tmp_path: pathlib.Path, content: str ) -> None: + # Arrange target = tmp_path / "settings.json" target.write_text(content, encoding="utf-8") + + # Act apply_changes(target, {ENABLED: True}, apply=True) + + # Assert assert json.loads(strip_jsonc(target.read_text(encoding="utf-8-sig")))[ENABLED] is True - def test_a_comments_only_document_keeps_its_comments(self, tmp_path: pathlib.Path) -> None: + def test_given_a_comments_only_document_when_applying_a_change_then_the_comment_survives( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange target = tmp_path / "settings.json" target.write_text("// a note the operator left\n", encoding="utf-8") + + # Act apply_changes(target, {ENABLED: True}, apply=True) + + # Assert assert "a note the operator left" in target.read_text(encoding="utf-8") - def test_a_byte_order_mark_is_preserved(self, tmp_path: pathlib.Path) -> None: + def test_given_a_document_with_a_byte_order_mark_when_applying_a_change_then_the_mark_remains( + self, + tmp_path: pathlib.Path, + ) -> None: """VS Code wrote it; removing it is an unrelated change to the file.""" + # Arrange target = tmp_path / "settings.json" target.write_bytes(b"\xef\xbb\xbf" + EXISTING.encode("utf-8")) + + # Act apply_changes(target, {ENABLED: True}, apply=True) + + # Assert assert target.read_bytes().startswith(b"\xef\xbb\xbf") assert json.loads(strip_jsonc(target.read_text(encoding="utf-8-sig")))[ENABLED] is True - def test_a_document_without_a_mark_does_not_gain_one(self, settings_file: pathlib.Path) -> None: + def test_given_a_document_without_a_byte_order_mark_when_applying_a_change_then_none_is_added( + self, + settings_file: pathlib.Path, + ) -> None: + # Act apply_changes(settings_file, {ENABLED: True}, apply=True) + + # Assert assert not settings_file.read_bytes().startswith(b"\xef\xbb\xbf") - def test_read_settings_reports_the_mark_separately(self, tmp_path: pathlib.Path) -> None: + def test_given_a_file_with_a_byte_order_mark_when_reading_settings_then_it_is_returned_apart( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange target = tmp_path / "settings.json" target.write_bytes(b"\xef\xbb\xbf{}\n") + + # Act bom, text = read_settings(target) + + # Assert assert bom == "\ufeff" assert not text.startswith("\ufeff") @@ -548,39 +834,54 @@ class TestAtomicWrite: the file is either the old one or the new one. """ - def test_a_failed_write_leaves_the_original_intact( + def test_given_os_replace_failing_when_applying_a_change_then_the_original_file_is_intact( self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange def fail(*args: object, **kwargs: object) -> None: raise OSError("disk full") monkeypatch.setattr("settings_upsert.os.replace", fail) + + # Act with pytest.raises(OSError, match="disk full"): apply_changes(settings_file, {ENABLED: True}, apply=True) + + # Assert assert settings_file.read_text(encoding="utf-8") == EXISTING - def test_a_failed_write_leaves_no_staged_file_behind( + def test_given_os_replace_failing_when_applying_a_change_then_no_staged_file_remains( self, settings_file: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange def fail(*args: object, **kwargs: object) -> None: raise OSError("disk full") monkeypatch.setattr("settings_upsert.os.replace", fail) + + # Act with pytest.raises(OSError): apply_changes(settings_file, {ENABLED: True}, apply=True) + + # Assert assert list(settings_file.parent.glob("*.staged")) == [] - def test_a_new_settings_file_is_not_created_by_a_failed_write( + def test_given_os_replace_failing_when_applying_to_a_missing_file_then_none_is_created( self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange target = tmp_path / "settings.json" def fail(*args: object, **kwargs: object) -> None: raise OSError("disk full") monkeypatch.setattr("settings_upsert.os.replace", fail) + + # Act with pytest.raises(OSError): apply_changes(target, {ENABLED: True}, apply=True) + + # Assert assert not target.exists() @@ -592,35 +893,49 @@ class TestAuditPathAliases: appended JSON line destroys the settings file or its backup. """ - def test_the_settings_file_itself_is_refused(self, settings_file: pathlib.Path) -> None: + def test_given_the_settings_path_as_the_audit_path_when_resolving_then_it_is_refused( + self, + settings_file: pathlib.Path, + ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="settings file itself"): resolve_audit_path(str(settings_file), settings_file) - def test_a_backup_beside_the_settings_file_is_refused( + def test_given_a_backup_path_as_the_audit_path_when_resolving_then_it_is_refused( self, settings_file: pathlib.Path ) -> None: + # Arrange backup = backup_path_for(settings_file) + + # Act & Assert with pytest.raises(SettingsError, match="settings backup"): resolve_audit_path(str(backup), settings_file) - def test_a_directory_is_refused( + def test_given_a_directory_as_the_audit_path_when_resolving_then_it_is_refused( self, settings_file: pathlib.Path, tmp_path: pathlib.Path ) -> None: + # Act & Assert with pytest.raises(SettingsError, match="directory"): resolve_audit_path(str(tmp_path), settings_file) - def test_an_operator_chosen_path_elsewhere_is_kept( + def test_given_an_audit_path_outside_the_settings_directory_when_resolving_then_it_is_kept( self, settings_file: pathlib.Path, tmp_path: pathlib.Path ) -> None: + # Arrange chosen = tmp_path.parent / "audit-elsewhere.jsonl" + + # Act & Assert assert resolve_audit_path(str(chosen), settings_file) == chosen.resolve() - def test_a_relative_path_resolves_against_the_caller( + def test_given_a_relative_audit_path_when_resolving_then_it_resolves_against_the_cwd( self, settings_file: pathlib.Path, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # Arrange elsewhere = tmp_path / "cwd" elsewhere.mkdir() monkeypatch.chdir(elsewhere) + + # Act & Assert assert ( resolve_audit_path("audit.jsonl", settings_file) == (elsewhere / "audit.jsonl").resolve() @@ -635,44 +950,62 @@ class TestBackupRetention: already-edited one, destroying the only record of the original. """ - def test_a_second_apply_does_not_overwrite_the_first_backup( + def test_given_two_applies_when_listing_backups_then_both_originals_are_retained( self, settings_file: pathlib.Path ) -> None: + # Act apply_changes(settings_file, {ENABLED: True}, apply=True) after_first = settings_file.read_text(encoding="utf-8") apply_changes(settings_file, {ENABLED: False}, apply=True) + # Assert backups = sorted(settings_file.parent.glob(f"{settings_file.name}.*.bak")) assert len(backups) == 2 contents = {path.read_text(encoding="utf-8") for path in backups} assert EXISTING in contents, "the operator's original file was not retained" assert after_first in contents - def test_a_same_second_collision_gets_its_own_name(self, tmp_path: pathlib.Path) -> None: + def test_given_an_existing_backup_name_when_building_another_then_a_distinct_name_is_returned( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange settings = tmp_path / "settings.json" settings.write_text("{}\n", encoding="utf-8") first = backup_path_for(settings) first.write_text("taken", encoding="utf-8") + + # Act second = backup_path_for(settings) + + # Assert assert second != first assert not second.exists() - def test_a_later_backup_sorts_after_an_earlier_one(self, tmp_path: pathlib.Path) -> None: + def test_given_three_backups_created_in_order_when_sorting_names_then_creation_order_is_kept( + self, + tmp_path: pathlib.Path, + ) -> None: """Retention reads name order to decide what to delete. An optional collision suffix put `...Z-1.bak` before `...Z.bak`, which would have made pruning remove the newest backups and keep the oldest. """ + # Arrange settings = tmp_path / "settings.json" settings.write_text("{}\n", encoding="utf-8") names = [] + + # Act for _ in range(3): candidate = backup_path_for(settings) candidate.write_text("taken", encoding="utf-8") names.append(candidate.name) + + # Assert assert names == sorted(names) - def test_a_pruned_index_is_not_reused_within_the_same_second( + def test_given_a_deleted_backup_in_one_second_when_building_another_then_the_index_advances( self, tmp_path: pathlib.Path ) -> None: """Retention frees a low index; reusing it would invert creation order. @@ -680,6 +1013,7 @@ def test_a_pruned_index_is_not_reused_within_the_same_second( Every name here shares one timestamp, which is what happens when several applies land inside the same second. """ + # Arrange settings = tmp_path / "settings.json" settings.write_text("{}\n", encoding="utf-8") stamp = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC) @@ -689,29 +1023,47 @@ def test_a_pruned_index_is_not_reused_within_the_same_second( second.write_text("newer", encoding="utf-8") first.unlink() + # Act third = backup_path_for(settings, now=stamp) + # Assert assert third.name != first.name assert third.name > second.name - def test_the_backup_sits_beside_the_settings_file(self, tmp_path: pathlib.Path) -> None: + def test_given_a_settings_path_when_building_a_backup_path_then_it_sits_beside_the_file( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange settings = tmp_path / "settings.json" + + # Act & Assert assert backup_path_for(settings).parent == tmp_path assert backup_path_for(settings).name.startswith("settings.json.") assert backup_path_for(settings).name.endswith(".bak") - def test_only_the_newest_backups_are_kept(self, settings_file: pathlib.Path) -> None: + def test_given_more_applies_than_the_retention_limit_when_listing_backups_then_the_limit_holds( + self, + settings_file: pathlib.Path, + ) -> None: """Each backup is a full copy of a file naming an endpoint and an output path. Unbounded retention left a growing pile of them beside the settings file, which is a disclosure surface that grows with ordinary use. """ + # Act for index in range(BACKUP_RETENTION + 3): apply_changes(settings_file, {MAXSIZE: index + 1}, apply=True) + + # Assert backups = sorted(settings_file.parent.glob(f"{settings_file.name}.*.bak")) assert len(backups) == BACKUP_RETENTION - def test_pruning_reports_what_it_removed(self, tmp_path: pathlib.Path) -> None: + def test_given_backups_beyond_the_retention_limit_when_pruning_then_removed_paths_are_returned( + self, + tmp_path: pathlib.Path, + ) -> None: + # Arrange settings = tmp_path / "settings.json" settings.write_text("{}\n", encoding="utf-8") created = [] @@ -719,11 +1071,15 @@ def test_pruning_reports_what_it_removed(self, tmp_path: pathlib.Path) -> None: backup = settings.with_name(f"{settings.name}.2026010{index}T000000Z-000.bak") backup.write_text("{}\n", encoding="utf-8") created.append(backup) + + # Act removed = prune_backups(settings) + + # Assert assert removed == created[:2] assert all(not path.exists() for path in removed) - def test_a_backwards_clock_still_sorts_the_newest_backup_last( + def test_given_a_clock_that_moves_backwards_when_building_a_backup_then_it_sorts_last( self, tmp_path: pathlib.Path ) -> None: """Retention reads name order, so a regressing clock must not reorder it. @@ -732,6 +1088,7 @@ def test_a_backwards_clock_still_sorts_the_newest_backup_last( back a second. Without this the newest backup takes the lowest name and retention deletes it first, which loses the only copy that matters. """ + # Arrange settings = tmp_path / "settings.json" settings.write_text("{}\n", encoding="utf-8") later = datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.UTC) @@ -739,13 +1096,22 @@ def test_a_backwards_clock_still_sorts_the_newest_backup_last( first = backup_path_for(settings, now=later) first.write_text("{}\n", encoding="utf-8") + + # Act second = backup_path_for(settings, now=earlier) + # Assert assert second.name > first.name - def test_pruning_keeps_the_newest_backup(self, settings_file: pathlib.Path) -> None: + def test_given_pruning_after_many_applies_when_reading_the_last_backup_then_it_is_the_newest( + self, + settings_file: pathlib.Path, + ) -> None: + # Act for index in range(BACKUP_RETENTION + 2): apply_changes(settings_file, {MAXSIZE: index + 1}, apply=True) + + # Assert backups = sorted(settings_file.parent.glob(f"{settings_file.name}.*.bak")) newest = json.loads(strip_jsonc(backups[-1].read_text(encoding="utf-8-sig"))) assert newest[MAXSIZE] == BACKUP_RETENTION + 1 @@ -760,42 +1126,71 @@ class TestTrailingCommas: because both sides of that comparison are produced by the same pass. """ - def test_a_trailing_comma_in_an_object_parses(self) -> None: + def test_given_an_object_with_a_trailing_comma_when_stripping_jsonc_then_it_parses( + self, + ) -> None: + # Act & Assert assert json.loads(strip_jsonc('{"a": 1,}')) == {"a": 1} - def test_a_trailing_comma_in_an_array_parses(self) -> None: + def test_given_an_array_with_a_trailing_comma_when_stripping_jsonc_then_it_parses(self) -> None: + # Act & Assert assert json.loads(strip_jsonc('{"a": [1, 2,],}')) == {"a": [1, 2]} - def test_a_trailing_comma_followed_by_a_comment_parses(self) -> None: + def test_given_a_trailing_comma_before_a_comment_when_stripping_jsonc_then_it_parses( + self, + ) -> None: + # Act & Assert assert json.loads(strip_jsonc('{"a": 1, // note\n}')) == {"a": 1} - def test_a_comma_inside_a_string_literal_survives_byte_for_byte(self) -> None: + def test_given_a_comma_inside_a_string_literal_when_stripping_jsonc_then_it_survives_intact( + self, + ) -> None: + # Arrange source = '{"note": "literal, }", "a": 1,}' + + # Act stripped = strip_jsonc(source) + + # Assert assert len(stripped) == len(source), "offsets were not preserved" assert '"literal, }"' in stripped, "a comma inside a string was rewritten" assert json.loads(stripped) == {"note": "literal, }", "a": 1} - def test_the_scan_preserves_every_offset(self) -> None: + def test_given_a_trailing_comma_and_comment_when_stripping_jsonc_then_the_length_is_unchanged( + self, + ) -> None: + # Arrange source = '{\n "a": 1, // trailing\n}\n' + + # Act & Assert assert len(strip_jsonc(source)) == len(source) - def test_an_edit_survives_a_document_with_a_trailing_comma( + def test_given_a_document_with_a_trailing_comma_when_applying_a_change_then_both_keys_parse( self, tmp_path: pathlib.Path ) -> None: + # Arrange settings = tmp_path / "settings.json" settings.write_text('{\n "editor.fontSize": 13,\n}\n', encoding="utf-8") + + # Act apply_changes(settings, {ENABLED: True}, apply=True) + + # Assert parsed = json.loads(strip_jsonc(settings.read_text(encoding="utf-8"))) assert parsed[ENABLED] is True assert parsed["editor.fontSize"] == 13 - def test_a_string_containing_a_brace_is_not_read_as_structure( + def test_given_a_string_containing_a_brace_when_applying_a_change_then_the_string_is_intact( self, tmp_path: pathlib.Path ) -> None: + # Arrange settings = tmp_path / "settings.json" settings.write_text('{\n "note": "literal, }",\n}\n', encoding="utf-8") + + # Act apply_changes(settings, {ENABLED: True}, apply=True) + + # Assert text = settings.read_text(encoding="utf-8") assert '"literal, }"' in text assert json.loads(strip_jsonc(text))["note"] == "literal, }" From 9001846ea562fe199e52343be620fca3e0708780 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Fri, 21 Aug 2026 22:34:20 -0700 Subject: [PATCH 26/28] fix(workflows): gate the copilot-otel runtime lane in PR validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The strict container-backed slow lane ran but was absent from pr-validation-success needs, so it could never block a merge. 🚦 - Generated by Copilot --- .github/workflows/pr-validation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 55f19fcd7..2dd1e79dc 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -695,6 +695,7 @@ jobs: - yaml-lint - pester-tests - pytest + - copilot-otel-runtime-tests - discover-node-projects - node-tests - accessibility-browser-smoke From 53a35176872269928ae3a443eb3f43a01546a298 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Sat, 22 Aug 2026 08:27:53 -0700 Subject: [PATCH 27/28] fix(workflows): remove untrusted checkout and credential exposure from release paths Bind extension packaging, provenance, SBOM, and provenance-verification checkouts to the triggering commit after the existing release identity gates, so no reusable-workflow input or needs output selects the checked-out tree. Replace pull-request derived git transport in the promotion and Stable trigger gates with API reads pinned to immutable commits, including bounded annotated-tag peeling and compare-based containment. Add the minimal pull-requests read scope only where pull request metadata is required. Separate release preparation data from the write credential: resolve the managed branch to one immutable commit, fetch it over a credential-free remote, transform it without a token, and scope the App token to a single force-with-lease push. Add a request budget to the docs e2e static server and pin the devcontainer base image by multi-architecture digest. --- .devcontainer/Dockerfile | 6 +- .github/workflows/extension-package.yml | 20 +- .github/workflows/extension-provenance.yml | 4 +- .github/workflows/pr-validation.yml | 106 +++++++--- .github/workflows/release-prerelease.yml | 71 +++++-- .github/workflows/release-stable-publish.yml | 165 ++++++++++++---- docs/docusaurus/e2e/static-server.mjs | 32 +++ docs/docusaurus/e2e/static-server.spec.ts | 102 ++++++++++ docs/docusaurus/package-lock.json | 31 +++ docs/docusaurus/package.json | 1 + .../Workflow-PackagingContracts.Tests.ps1 | 182 +++++++++++++++++- 11 files changed, 625 insertions(+), 95 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e0bfbc7da..8fe734ca2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,9 @@ ARG BASE_IMAGE= -FROM ${BASE_IMAGE:-mcr.microsoft.com/devcontainers/base:2-jammy} +# mcr.microsoft.com/devcontainers/base:2-jammy, pinned by multi-architecture +# manifest digest (linux/amd64, linux/arm64) so the tag cannot move underneath +# a reviewed configuration. Resolve a replacement from the publisher's own +# registry metadata, not from a local pull. +FROM ${BASE_IMAGE:-mcr.microsoft.com/devcontainers/base@sha256:55126de3b3e113096201055951b4cb717900bafb379c59dcbada65c49cd7ce22} ARG NPM_CONFIG_REGISTRY= ENV NPM_CONFIG_REGISTRY=${NPM_CONFIG_REGISTRY:-https://registry.npmjs.org/} diff --git a/.github/workflows/extension-package.yml b/.github/workflows/extension-package.yml index d0dae6e8d..bdb91eb82 100644 --- a/.github/workflows/extension-package.yml +++ b/.github/workflows/extension-package.yml @@ -32,6 +32,7 @@ jobs: - name: Validate source ref shell: bash env: + EVENT_SHA: ${{ github.sha }} INPUT_SOURCE_REF: ${{ inputs.source-ref }} run: | set -euo pipefail @@ -40,22 +41,21 @@ jobs: echo "::error::source-ref is required and must not be blank" exit 1 fi - if printf '%s' "$source_ref" | grep -Eq '^[0-9a-fA-F]{40}$'; then - exit 0 - fi - if git check-ref-format --branch "$source_ref" >/dev/null 2>&1; then - exit 0 + # source-ref is an asserted contract value; the checkout below uses the + # run's own commit, so a mutable ref can never select the packaged tree. + if ! printf '%s' "$source_ref" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::source-ref must be a full 40-character lowercase commit SHA" + exit 1 fi - if [[ "$source_ref" == refs/* ]] && git check-ref-format "$source_ref" >/dev/null 2>&1; then - exit 0 + if [ "$source_ref" != "$EVENT_SHA" ]; then + echo "::error::source-ref $source_ref is not the packaging source commit $EVENT_SHA" + exit 1 fi - echo "::error::source-ref must be a full commit SHA or valid Git ref" - exit 1 - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.source-ref }} + ref: ${{ github.sha }} persist-credentials: false # The gate runs before any dependency install, so a mismatched caller diff --git a/.github/workflows/extension-provenance.yml b/.github/workflows/extension-provenance.yml index de2d6c560..36275e1df 100644 --- a/.github/workflows/extension-provenance.yml +++ b/.github/workflows/extension-provenance.yml @@ -53,13 +53,13 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.source-ref }} + ref: ${{ github.sha }} persist-credentials: false - name: Checkout Syft config uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.source-ref }} + ref: ${{ github.sha }} sparse-checkout: .syft.yaml sparse-checkout-cone-mode: false persist-credentials: false diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index b012e84e3..9b77be1df 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -507,6 +507,9 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + # The promotion checks read the pull request head through the API instead + # of fetching its ref, which needs this scope and nothing more. + pull-requests: read steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -533,10 +536,12 @@ jobs: && github.event.pull_request.base.ref == 'release/prerelease' && github.event.pull_request.head.ref == 'release-promotion--main--to--release-prerelease' env: + GH_TOKEN: ${{ github.token }} HEAD_BRANCH: ${{ github.event.pull_request.head.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} PROMOTION_HEAD: release-promotion--main--to--release-prerelease + REPOSITORY: ${{ github.repository }} run: | set -euo pipefail if [ "$HEAD_BRANCH" != "$PROMOTION_HEAD" ]; then @@ -552,22 +557,30 @@ jobs: exit 1 fi - # hve-core is public, so this origin access is intentionally - # credential free; the fetch and ls-remote below stop working if the - # repository ever becomes private. - - # The verified commit is the exact event head, so a head that moved - # after the event fails rather than being proved at a later tip. - git fetch --no-tags origin \ - "+refs/pull/$PR_NUMBER/head:refs/remotes/pull/$PR_NUMBER/head" \ - "+refs/heads/release/prerelease:refs/remotes/origin/release/prerelease" - if [ "$(git rev-parse "refs/remotes/pull/$PR_NUMBER/head")" != "$HEAD_SHA" ]; then - echo "::error::Fetched PreRelease promotion head does not match event head $HEAD_SHA" + # Every read below is pinned to an immutable commit and served by the + # API, so no pull-request ref enters the local object store. A head + # that moved after the event fails rather than being proved at a + # later tip. + CURRENT_HEAD=$(gh api "/repos/$REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha') + if [ "$CURRENT_HEAD" != "$HEAD_SHA" ]; then + echo "::error::PreRelease promotion head moved from $HEAD_SHA to $CURRENT_HEAD" exit 1 fi - CANDIDATE=$(git show "refs/remotes/pull/$PR_NUMBER/head:release-please-prerelease-config.json" | jq -r '.packages["."]["release-as"] // ""') - BASELINE=$(git show 'refs/remotes/origin/release/prerelease:.release-please-prerelease-manifest.json' | jq -r '.["."] // ""') + # release/prerelease is mutable, so it is resolved once and every later + # read uses the resolved commit rather than the branch name. + BASELINE_SHA=$(gh api "/repos/$REPOSITORY/git/ref/heads/release/prerelease" --jq '.object.sha') + if [[ ! "$BASELINE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::release/prerelease did not resolve to a commit SHA" + exit 1 + fi + + CANDIDATE=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/release-please-prerelease-config.json?ref=$HEAD_SHA" \ + | jq -r '.packages["."]["release-as"] // ""') + BASELINE=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/.release-please-prerelease-manifest.json?ref=$BASELINE_SHA" \ + | jq -r '.["."] // ""') for entry in "candidate:$CANDIDATE" "baseline:$BASELINE"; do value=${entry#*:} if [[ "$value" == *$'\n'* || "$value" == *$'\r'* || ! "$value" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -603,9 +616,11 @@ jobs: && github.event.pull_request.base.ref == 'release/stable' && startsWith(github.head_ref, 'release-promotion--release-prerelease--to--release-stable--prerelease-v') env: + GH_TOKEN: ${{ github.token }} HEAD_BRANCH: ${{ github.head_ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} + REPOSITORY: ${{ github.repository }} run: | set -euo pipefail if [[ ! "$HEAD_BRANCH" =~ ^release-promotion--release-prerelease--to--release-stable--(prerelease-v[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then @@ -625,17 +640,26 @@ jobs: exit 1 fi - git fetch --no-tags origin \ - "+refs/pull/$PR_NUMBER/head:refs/remotes/pull/$PR_NUMBER/head" \ - "+refs/heads/release/stable:refs/remotes/origin/release/stable" \ - "+refs/tags/$SOURCE_TAG:refs/tags/$SOURCE_TAG" - if [ "$(git rev-parse "refs/remotes/pull/$PR_NUMBER/head")" != "$HEAD_SHA" ]; then - echo "::error::Fetched Stable promotion head does not match event head $HEAD_SHA" + # Every read below is pinned to an immutable commit and served by the + # API, so no pull-request ref enters the local object store. + CURRENT_HEAD=$(gh api "/repos/$REPOSITORY/pulls/$PR_NUMBER" --jq '.head.sha') + if [ "$CURRENT_HEAD" != "$HEAD_SHA" ]; then + echo "::error::Stable promotion head moved from $HEAD_SHA to $CURRENT_HEAD" exit 1 fi - CANDIDATE=$(git show "refs/remotes/pull/$PR_NUMBER/head:release-please-config.json" | jq -r '.packages["."]["release-as"] // ""') - BASELINE=$(git show 'refs/remotes/origin/release/stable:.release-please-manifest.json' | jq -r '.["."] // ""') + BASELINE_SHA=$(gh api "/repos/$REPOSITORY/git/ref/heads/release/stable" --jq '.object.sha') + if [[ ! "$BASELINE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::release/stable did not resolve to a commit SHA" + exit 1 + fi + + CANDIDATE=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/release-please-config.json?ref=$HEAD_SHA" \ + | jq -r '.packages["."]["release-as"] // ""') + BASELINE=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/.release-please-manifest.json?ref=$BASELINE_SHA" \ + | jq -r '.["."] // ""') for entry in "candidate:$CANDIDATE" "baseline:$BASELINE"; do value=${entry#*:} if [[ "$value" == *$'\n'* || "$value" == *$'\r'* || ! "$value" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -654,8 +678,44 @@ jobs: exit 1 fi - SOURCE_SHA=$(git rev-parse --verify --end-of-options "refs/tags/$SOURCE_TAG^{commit}") - if git merge-base --is-ancestor "$SOURCE_SHA" refs/remotes/origin/release/stable; then + # Annotated tags peel to a commit through the API, bounded by a hop + # limit and a visited set so a malformed chain cannot loop. + tag_object=$(gh api "/repos/$REPOSITORY/git/ref/tags/$SOURCE_TAG" --jq '.object.type + " " + .object.sha') + object_type=${tag_object%% *} + object_sha=${tag_object##* } + visited='' + SOURCE_SHA='' + hop=0 + while [ "$hop" -lt 10 ]; do + hop=$((hop + 1)) + if [ "$object_type" = 'commit' ]; then + SOURCE_SHA="$object_sha" + break + fi + if [ "$object_type" != 'tag' ]; then + echo "::error::$SOURCE_TAG resolves to unsupported object type $object_type" + exit 1 + fi + case " $visited " in + *" $object_sha "*) + echo "::error::$SOURCE_TAG tag chain revisits $object_sha" + exit 1 + ;; + esac + visited="$visited $object_sha" + tag_object=$(gh api "/repos/$REPOSITORY/git/tags/$object_sha" --jq '.object.type + " " + .object.sha') + object_type=${tag_object%% *} + object_sha=${tag_object##* } + done + if [[ ! "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::$SOURCE_TAG did not peel to a commit within the allowed hops" + exit 1 + fi + + # ahead or identical means the source commit is already reachable from + # release/stable, which is what the previous ancestry test proved. + CONTAINMENT=$(gh api "/repos/$REPOSITORY/compare/$SOURCE_SHA...$BASELINE_SHA" --jq '.status') + if [ "$CONTAINMENT" = 'ahead' ] || [ "$CONTAINMENT" = 'identical' ]; then echo "::error::$SOURCE_TAG at $SOURCE_SHA is already contained in release/stable" exit 1 fi diff --git a/.github/workflows/release-prerelease.yml b/.github/workflows/release-prerelease.yml index ef852fdcc..2f44dce4d 100644 --- a/.github/workflows/release-prerelease.yml +++ b/.github/workflows/release-prerelease.yml @@ -344,17 +344,45 @@ jobs: scripts/plugins/Sync-PluginManifest.ps1 sparse-checkout-cone-mode: false persist-credentials: false - # Data-only checkout: executable tooling comes from the trusted github.sha checkout. - - name: Checkout release preparation data - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ needs.release-please.outputs.release-pr-branch }} - path: release-preparation - token: ${{ steps.app-token.outputs.token }} - persist-credentials: true - - name: Update committed version fields and retire the promotion intent + # The managed branch is resolved to one immutable commit and fetched over a + # credential-free remote, so no write token exists while the tree is + # transformed. + - name: Resolve and fetch the release preparation commit + id: preparation env: + GH_TOKEN: ${{ github.token }} RELEASE_BRANCH: ${{ needs.release-please.outputs.release-pr-branch }} + REPOSITORY: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + set -euo pipefail + if [[ ! "$RELEASE_BRANCH" =~ ^release-please--[A-Za-z0-9._/-]+$ ]]; then + echo "::error::$RELEASE_BRANCH is not a managed release preparation branch" + exit 1 + fi + + resolved_object=$(gh api "/repos/$REPOSITORY/git/ref/heads/$RELEASE_BRANCH" --jq '.object.sha') + if [[ ! "$resolved_object" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::$RELEASE_BRANCH did not resolve to a commit SHA" + exit 1 + fi + + RELEASE_REPO="$GITHUB_WORKSPACE/release-preparation" + mkdir -p "$RELEASE_REPO" + git -C "$RELEASE_REPO" init --quiet + git -C "$RELEASE_REPO" -c credential.helper= fetch --no-tags --depth 1 \ + "$SERVER_URL/$REPOSITORY.git" \ + "+refs/heads/$RELEASE_BRANCH:refs/remotes/preparation/head" + fetched_object=$(git -C "$RELEASE_REPO" rev-parse refs/remotes/preparation/head) + if [ "$fetched_object" != "$resolved_object" ]; then + echo "::error::$RELEASE_BRANCH moved from $resolved_object to $fetched_object during preparation" + exit 1 + fi + git -C "$RELEASE_REPO" checkout --quiet --detach refs/remotes/preparation/head + + echo "commit=$resolved_object" >> "$GITHUB_OUTPUT" + - name: Update committed version fields and retire the promotion intent + id: transform run: | set -euo pipefail RELEASE_REPO="$GITHUB_WORKSPACE/release-preparation" @@ -392,10 +420,29 @@ jobs: git add --all if git diff --cached --quiet; then echo "Preparation already carries $VERSION and its synchronized plugin manifest" + echo "committed=false" >> "$GITHUB_OUTPUT" exit 0 fi git commit -m "chore(release): sync committed versions and plugin manifest to $VERSION" - git push origin "HEAD:refs/heads/$RELEASE_BRANCH" + echo "committed=true" >> "$GITHUB_OUTPUT" + # The write credential exists only for this push, and the lease binds the + # update to the exact commit the tree was built from. + - name: Push the synchronized release preparation commit + if: ${{ steps.transform.outputs.committed == 'true' }} + env: + EXPECTED_COMMIT: ${{ steps.preparation.outputs.commit }} + GH_APP_TOKEN: ${{ steps.app-token.outputs.token }} + RELEASE_BRANCH: ${{ needs.release-please.outputs.release-pr-branch }} + REPOSITORY: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + set -euo pipefail + RELEASE_REPO="$GITHUB_WORKSPACE/release-preparation" + push_host=${SERVER_URL#https://} + git -C "$RELEASE_REPO" -c credential.helper= push \ + --force-with-lease="refs/heads/$RELEASE_BRANCH:$EXPECTED_COMMIT" \ + "https://x-access-token:$GH_APP_TOKEN@$push_host/$REPOSITORY.git" \ + "HEAD:refs/heads/$RELEASE_BRANCH" validate-release: name: Validate release-please Pre-Release output needs: @@ -663,7 +710,7 @@ jobs: - name: Checkout dependency manifests uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ needs.validate-release.outputs.sha }} + ref: ${{ github.sha }} sparse-checkout: | package.json package-lock.json @@ -726,7 +773,7 @@ jobs: - name: Checkout verification script uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ needs.validate-release.outputs.sha }} + ref: ${{ github.sha }} persist-credentials: false - name: Download release artifacts env: diff --git a/.github/workflows/release-stable-publish.yml b/.github/workflows/release-stable-publish.yml index e9aa43230..1e1a3c096 100644 --- a/.github/workflows/release-stable-publish.yml +++ b/.github/workflows/release-stable-publish.yml @@ -83,14 +83,6 @@ jobs: echo "source-tag=$SOURCE_TAG" } >> "$GITHUB_OUTPUT" - - name: Checkout Stable trigger validation workspace - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: main - fetch-depth: 0 - path: trigger-validation - persist-credentials: false - - name: Validate selected promotion source and intent id: source if: ${{ steps.identity.outputs.mode == 'promotion' }} @@ -98,29 +90,51 @@ jobs: EVENT_SHA: ${{ github.sha }} GH_TOKEN: ${{ github.token }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} - PR_NUMBER: ${{ github.event.pull_request.number }} REPOSITORY: ${{ github.repository }} SOURCE_TAG: ${{ steps.identity.outputs.source-tag }} run: | set -euo pipefail - cd "$GITHUB_WORKSPACE/trigger-validation" if [[ ! "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || [[ ! "$EVENT_SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "::error::Stable promotion event carries an invalid commit SHA" exit 1 fi SOURCE_VERSION=${SOURCE_TAG#prerelease-v} - git fetch --no-tags origin \ - "+refs/pull/$PR_NUMBER/head:refs/remotes/pull/$PR_NUMBER/head" \ - "+refs/heads/release/prerelease:refs/remotes/origin/release/prerelease" \ - "+refs/heads/release/stable:refs/remotes/origin/release/stable" - if [ "$(git rev-parse "refs/remotes/pull/$PR_NUMBER/head")" != "$HEAD_SHA" ]; then - echo "::error::Fetched pull request head does not match event head $HEAD_SHA" + + # Annotated tags peel to a commit through the API, bounded by a hop + # limit and a visited set so a malformed chain cannot loop. + tag_object=$(gh api "/repos/$REPOSITORY/git/ref/tags/$SOURCE_TAG" --jq '.object.type + " " + .object.sha') + object_type=${tag_object%% *} + object_sha=${tag_object##* } + visited='' + SOURCE_SHA='' + hop=0 + while [ "$hop" -lt 10 ]; do + hop=$((hop + 1)) + if [ "$object_type" = 'commit' ]; then + SOURCE_SHA="$object_sha" + break + fi + if [ "$object_type" != 'tag' ]; then + echo "::error::$SOURCE_TAG resolves to unsupported object type $object_type" + exit 1 + fi + case " $visited " in + *" $object_sha "*) + echo "::error::$SOURCE_TAG tag chain revisits $object_sha" + exit 1 + ;; + esac + visited="$visited $object_sha" + tag_object=$(gh api "/repos/$REPOSITORY/git/tags/$object_sha" --jq '.object.type + " " + .object.sha') + object_type=${tag_object%% *} + object_sha=${tag_object##* } + done + if [[ ! "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::$SOURCE_TAG did not peel to a commit within the allowed hops" exit 1 fi - git fetch --no-tags origin "+refs/tags/$SOURCE_TAG:refs/tags/$SOURCE_TAG" - SOURCE_SHA=$(git rev-parse --verify --end-of-options "refs/tags/$SOURCE_TAG^{commit}") release_state=$(gh api "/repos/$REPOSITORY/releases/tags/$SOURCE_TAG" \ --jq '[.draft, .prerelease, (.published_at != null), .tag_name] | @tsv' 2>/dev/null || true) if [ "$release_state" != $'false\ttrue\ttrue\t'"$SOURCE_TAG" ]; then @@ -128,22 +142,45 @@ jobs: exit 1 fi - MANIFEST_VERSION=$(git show "$SOURCE_SHA:.release-please-prerelease-manifest.json" | jq -r '.["."] // ""') + MANIFEST_VERSION=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/.release-please-prerelease-manifest.json?ref=$SOURCE_SHA" \ + | jq -r '.["."] // ""') if [ "$MANIFEST_VERSION" != "$SOURCE_VERSION" ]; then echo "::error::$SOURCE_TAG resolves to $SOURCE_SHA but the PreRelease manifest carries $MANIFEST_VERSION" exit 1 fi - if ! git merge-base --is-ancestor "$SOURCE_SHA" refs/remotes/origin/release/prerelease; then + + # Mutable release branches are resolved once, and containment is proved + # against the resolved immutable commits through the compare API. + PRERELEASE_SHA=$(gh api "/repos/$REPOSITORY/git/ref/heads/release/prerelease" --jq '.object.sha') + STABLE_SHA=$(gh api "/repos/$REPOSITORY/git/ref/heads/release/stable" --jq '.object.sha') + for resolved in "release/prerelease:$PRERELEASE_SHA" "release/stable:$STABLE_SHA"; do + if [[ ! "${resolved#*:}" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::${resolved%%:*} did not resolve to a commit SHA" + exit 1 + fi + done + + CONTAINMENT=$(gh api "/repos/$REPOSITORY/compare/$SOURCE_SHA...$PRERELEASE_SHA" --jq '.status') + if [ "$CONTAINMENT" != 'ahead' ] && [ "$CONTAINMENT" != 'identical' ]; then echo "::error::$SOURCE_TAG commit $SOURCE_SHA is not contained in release/prerelease" exit 1 fi - if ! git merge-base --is-ancestor "$SOURCE_SHA" "refs/remotes/pull/$PR_NUMBER/head"; then + + # The event head SHA is immutable, so the promotion head is compared + # directly rather than fetched. + HEAD_CONTAINMENT=$(gh api "/repos/$REPOSITORY/compare/$SOURCE_SHA...$HEAD_SHA" --jq '.status') + if [ "$HEAD_CONTAINMENT" != 'ahead' ] && [ "$HEAD_CONTAINMENT" != 'identical' ]; then echo "::error::Promotion head $HEAD_SHA does not contain selected source $SOURCE_SHA" exit 1 fi - CANDIDATE=$(git show "$EVENT_SHA:release-please-config.json" | jq -r '.packages["."]["release-as"] // ""') - BASELINE=$(git show 'refs/remotes/origin/release/stable:.release-please-manifest.json' | jq -r '.["."] // ""') + CANDIDATE=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/release-please-config.json?ref=$EVENT_SHA" \ + | jq -r '.packages["."]["release-as"] // ""') + BASELINE=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/.release-please-manifest.json?ref=$STABLE_SHA" \ + | jq -r '.["."] // ""') for entry in "candidate:$CANDIDATE" "baseline:$BASELINE"; do value=${entry#*:} if [[ "$value" == *$'\n'* || "$value" == *$'\r'* || ! "$value" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -168,11 +205,13 @@ jobs: if: ${{ steps.identity.outputs.mode == 'managed' }} env: EVENT_SHA: ${{ github.sha }} + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} run: | set -euo pipefail - cd "$GITHUB_WORKSPACE/trigger-validation" - git fetch --no-tags origin "+refs/heads/release/stable:refs/remotes/origin/release/stable" - INTENT=$(git show "$EVENT_SHA:release-please-config.json" | jq -r '.packages["."]["release-as"] // ""') + INTENT=$(gh api -H 'Accept: application/vnd.github.raw' \ + "/repos/$REPOSITORY/contents/release-please-config.json?ref=$EVENT_SHA" \ + | jq -r '.packages["."]["release-as"] // ""') if [ -n "$INTENT" ]; then echo "::error::Managed Stable merge still carries release-as $INTENT" exit 1 @@ -342,18 +381,46 @@ jobs: sparse-checkout-cone-mode: false persist-credentials: false - # Data-only checkout: executable tooling comes from the trusted github.sha checkout. - - name: Checkout release preparation data - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ needs.release-please.outputs.release-pr-branch }} - path: release-preparation - token: ${{ steps.app-token.outputs.token }} - persist-credentials: true - - - name: Update committed version fields and retire the promotion intent + # The managed branch is resolved to one immutable commit and fetched over a + # credential-free remote, so no write token exists while the tree is + # transformed. + - name: Resolve and fetch the release preparation commit + id: preparation env: + GH_TOKEN: ${{ github.token }} RELEASE_BRANCH: ${{ needs.release-please.outputs.release-pr-branch }} + REPOSITORY: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + set -euo pipefail + if [[ ! "$RELEASE_BRANCH" =~ ^release-please--[A-Za-z0-9._/-]+$ ]]; then + echo "::error::$RELEASE_BRANCH is not a managed release preparation branch" + exit 1 + fi + + resolved_object=$(gh api "/repos/$REPOSITORY/git/ref/heads/$RELEASE_BRANCH" --jq '.object.sha') + if [[ ! "$resolved_object" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::$RELEASE_BRANCH did not resolve to a commit SHA" + exit 1 + fi + + RELEASE_REPO="$GITHUB_WORKSPACE/release-preparation" + mkdir -p "$RELEASE_REPO" + git -C "$RELEASE_REPO" init --quiet + git -C "$RELEASE_REPO" -c credential.helper= fetch --no-tags --depth 1 \ + "$SERVER_URL/$REPOSITORY.git" \ + "+refs/heads/$RELEASE_BRANCH:refs/remotes/preparation/head" + fetched_object=$(git -C "$RELEASE_REPO" rev-parse refs/remotes/preparation/head) + if [ "$fetched_object" != "$resolved_object" ]; then + echo "::error::$RELEASE_BRANCH moved from $resolved_object to $fetched_object during preparation" + exit 1 + fi + git -C "$RELEASE_REPO" checkout --quiet --detach refs/remotes/preparation/head + + echo "commit=$resolved_object" >> "$GITHUB_OUTPUT" + + - name: Update committed version fields and retire the promotion intent + id: transform run: | set -euo pipefail RELEASE_REPO="$GITHUB_WORKSPACE/release-preparation" @@ -392,10 +459,30 @@ jobs: git add --all if git diff --cached --quiet; then echo "Preparation already carries $VERSION and its synchronized plugin manifest" + echo "committed=false" >> "$GITHUB_OUTPUT" exit 0 fi git commit -m "chore(release): sync committed versions and plugin manifest to $VERSION" - git push origin "HEAD:refs/heads/$RELEASE_BRANCH" + echo "committed=true" >> "$GITHUB_OUTPUT" + + # The write credential exists only for this push, and the lease binds the + # update to the exact commit the tree was built from. + - name: Push the synchronized release preparation commit + if: ${{ steps.transform.outputs.committed == 'true' }} + env: + EXPECTED_COMMIT: ${{ steps.preparation.outputs.commit }} + GH_APP_TOKEN: ${{ steps.app-token.outputs.token }} + RELEASE_BRANCH: ${{ needs.release-please.outputs.release-pr-branch }} + REPOSITORY: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + set -euo pipefail + RELEASE_REPO="$GITHUB_WORKSPACE/release-preparation" + push_host=${SERVER_URL#https://} + git -C "$RELEASE_REPO" -c credential.helper= push \ + --force-with-lease="refs/heads/$RELEASE_BRANCH:$EXPECTED_COMMIT" \ + "https://x-access-token:$GH_APP_TOKEN@$push_host/$REPOSITORY.git" \ + "HEAD:refs/heads/$RELEASE_BRANCH" validate-release: name: Validate release-please Stable output @@ -681,7 +768,7 @@ jobs: - name: Checkout dependency manifests uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ needs.validate-release.outputs.sha }} + ref: ${{ github.sha }} sparse-checkout: | package.json package-lock.json @@ -741,7 +828,7 @@ jobs: - name: Checkout verification script uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ needs.validate-release.outputs.sha }} + ref: ${{ github.sha }} persist-credentials: false - name: Download release artifacts diff --git a/docs/docusaurus/e2e/static-server.mjs b/docs/docusaurus/e2e/static-server.mjs index 7d5c2a921..edf67c752 100644 --- a/docs/docusaurus/e2e/static-server.mjs +++ b/docs/docusaurus/e2e/static-server.mjs @@ -5,6 +5,7 @@ import path from 'node:path'; import fs from 'node:fs'; import express from 'express'; import compression from 'compression'; +import rateLimit from 'express-rate-limit'; // Production-grade static server for the e2e suite. // @@ -19,6 +20,26 @@ const BASE = '/hve-core/'; const PORT = Number(process.env.PORT ?? 3001); const HOST = process.env.HOST ?? '127.0.0.1'; +function readPositiveInteger(name, fallback) { + const raw = process.env[name]; + if (raw === undefined || raw.trim() === '') { + return fallback; + } + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + console.error(`[e2e static server] ${name} must be a positive integer, received "${raw}".`); + process.exit(1); + } + return value; +} + +// Request budget for every filesystem-backed route. The default ceiling is +// deliberately high because all parallel Playwright workers share one loopback +// address, so a production-sized limit would throttle the suite rather than an +// abusive client. The overrides let a test drive a deterministic 429. +const RATE_LIMIT_WINDOW_MS = readPositiveInteger('RATE_LIMIT_WINDOW_MS', 60_000); +const RATE_LIMIT_MAX = readPositiveInteger('RATE_LIMIT_MAX', 100_000); + // Fail loudly when the build output is missing rather than serving 404s that // surface downstream as opaque navigation failures. Callers must build first. if (!fs.existsSync(path.join(buildDir, 'index.html'))) { @@ -39,6 +60,17 @@ const notFoundPage = path.join(buildDir, '404.html'); const app = express(); app.use(compression()); +// Applied before any static or not-found handler so no filesystem read is served +// outside the budget. +app.use( + rateLimit({ + windowMs: RATE_LIMIT_WINDOW_MS, + limit: RATE_LIMIT_MAX, + standardHeaders: 'draft-8', + legacyHeaders: false, + }), +); + // Serve the static build under the configured baseUrl. Directory requests // resolve to their index.html, matching Docusaurus trailing-slash routes. app.use( diff --git a/docs/docusaurus/e2e/static-server.spec.ts b/docs/docusaurus/e2e/static-server.spec.ts index 8a7cc2e70..49e717f6f 100644 --- a/docs/docusaurus/e2e/static-server.spec.ts +++ b/docs/docusaurus/e2e/static-server.spec.ts @@ -2,6 +2,8 @@ // SPDX-License-Identifier: MIT import fs from 'node:fs'; import path from 'node:path'; +import net from 'node:net'; +import { spawn, type ChildProcess } from 'node:child_process'; import { test, expect } from '@playwright/test'; // Regression lock for the e2e static server's not-found handler. @@ -73,3 +75,103 @@ test.describe('e2e static server not-found handler', () => { expect(response.headers()['content-type']).toContain('text/html'); }); }); + +// The shipped limit is sized for parallel workers sharing one loopback address, +// so enforcement is proved against a second server instance configured with a +// tiny budget. It runs on its own reserved port and never touches the port the +// rest of the suite is using. +test.describe('e2e static server rate limiting', () => { + test.describe.configure({ mode: 'serial' }); + + const windowMs = 2000; + const limit = 3; + let server: ChildProcess | undefined; + let port = 0; + let origin = ''; + + async function reserveLoopbackPort(): Promise { + return await new Promise((resolve, reject) => { + const probe = net.createServer(); + probe.once('error', reject); + probe.listen(0, '127.0.0.1', () => { + const address = probe.address() as net.AddressInfo; + probe.close(() => resolve(address.port)); + }); + }); + } + + async function canConnect(target: number): Promise { + return await new Promise((resolve) => { + const socket = net.connect({ port: target, host: '127.0.0.1' }); + socket.once('connect', () => { + socket.destroy(); + resolve(true); + }); + socket.once('error', () => { + socket.destroy(); + resolve(false); + }); + }); + } + + // A TCP probe rather than an HTTP request, so readiness polling cannot consume + // the very budget these tests assert on. + async function waitForPort(target: number, expected: boolean): Promise { + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + if ((await canConnect(target)) === expected) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`port ${target} never reached connectable=${expected}`); + } + + test.beforeAll(async () => { + port = await reserveLoopbackPort(); + origin = `http://127.0.0.1:${port}`; + server = spawn(process.execPath, [path.resolve(currentDir, 'static-server.mjs')], { + env: { + ...process.env, + PORT: String(port), + HOST: '127.0.0.1', + RATE_LIMIT_MAX: String(limit), + RATE_LIMIT_WINDOW_MS: String(windowMs), + }, + stdio: 'ignore', + }); + await waitForPort(port, true); + }); + + test.afterAll(async () => { + server?.kill(); + server = undefined; + await waitForPort(port, false); + }); + + test('advertises the standard rate limit policy and omits legacy headers', async () => { + const response = await fetch(`${origin}/hve-core/`); + + expect(response.status).toBe(200); + expect(response.headers.get('ratelimit-policy')).toContain(String(limit)); + expect(response.headers.get('x-ratelimit-limit')).toBeNull(); + }); + + test('returns 429 past the budget and serves again after the window resets', async () => { + // Start from a fresh window so the previous test's request cannot shift the count. + await new Promise((resolve) => setTimeout(resolve, windowMs + 100)); + + const statuses: number[] = []; + for (let attempt = 0; attempt < limit + 1; attempt += 1) { + const response = await fetch(`${origin}/hve-core/`); + statuses.push(response.status); + } + + expect(statuses.slice(0, limit)).toEqual(Array(limit).fill(200)); + expect(statuses[limit]).toBe(429); + + await new Promise((resolve) => setTimeout(resolve, windowMs + 100)); + const recovered = await fetch(`${origin}/hve-core/`); + expect(recovered.status).toBe(200); + }); +}); diff --git a/docs/docusaurus/package-lock.json b/docs/docusaurus/package-lock.json index bf7819558..8d54525a9 100644 --- a/docs/docusaurus/package-lock.json +++ b/docs/docusaurus/package-lock.json @@ -37,6 +37,7 @@ "eslint-plugin-jsx-a11y": "6.10.2", "eslint-plugin-mdx": "3.8.1", "express": "5.2.1", + "express-rate-limit": "8.6.2", "identity-obj-proxy": "3.0.0", "jest": "30.4.2", "jest-axe": "11.0.0", @@ -13880,6 +13881,26 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.6.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.6.2.tgz", + "integrity": "sha512-YH4ru+eOJxQABscKFfRCy9R7x9QFGdezclVMwwgFFndzS2Xnm0uo6B0ABZsLhcpeptGv2qvuJVWlQr9gQZoC3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/express/node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -15861,6 +15882,16 @@ "loose-envify": "^1.0.0" } }, + "node_modules/ip-address": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", + "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.3.0.tgz", diff --git a/docs/docusaurus/package.json b/docs/docusaurus/package.json index 1b36f2937..99b9133cc 100644 --- a/docs/docusaurus/package.json +++ b/docs/docusaurus/package.json @@ -53,6 +53,7 @@ "eslint-plugin-jsx-a11y": "6.10.2", "eslint-plugin-mdx": "3.8.1", "express": "5.2.1", + "express-rate-limit": "8.6.2", "identity-obj-proxy": "3.0.0", "jest": "30.4.2", "jest-axe": "11.0.0", diff --git a/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 b/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 index bd245db78..3fe05f673 100644 --- a/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 +++ b/scripts/tests/extension/Workflow-PackagingContracts.Tests.ps1 @@ -292,7 +292,7 @@ Describe 'One-package build and artifact naming' -Tag 'Unit' { $checkout = Get-NamedJobStep -Document $document -JobName 'package' -StepName 'Checkout code' [string]$checkout['with']['ref'] | - Should -BeExactly '${{ inputs.source-ref }}' + Should -BeExactly '${{ github.sha }}' $steps = Get-JobStepText -Document $document -JobName 'package' @($steps | Where-Object { $_ -match 'Prepare-Extension\.ps1$' }) | Should -HaveCount 1 @@ -307,16 +307,12 @@ Describe 'One-package build and artifact naming' -Tag 'Unit' { $names.IndexOf('Validate packaging inputs') | Should -BeLessThan $names.IndexOf('Install dependencies') } - It 'Accepts source-ref form before checkout' -Skip:$script:SkipShellFixtureTests -ForEach @( - @{ SourceRef = '1111111111111111111111111111111111111111' } - @{ SourceRef = 'main' } - @{ SourceRef = 'v3.2.2' } - @{ SourceRef = 'refs/tags/v3.2.2' } - ) { + It 'Accepts a source-ref equal to the run commit before checkout' -Skip:$script:SkipShellFixtureTests { $step = Get-NamedJobStep -Document (Get-WorkflowDocument -Name 'extension-package.yml') ` -JobName 'package' -StepName 'Validate source ref' $result = Invoke-WorkflowShellStep -Body ([string]$step['run']) -TemplateVersion '3.3.0' -Environment @{ - INPUT_SOURCE_REF = $SourceRef + EVENT_SHA = 'abcdef1111111111111111111111111111111111' + INPUT_SOURCE_REF = 'abcdef1111111111111111111111111111111111' } $result.ExitCode | Should -Be 0 } @@ -326,10 +322,16 @@ Describe 'One-package build and artifact naming' -Tag 'Unit' { @{ SourceRef = '-branch' } @{ SourceRef = 'bad..ref' } @{ SourceRef = 'refs/heads/bad ref' } + @{ SourceRef = 'main' } + @{ SourceRef = 'v3.2.2' } + @{ SourceRef = 'refs/tags/v3.2.2' } + @{ SourceRef = 'ABCDEF1111111111111111111111111111111111' } + @{ SourceRef = '2222222222222222222222222222222222222222' } ) { $step = Get-NamedJobStep -Document (Get-WorkflowDocument -Name 'extension-package.yml') ` -JobName 'package' -StepName 'Validate source ref' $result = Invoke-WorkflowShellStep -Body ([string]$step['run']) -TemplateVersion '3.3.0' -Environment @{ + EVENT_SHA = 'abcdef1111111111111111111111111111111111' INPUT_SOURCE_REF = $SourceRef } $result.ExitCode | Should -Not -Be 0 @@ -366,6 +368,170 @@ Describe 'One-package build and artifact naming' -Tag 'Unit' { } } +# A caller-supplied ref must never select a checked-out tree; the run's own +# commit is the only trusted selector, and the gates below are what make it so. +Describe 'Trusted source binding' -Tag 'Unit' { + It 'Checks out the run commit in step ' -ForEach @( + @{ Workflow = 'extension-package.yml'; JobName = 'package'; StepName = 'Checkout code' } + @{ Workflow = 'extension-provenance.yml'; JobName = 'attest'; StepName = 'Checkout code' } + @{ Workflow = 'extension-provenance.yml'; JobName = 'attest'; StepName = 'Checkout Syft config' } + @{ Workflow = 'release-prerelease.yml'; JobName = 'generate-dependency-sbom'; StepName = 'Checkout dependency manifests' } + @{ Workflow = 'release-prerelease.yml'; JobName = 'verify-provenance'; StepName = 'Checkout verification script' } + @{ Workflow = 'release-stable-publish.yml'; JobName = 'generate-dependency-sbom'; StepName = 'Checkout dependency manifests' } + @{ Workflow = 'release-stable-publish.yml'; JobName = 'verify-provenance'; StepName = 'Checkout verification script' } + ) { + $step = Get-NamedJobStep -Document (Get-WorkflowDocument -Name $Workflow) -JobName $JobName -StepName $StepName + [string]$step['with']['ref'] | Should -BeExactly '${{ github.sha }}' + } + + It 'Retains the release identity gate that makes the run commit trusted in ' -ForEach @( + @{ Workflow = 'release-prerelease.yml' } + @{ Workflow = 'release-stable-publish.yml' } + ) { + $document = Get-WorkflowDocument -Name $Workflow + $identity = Get-NamedJobStep -Document $document -JobName 'validate-release' -StepName 'Verify trusted release identity' + [string]$identity['run'] | Should -Match 'MERGE_SHA' + [string]$identity['run'] | Should -Match 'RELEASE_SHA' + [string]$document['jobs']['validate-release']['outputs']['sha'] | Should -BeExactly '${{ github.sha }}' + [string[]]@($document['jobs']['generate-dependency-sbom']['needs']) | Should -Contain 'validate-release' + [string[]]@($document['jobs']['verify-provenance']['needs']) | Should -Contain 'validate-release' + } +} + +# Promotion validation reads through the API now, so the contracts that matter +# are the absence of pull-request Git transport and the immutability of the refs +# every read is pinned to. +Describe 'Immutable promotion validation' -Tag 'Unit' { + $script:PromotionJobs = @( + @{ Workflow = 'pr-validation.yml'; JobName = 'gate-completeness-check' } + @{ Workflow = 'release-stable-publish.yml'; JobName = 'validate-trigger' } + ) + + It 'Uses no pull-request Git transport in job ' -ForEach $script:PromotionJobs { + $steps = Get-JobStepText -Document (Get-WorkflowDocument -Name $Workflow) -JobName $JobName + @($steps | Where-Object { $_ -match 'git\s+fetch|git\s+pull|gh\s+pr\s+checkout|refs/pull/' }) | Should -HaveCount 0 + } + + It 'Pins every promotion content read to a resolved commit in job ' -ForEach $script:PromotionJobs { + $text = (Get-JobStepText -Document (Get-WorkflowDocument -Name $Workflow) -JobName $JobName) -join "`n" + $refs = @([regex]::Matches($text, '\?ref=\$(?[A-Za-z_]+)') | ForEach-Object { $_.Groups['name'].Value } | Sort-Object -Unique) + $refs | Should -Not -BeNullOrEmpty + @($refs | Where-Object { $_ -notin @('HEAD_SHA', 'EVENT_SHA', 'BASELINE_SHA', 'SOURCE_SHA', 'STABLE_SHA', 'PRERELEASE_SHA') }) | + Should -BeNullOrEmpty + } + + It 'Bounds annotated tag peeling in job ' -ForEach $script:PromotionJobs { + $text = (Get-JobStepText -Document (Get-WorkflowDocument -Name $Workflow) -JobName $JobName) -join "`n" + $text | Should -Match 'hop.*-lt 10' + $text | Should -Match 'tag chain revisits' + $text | Should -Match 'unsupported object type' + $text | Should -Match 'did not peel to a commit' + } + + It 'Proves containment through compare status rather than local ancestry in job ' -ForEach $script:PromotionJobs { + $text = (Get-JobStepText -Document (Get-WorkflowDocument -Name $Workflow) -JobName $JobName) -join "`n" + $text | Should -Match 'compare/\$SOURCE_SHA\.\.\.' + $text | Should -Match "'ahead'" + $text | Should -Match "'identical'" + $text | Should -Not -Match 'merge-base' + } + + It 'Declares only the token scope each promotion job needs' { + $gate = (Get-WorkflowDocument -Name 'pr-validation.yml')['jobs']['gate-completeness-check']['permissions'] + [string]$gate['contents'] | Should -BeExactly 'read' + [string]$gate['pull-requests'] | Should -BeExactly 'read' + @($gate.Keys) | Should -HaveCount 2 + + # The Stable trigger compares immutable event SHAs, so it needs no + # pull-request scope at all. + $trigger = (Get-WorkflowDocument -Name 'release-stable-publish.yml')['jobs']['validate-trigger']['permissions'] + [string]$trigger['contents'] | Should -BeExactly 'read' + @($trigger.Keys) | Should -HaveCount 1 + } + + It 'Retains every Stable trigger gate without a validation checkout' { + $document = Get-WorkflowDocument -Name 'release-stable-publish.yml' + $names = [string[]]@($document['jobs']['validate-trigger']['steps'] | ForEach-Object { [string]$_['name'] }) + $names | Should -Not -Contain 'Checkout Stable trigger validation workspace' + + $steps = Get-JobStepText -Document $document -JobName 'validate-trigger' + foreach ($gate in @( + 'is not a merged same-repository pull request', + 'does not match event SHA', + 'is not an authorized Stable release head', + 'is not a published, non-draft GitHub PreRelease', + 'but the PreRelease manifest carries', + 'is not contained in release/prerelease', + 'does not contain selected source', + 'still carries release-as')) { + @($steps | Where-Object { $_ -match [regex]::Escape($gate) }) | + Should -Not -BeNullOrEmpty -Because "the '$gate' check must survive the API migration" + } + } +} + +# The release preparation tree is untrusted data. These contracts keep the write +# credential out of it and bind the push to the commit the tree was built from. +Describe 'Release preparation transport' -Tag 'Unit' { + $script:SyncWorkflows = @( + @{ Workflow = 'release-prerelease.yml' } + @{ Workflow = 'release-stable-publish.yml' } + ) + + It 'Fetches release preparation data without a credential in ' -ForEach $script:SyncWorkflows { + $document = Get-WorkflowDocument -Name $Workflow + $names = [string[]]@($document['jobs']['sync-release-pr']['steps'] | ForEach-Object { [string]$_['name'] }) + $names | Should -Not -Contain 'Checkout release preparation data' + + $checkouts = @($document['jobs']['sync-release-pr']['steps'] | Where-Object { [string]$_['uses'] -match 'actions/checkout@' }) + $checkouts | Should -HaveCount 1 + [string]$checkouts[0]['with']['ref'] | Should -BeExactly '${{ github.sha }}' + [string]$checkouts[0]['with']['persist-credentials'] | Should -Match '^[Ff]alse$' + + $prepare = Get-NamedJobStep -Document $document -JobName 'sync-release-pr' ` + -StepName 'Resolve and fetch the release preparation commit' + [string]$prepare['run'] | Should -Match 'credential\.helper= fetch' + [string]$prepare['run'] | Should -Not -Match 'x-access-token' + } + + It 'Binds the preparation tree to an API-resolved immutable commit in ' -ForEach $script:SyncWorkflows { + $prepare = Get-NamedJobStep -Document (Get-WorkflowDocument -Name $Workflow) -JobName 'sync-release-pr' ` + -StepName 'Resolve and fetch the release preparation commit' + $run = [string]$prepare['run'] + $run | Should -Match 'release-please--' + $run | Should -Match 'is not a managed release preparation branch' + $run | Should -Match 'did not resolve to a commit SHA' + $run | Should -Match 'moved from .* during preparation' + $run | Should -Match 'commit=\$resolved_object' + } + + It 'Transforms the preparation tree without the write token in ' -ForEach $script:SyncWorkflows { + $transform = Get-NamedJobStep -Document (Get-WorkflowDocument -Name $Workflow) -JobName 'sync-release-pr' ` + -StepName 'Update committed version fields and retire the promotion intent' + $transformEnv = if ($transform.Contains('env')) { $transform['env'] } else { $null } + $envKeys = if ($null -ne $transformEnv) { [string[]]@($transformEnv.Keys) } else { @() } + @($envKeys | Where-Object { $_ -match 'TOKEN' }) | Should -BeNullOrEmpty + [string]$transform['run'] | Should -Not -Match 'git push' + [string]$transform['run'] | Should -Match 'committed=false' + [string]$transform['run'] | Should -Match 'committed=true' + } + + It 'Scopes the write token to a lease-guarded push in ' -ForEach $script:SyncWorkflows { + $document = Get-WorkflowDocument -Name $Workflow + $push = Get-NamedJobStep -Document $document -JobName 'sync-release-pr' ` + -StepName 'Push the synchronized release preparation commit' + [string]$push['if'] | Should -Match "steps\.transform\.outputs\.committed == 'true'" + [string]$push['env']['GH_APP_TOKEN'] | Should -BeExactly '${{ steps.app-token.outputs.token }}' + [string]$push['run'] | Should -Match '--force-with-lease="refs/heads/\$RELEASE_BRANCH:\$EXPECTED_COMMIT"' + + $tokenSteps = @($document['jobs']['sync-release-pr']['steps'] | Where-Object { + $stepEnv = if ($_.Contains('env')) { $_['env'] } else { $null } + $null -ne $stepEnv -and @($stepEnv.Keys) -contains 'GH_APP_TOKEN' + }) + $tokenSteps | Should -HaveCount 1 + } +} + Describe 'Retained provenance and SBOM assurance' -Tag 'Unit' { BeforeAll { $script:ProvenanceDocument = Get-WorkflowDocument -Name 'extension-provenance.yml' From 88c7a152c8d0e19ec1ce4ba6cee3f24ac8fd3dd9 Mon Sep 17 00:00:00 2001 From: Bill Berry Date: Sat, 22 Aug 2026 14:47:16 -0700 Subject: [PATCH 28/28] fix(workflows): fall back to origin/main when base_ref is empty Outside a pull request both github.event.pull_request.base.sha and github.base_ref are empty, so base-branch resolved to the literal 'origin/'. That value is non-empty, so it suppressed the consuming script's own origin/main default and change detection failed with an ambiguous argument error. Guard the format call so an empty base_ref falls through to origin/main. --- .github/workflows/pr-validation.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 2dd1e79dc..b2d10cc87 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -348,7 +348,9 @@ jobs: with: soft-fail: false changed-files-only: true - base-branch: ${{ github.event.pull_request.base.sha || format('origin/{0}', github.base_ref) }} + # Outside a pull request both operands are empty, so fall back to a real + # ref instead of the literal 'origin/'. + base-branch: ${{ github.event.pull_request.base.sha || (github.base_ref && format('origin/{0}', github.base_ref)) || 'origin/main' }} msdate-freshness: name: ms.date Freshness Check @@ -376,7 +378,7 @@ jobs: with: soft-fail: false changed-files-only: true - base-branch: ${{ github.event.pull_request.base.sha || format('origin/{0}', github.base_ref) }} + base-branch: ${{ github.event.pull_request.base.sha || (github.base_ref && format('origin/{0}', github.base_ref)) || 'origin/main' }} eval-validation: name: Eval Validation @@ -387,7 +389,7 @@ jobs: with: soft-fail: false changed-files-only: true - base-branch: ${{ github.event.pull_request.base.sha || format('origin/{0}', github.base_ref) }} + base-branch: ${{ github.event.pull_request.base.sha || (github.base_ref && format('origin/{0}', github.base_ref)) || 'origin/main' }} secrets: copilot-github-token: ${{ secrets.COPILOT_GITHUB_TOKEN }}