feat: metadata validator

.github/workflows/benchmark.yml

@@ -97,3 +97,36 @@ jobs:
           comment-on-alert: true
           alert-threshold: "150%"
           fail-on-alert: false
+
+      # Re-assume the role: the benchmark run can outlast the first session.
+      - name: Configure AWS credentials for golden references (OIDC)
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_ARN }}
+          role-session-name: gha-${{ github.run_id }}-golden
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Fetch golden reference outputs
+        env:
+          GOLDEN_REFERENCE_PREFIX: ${{ vars.GOLDEN_REFERENCE_PREFIX }}
+        run: |
+          mkdir -p golden-reference
+          if [ -n "$GOLDEN_REFERENCE_PREFIX" ]; then
+            aws s3 sync --only-show-errors \
+              "s3://hls-validation-datasets/${GOLDEN_REFERENCE_PREFIX}" golden-reference/
+          else
+            echo "::warning::GOLDEN_REFERENCE_PREFIX is not set; every granule will be reported as skipped"
+          fi
+
+      - name: Compare metadata against golden references
+        env:
+          IMAGE: ${{ env.ECR_REPOSITORY }}:benchmark
+        run: scripts/golden
+
+      - name: Upload golden report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: golden-report
+          path: benchmark-output/golden-report.json
+          if-no-files-found: warn

.github/workflows/test.yml

@@ -36,6 +36,9 @@ jobs:
               - *shared
               - "src/hls-workflows/**"
               - "src/hls-utilities/**"
+            hls-metadata-validation:
+              - *shared
+              - "src/hls-metadata-validation/**"
 
       - name: List changed packages
         id: packages

docs/benchmarks.md

@@ -66,6 +66,7 @@ configured in the repository's **Settings → Secrets and variables → Actions*
 | `BENCHMARK_AUX_PREFIX`     | S3 key prefix for LaSRC ancillary data              | `lasrc_aux` |
 | `BENCHMARK_S2_GRANULE_IDS` | Comma-separated Sentinel-2 granule IDs to benchmark | see below   |
 | `BENCHMARK_LS_GRANULE_IDS` | Comma-separated Landsat granule IDs to benchmark    | see below   |
+| `GOLDEN_REFERENCE_PREFIX`  | Key prefix in `hls-validation-datasets` holding golden reference outputs (optional) | `golden/v2.0` |
 
 ### Thread pinning
 
@@ -95,6 +96,37 @@ s3://{BENCHMARK_AUX_BUCKET}/
     ...
 ```
 
+## Golden metadata comparison
+
+After the benchmarks are published, the workflow compares the metadata of the
+products the S30 benchmark produced against reference production outputs, using
+`hls-validate-metadata golden` (`src/hls-metadata-validation`). The S30
+benchmark keeps its final products (band/Fmask/angle COGs, `.cmr.xml`,
+`_stac.json`, `.jpg`) in `$BENCHMARK_OUTPUT_DIR/products/<granule>/`;
+`scripts/golden` then runs the comparison inside the benchmark image and writes
+`golden-report.json`, uploaded as the `golden-report` workflow artifact. A
+failing comparison fails the workflow after the charts are published.
+
+References are synced from `s3://hls-validation-datasets/{GOLDEN_REFERENCE_PREFIX}/`
+and must use the same layout as the products directory:
+
+```
+s3://hls-validation-datasets/{GOLDEN_REFERENCE_PREFIX}/
+  HLS.S30.T18UYA.2026027T160429.v2.0/
+    HLS.S30.T18UYA.2026027T160429.v2.0.B01.tif
+    ...
+    HLS.S30.T18UYA.2026027T160429.v2.0.cmr.xml
+    HLS.S30.T18UYA.2026027T160429.v2.0_stac.json
+```
+
+`hls-validate-metadata dump` output (`.tif.json` metadata instead of full COGs)
+is also accepted as a reference. Granules without a reference are reported as
+`reference.skipped` warnings, so the step passes until references are uploaded.
+The OIDC role needs `s3:ListBucket` and `s3:GetObject` on `hls-validation-datasets`.
+
+L30 products are not compared yet: the Landsat benchmark stops after
+atmospheric correction and does not run the tiling pipeline.
+
 ### One-time repo setup
 
 Enable GitHub Pages: **Settings → Pages → Source: `gh-pages` branch, root folder**. The first benchmark run will create

pixi.toml

@@ -15,6 +15,7 @@ hls-libs = "=v3.5.1.2"
 [pypi-dependencies]
 hls-utilities = { path = "./src/hls-utilities", editable = false }
 hls-workflows = { path = "./src/hls-workflows", editable = false}
+hls-metadata-validation = { path = "./src/hls-metadata-validation", editable = false }
 # Remote repositories
 hls-browse-imagery-creator = { git = "https://github.com/NASA-IMPACT/hls-browse_imagery", tag = "v1.9"}
 hls-cmr_stac = { git = "https://github.com/NASA-IMPACT/hls-cmr_stac", tag = "v1.9" }
@@ -67,11 +68,13 @@ py-spy = "*"
 [feature.dev.pypi-dependencies]
 hls-utilities = { path = "./src/hls-utilities", editable = true, extras = ["dev"] }
 hls-workflows = { path = "./src/hls-workflows", editable = true, extras = ["dev"] }
+hls-metadata-validation = { path = "./src/hls-metadata-validation", editable = true, extras = ["dev"] }
 # Type stubs for boto3 clients (e.g. CloudWatchLogsClient in scripts/plot_metrics.py)
 boto3-stubs = { version = ">=1", extras = ["logs"] }
 
 [feature.benchmark.pypi-dependencies]
 hls-workflows = { path = "./src/hls-workflows", editable = true, extras = ["dev"] }
+hls-metadata-validation = { path = "./src/hls-metadata-validation", editable = true }
 
 
 [environments]

pyproject.toml

@@ -14,6 +14,7 @@ ignore = ["E501", "E731"]
 known-first-party = [
   "hls_utilities",
   "hls_browse_imagery",
+  "hls_metadata_validation",
 ]
 
 [tool.mypy]

src/hls-metadata-validation/README.md

@@ -0,0 +1,38 @@
+# hls-metadata-validation
+
+Validates the metadata of final HLS products (COG GeoTIFF tags, CMR XML, STAC
+Item JSON) against a data-driven JSON spec, and compares freshly produced
+outputs against reference production outputs.
+
+Design: `docs/superpowers/specs/2026-11-12-hls-metadata-validation-design.md`.
+
+## Specs
+
+Product specs live in `src/hls_metadata_validation/specs/` (`L30.json`,
+`S30.json`, shared fragments under `common/`). A JSON object of the form
+`{"$include": "<path>"}` or `{"$include": ["<path>", ...], ...}` is replaced by
+the merged contents of the referenced files (relative to the including file);
+duplicate keys are an error so each fact lives in one place.
+
+Golden comparison: `default_tolerance` is a relative tolerance applied to every
+numeric value; an attribute's `tolerance` is an additional absolute tolerance
+(e.g. `1.0` percentage point for `CLOUD_COVERAGE`).
+
+## Usage
+
+```bash
+# Validate final outputs (COGs, .cmr.xml, _stac.json, .jpg) in a directory tree
+hls-validate-metadata spec PRODUCT_DIR [--product L30|S30] [--json report.json]
+
+# Write normalized COG metadata (.tif.json) plus CMR/STAC/thumbnail copies,
+# e.g. to create small test fixtures; `spec` accepts the dumped directory too
+hls-validate-metadata dump PRODUCT_DIR OUT_DIR
+```
+
+Exit status is 0 when there are no errors; known gaps are reported as warnings.
+
+```bash
+# Validate NEW_DIR, then compare it to reference outputs (same layout) in REF_DIR.
+# Granules with no reference are reported as skipped (warning).
+hls-validate-metadata golden NEW_DIR REF_DIR [--json report.json]
+```

src/hls-metadata-validation/pyproject.toml

@@ -0,0 +1,61 @@
+[project]
+name = "hls-metadata-validation"
+version = "0.1.0"
+description = "Validate HLS product metadata (COG tags, CMR XML, STAC) against a data-driven spec"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "click>=8.1",
+    "jsonschema>=4.20",
+    "lxml",
+    "metadata-creator @ git+https://github.com/NASA-IMPACT/hls-metadata@v2.9",
+    "pystac[validation]>=1.10,<1.12",
+    "rasterio",
+    "rio-cogeo",
+]
+
+[project.scripts]
+hls-validate-metadata = "hls_metadata_validation.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hls_metadata_validation"]
+
+[project.optional-dependencies]
+dev = [
+    "lxml-stubs>=0.5.1",
+    "mypy>=1.14.0",
+    "pytest>=9.0.2",
+    "ruff>=0.9.0",
+    "types-jsonschema",
+]
+
+[dependency-groups]
+dev = [
+    "hls-metadata-validation[dev]",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["F", "E", "W", "I", "UP"]
+ignore = ["E501", "E731"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["hls_metadata_validation"]
+
+[tool.mypy]
+python_version = "3.13"
+strict = true
+
+[[tool.mypy.overrides]]
+module = ["metadata_creator", "metadata_creator.*", "rasterio", "rasterio.*", "rio_cogeo", "rio_cogeo.*"]
+ignore_missing_imports = true

src/hls-metadata-validation/src/hls_metadata_validation/__init__.py

@@ -0,0 +1 @@
+"""Validate HLS product metadata against a data-driven spec."""

src/hls-metadata-validation/src/hls_metadata_validation/cli.py

@@ -0,0 +1,58 @@
+"""Command line interface: ``hls-validate-metadata``."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import click
+
+from hls_metadata_validation.findings import Report
+from hls_metadata_validation.golden import golden_dir
+from hls_metadata_validation.runner import dump_dir, validate_dir
+
+PRODUCTS = click.Choice(["L30", "S30"])
+DIR = click.Path(exists=True, file_okay=False, path_type=Path)
+
+
+def _finish(report: Report, json_path: Path | None) -> None:
+    click.echo(report.to_text())
+    if json_path is not None:
+        json_path.write_text(json.dumps(report.to_dict(), indent=2) + "\n")
+    raise SystemExit(0 if report.ok else 1)
+
+
+@click.group()
+def main() -> None:
+    """Validate HLS product metadata against the packaged specs."""
+
+
+@main.command()
+@click.argument("product_dir", type=DIR)
+@click.argument("out_dir", type=click.Path(file_okay=False, path_type=Path))
+def dump(product_dir: Path, out_dir: Path) -> None:
+    """Write normalized metadata of PRODUCT_DIR's outputs into OUT_DIR."""
+    written = dump_dir(product_dir, out_dir)
+    click.echo(f"wrote {len(written)} file(s) to {out_dir}")
+
+
+@main.command()
+@click.argument("product_dir", type=DIR)
+@click.option("--product", type=PRODUCTS, default=None, help="Override product type.")
+@click.option(
+    "--json", "json_path", type=click.Path(dir_okay=False, path_type=Path), default=None
+)
+def spec(product_dir: Path, product: str | None, json_path: Path | None) -> None:
+    """Validate outputs in PRODUCT_DIR against the product spec."""
+    _finish(validate_dir(product_dir, product), json_path)
+
+
+@main.command()
+@click.argument("new_dir", type=DIR)
+@click.argument("ref_dir", type=DIR)
+@click.option(
+    "--json", "json_path", type=click.Path(dir_okay=False, path_type=Path), default=None
+)
+def golden(new_dir: Path, ref_dir: Path, json_path: Path | None) -> None:
+    """Validate NEW_DIR, then compare it against reference outputs in REF_DIR."""
+    _finish(golden_dir(new_dir, ref_dir), json_path)