From 1a20236e6832da4cb3ad0b293c2d80b651cffa3f Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 6 Nov 2026 12:00:12 -0500 Subject: [PATCH 1/5] feat: improve granule type lookup --- .../src/hls_workflows/granules.py | 28 +++++++++++++ .../src/hls_workflows/lasrc/workflow.py | 41 +++++++------------ .../tests/lasrc/test_workflow.py | 19 ++++++--- src/hls-workflows/tests/test_granules.py | 24 +++++++++++ 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/src/hls-workflows/src/hls_workflows/granules.py b/src/hls-workflows/src/hls_workflows/granules.py index 9a10f3a9..582bcfe7 100644 --- a/src/hls-workflows/src/hls_workflows/granules.py +++ b/src/hls-workflows/src/hls_workflows/granules.py @@ -330,3 +330,31 @@ def to_str(self) -> str: self.version.to_str(), ] ) + + +Granule = LandsatGranule | Sentinel2Granule | HlsGranule + +#: Granule ID parsers tried, in order, by ``resolve_granule``. Each format is +#: distinct, so at most one parser accepts a given ID. +GRANULE_TYPES: tuple[type[Granule], ...] = ( + LandsatGranule, + Sentinel2Granule, + HlsGranule, +) + + +def resolve_granule(granule_id: str) -> Granule: + """Parse a granule ID with whichever registered granule type accepts it. + + Raises + ------ + ValueError + If no type in ``GRANULE_TYPES`` can parse ``granule_id``. + """ + for granule_type in GRANULE_TYPES: + try: + return granule_type.from_str(granule_id) + except ValueError: + continue + names = ", ".join(t.__name__ for t in GRANULE_TYPES) + raise ValueError(f"Could not parse {granule_id!r} as any of: {names}") diff --git a/src/hls-workflows/src/hls_workflows/lasrc/workflow.py b/src/hls-workflows/src/hls_workflows/lasrc/workflow.py index 9bc84657..936b54be 100644 --- a/src/hls-workflows/src/hls_workflows/lasrc/workflow.py +++ b/src/hls-workflows/src/hls_workflows/lasrc/workflow.py @@ -22,7 +22,7 @@ from pathlib import Path from hls_workflows.constants import LASRC_VERSION -from hls_workflows.granules import LandsatGranule, Sentinel2Granule +from hls_workflows.granules import LandsatGranule, Sentinel2Granule, resolve_granule from hls_workflows.landsat_ac import tasks as l_tasks from hls_workflows.metrics import MetricsCollector, MetricSink from hls_workflows.pipeline import Pipeline, PipelineBuilder @@ -35,27 +35,6 @@ logger = logging.getLogger(__name__) -def _is_sentinel(granule_id: str) -> bool: - """Detect Sentinel-2 vs Landsat from a granule ID. - - Tries the Sentinel-2 parser first, then the Landsat parser, raising a clear - error if neither matches. - """ - try: - Sentinel2Granule.from_str(granule_id) - return True - except ValueError: - pass - try: - LandsatGranule.from_str(granule_id) - return False - except ValueError: - pass - raise ValueError( - f"Could not identify {granule_id!r} as a Sentinel-2 or Landsat granule ID" - ) - - def construct_pipeline( granule_id: str, *, @@ -80,11 +59,19 @@ def construct_pipeline( metric_sink Optional metrics sink. """ - builder = ( - _build_sentinel(granule_id, lasrc_version, working_dir, local_granule) - if _is_sentinel(granule_id) - else _build_landsat(granule_id, lasrc_version, working_dir, local_granule) - ) + match resolve_granule(granule_id): + case Sentinel2Granule(): + builder = _build_sentinel( + granule_id, lasrc_version, working_dir, local_granule + ) + case LandsatGranule(): + builder = _build_landsat( + granule_id, lasrc_version, working_dir, local_granule + ) + case other: + raise ValueError( + f"Expected a Sentinel-2 or Landsat granule ID, got {type(other).__name__}" + ) return builder.build( metrics=MetricsCollector( diff --git a/src/hls-workflows/tests/lasrc/test_workflow.py b/src/hls-workflows/tests/lasrc/test_workflow.py index a8fcbda6..e355c7f3 100644 --- a/src/hls-workflows/tests/lasrc/test_workflow.py +++ b/src/hls-workflows/tests/lasrc/test_workflow.py @@ -6,7 +6,7 @@ import pytest from hls_workflows.lasrc import sentinel as s2_lasrc -from hls_workflows.lasrc.workflow import _is_sentinel, construct_pipeline +from hls_workflows.lasrc.workflow import construct_pipeline S2_GRANULE = "S2B_MSIL1C_20230602T155819_N0510_R097_T18SUJ_20240911T062140" L_GRANULE = "LC09_L1TP_001066_20260314_20260314_02_T1" @@ -63,11 +63,18 @@ def base_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: monkeypatch.setenv("SCRATCH_DIR", str(tmp_path / "scratch")) -def test_is_sentinel_detection() -> None: - assert _is_sentinel(S2_GRANULE) is True - assert _is_sentinel(L_GRANULE) is False - with pytest.raises(ValueError, match="Could not identify"): - _is_sentinel("not-a-granule") +@pytest.mark.parametrize( + ("granule_id", "match"), + [ + ("HLS.S30.T18TYL.2020001T153621.v2.0", "got HlsGranule"), + ("not-a-granule", "Could not parse"), + ], +) +def test_rejects_non_l1_granule_ids( + base_env: None, granule_id: str, match: str +) -> None: + with pytest.raises(ValueError, match=match): + construct_pipeline(granule_id) # ----- Rust path (the new behaviour): minimal Download -> LaSRC -> Upload diff --git a/src/hls-workflows/tests/test_granules.py b/src/hls-workflows/tests/test_granules.py index f1e4cc95..c39fa78a 100644 --- a/src/hls-workflows/tests/test_granules.py +++ b/src/hls-workflows/tests/test_granules.py @@ -6,6 +6,7 @@ HlsGranule, LandsatGranule, Sentinel2Granule, + resolve_granule, ) @@ -215,3 +216,26 @@ def test_sentinel_to_hls_granule(self) -> None: # 2020-01-01 is DOY 001. HMS is 102431 (from T102431) expected = "HLS.S30.T32TQM.2020001T102431.v2.0" assert HlsGranule.from_sentinel2("HLS", granule).to_str() == expected + + +# --- resolve_granule Tests --- +@pytest.mark.parametrize( + ("granule_id", "expected_type"), + [ + ("LC08_L1TP_046028_20200908_20200918_02_T1", LandsatGranule), + ( + "S2A_MSIL1C_20200101T102431_N0208_R065_T32TQM_20200101T122841", + Sentinel2Granule, + ), + ("HLS.S30.T18TYL.2020001T153621.v2.0", HlsGranule), + ], +) +def test_resolve_granule(granule_id: str, expected_type: type) -> None: + granule = resolve_granule(granule_id) + assert isinstance(granule, expected_type) + assert granule.to_str() == granule_id + + +def test_resolve_granule_unknown() -> None: + with pytest.raises(ValueError, match="Could not parse 'not-a-granule'"): + resolve_granule("not-a-granule") From ac8bfb5b6bcfd79dded64e6b9c6cdf717a62ffa3 Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 6 Nov 2026 12:00:18 -0500 Subject: [PATCH 2/5] fix: type hint --- src/hls-workflows/src/hls_workflows/sentinel/mapped_tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hls-workflows/src/hls_workflows/sentinel/mapped_tasks.py b/src/hls-workflows/src/hls_workflows/sentinel/mapped_tasks.py index e1548087..04c616ae 100644 --- a/src/hls-workflows/src/hls_workflows/sentinel/mapped_tasks.py +++ b/src/hls-workflows/src/hls_workflows/sentinel/mapped_tasks.py @@ -259,7 +259,10 @@ class ApplyQualityMask(MappedTask): Ports: apply_s2_quality_mask """ - requires_factory = lambda gid: (granule_dir_asset(gid), solar_valid_asset(gid)) + requires_factory: ClassVar[Callable[[str], Assets] | None] = lambda gid: ( + granule_dir_asset(gid), + solar_valid_asset(gid), + ) provides_factory = lambda gid: (quality_mask_applied_asset(gid),) def run(self, bundle: AssetBundle) -> AssetBundle: From 803803474f69850f7e4ea31e012ecb7d1b63daa4 Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 6 Nov 2026 12:03:58 -0500 Subject: [PATCH 3/5] feat: fmask standalone pipeline --- src/hls-workflows/src/hls_workflows/cli.py | 38 +++++ .../src/hls_workflows/fmask/__init__.py | 11 ++ .../src/hls_workflows/fmask/landsat.py | 63 +++++++++ .../src/hls_workflows/fmask/products.py | 17 +++ .../src/hls_workflows/fmask/sentinel.py | 76 ++++++++++ .../src/hls_workflows/fmask/workflow.py | 131 ++++++++++++++++++ src/hls-workflows/tests/fmask/__init__.py | 0 .../tests/fmask/test_upload_debug.py | 99 +++++++++++++ .../tests/fmask/test_workflow.py | 92 ++++++++++++ 9 files changed, 527 insertions(+) create mode 100644 src/hls-workflows/src/hls_workflows/fmask/__init__.py create mode 100644 src/hls-workflows/src/hls_workflows/fmask/landsat.py create mode 100644 src/hls-workflows/src/hls_workflows/fmask/products.py create mode 100644 src/hls-workflows/src/hls_workflows/fmask/sentinel.py create mode 100644 src/hls-workflows/src/hls_workflows/fmask/workflow.py create mode 100644 src/hls-workflows/tests/fmask/__init__.py create mode 100644 src/hls-workflows/tests/fmask/test_upload_debug.py create mode 100644 src/hls-workflows/tests/fmask/test_workflow.py diff --git a/src/hls-workflows/src/hls_workflows/cli.py b/src/hls-workflows/src/hls_workflows/cli.py index b47805a1..d2196313 100644 --- a/src/hls-workflows/src/hls_workflows/cli.py +++ b/src/hls-workflows/src/hls_workflows/cli.py @@ -254,5 +254,43 @@ def lasrc( ) +@cli.command("fmask") +@click.option( + "--granule", + envvar="GRANULE", + required=True, + help="Sentinel-2 or Landsat granule ID, auto-detected (env: GRANULE).", +) +@click.option( + "--local-granule", + envvar="LOCAL_GRANULE", + default=None, + type=click.Path(path_type=Path), + help="Pre-downloaded granule (.zip for S2, dir for Landsat) (env: LOCAL_GRANULE).", +) +@_cleanup_option +def fmask( + granule: str, + local_granule: Path | None, + cleanup_working_dir: bool, +) -> None: + """Run the standalone Fmask pipeline (Download -> Fmask -> Upload). + + Debug pipeline for testing Fmask performance in isolation. Uploads the + Fmask cloud mask to DEBUG_BUCKET. Does not skip granules with an invalid + solar zenith angle. + """ + from hls_workflows.fmask import construct_pipeline + + _run( + construct_pipeline( + granule_id=granule, + local_granule=local_granule, + cleanup_working_dir=cleanup_working_dir, + ), + "fmask-workflow", + ) + + if __name__ == "__main__": cli() diff --git a/src/hls-workflows/src/hls_workflows/fmask/__init__.py b/src/hls-workflows/src/hls_workflows/fmask/__init__.py new file mode 100644 index 00000000..c9befb3e --- /dev/null +++ b/src/hls-workflows/src/hls_workflows/fmask/__init__.py @@ -0,0 +1,11 @@ +"""Standalone "just-Fmask" pipeline for Fmask performance testing. + +A debug-oriented pipeline that runs only the Fmask step (Download -> Fmask -> +Upload) so changes to Fmask can be measured in isolation from the rest of the +production workflow. Unlike production, it does not skip granules with an +invalid solar zenith angle. +""" + +from .workflow import construct_pipeline + +__all__ = ["construct_pipeline"] diff --git a/src/hls-workflows/src/hls_workflows/fmask/landsat.py b/src/hls-workflows/src/hls_workflows/fmask/landsat.py new file mode 100644 index 00000000..278f6f2a --- /dev/null +++ b/src/hls-workflows/src/hls_workflows/fmask/landsat.py @@ -0,0 +1,63 @@ +"""Landsat tasks specific to the standalone "just-Fmask" pipeline. + +Kept out of the production ``landsat_ac`` package so the main L30-AC pipeline +stays clean. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import boto3 + +from hls_workflows.base import AssetBundle, Task +from hls_workflows.common import S3Path +from hls_workflows.fmask.products import is_fmask_product +from hls_workflows.landsat_ac.assets import ( + CONFIG, + FMASK_BIN, + UPLOAD_COMPLETE, + EnvConfig, +) + +if TYPE_CHECKING: + from mypy_boto3_s3 import S3Client + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class UploadFmaskDebug(Task): + """Upload the Fmask cloud mask to the debug bucket. + + Uploads only the ``*_UPL.tif`` mask, flattened into a single + ``//`` directory. Depends on the Fmask ENVI binary + purely to order after Fmask. + + No-ops (with a warning) when ``DEBUG_BUCKET`` is unset. + """ + + prefix: str + + requires = (CONFIG, FMASK_BIN) + provides = (UPLOAD_COMPLETE,) + + def run(self, inputs: AssetBundle) -> AssetBundle: + config: EnvConfig = inputs[CONFIG] + + if not config.debug_bucket: + logger.warning("DEBUG_BUCKET not set; skipping Fmask debug upload") + return {UPLOAD_COMPLETE: True} + + s3: S3Client = boto3.client("s3") + base = S3Path(config.debug_bucket, f"{self.prefix}/{config.granule}") + logger.info(f"Uploading Fmask debug files to {base}") + + for f in config.granule_dir.rglob("*"): + if f.is_file() and is_fmask_product(f.name): + dest = base / f.name + s3.upload_file(str(f), dest.bucket, dest.key) + + return {UPLOAD_COMPLETE: True} diff --git a/src/hls-workflows/src/hls_workflows/fmask/products.py b/src/hls-workflows/src/hls_workflows/fmask/products.py new file mode 100644 index 00000000..e3becbaf --- /dev/null +++ b/src/hls-workflows/src/hls_workflows/fmask/products.py @@ -0,0 +1,17 @@ +"""Helpers for the standalone Fmask pipeline's outputs. + +Kept dependency-free so both ``fmask.sentinel`` and ``fmask.landsat`` can import +it without an import cycle through ``fmask.workflow``. +""" + +from __future__ import annotations + +# Both the Landsat and Sentinel-2 ``RunFmaskV5`` tasks run the "UPL" model, and +# Fmask names its mask "_UPL.tif". The ENVI ``fmask.bin`` is a plain +# gdal_translate copy of it, so only the TIF is uploaded. +FMASK_PRODUCT_SUFFIX = "_UPL.tif" + + +def is_fmask_product(name: str) -> bool: + """True if ``name`` is the Fmask cloud mask GeoTIFF.""" + return name.endswith(FMASK_PRODUCT_SUFFIX) diff --git a/src/hls-workflows/src/hls_workflows/fmask/sentinel.py b/src/hls-workflows/src/hls_workflows/fmask/sentinel.py new file mode 100644 index 00000000..d067f758 --- /dev/null +++ b/src/hls-workflows/src/hls_workflows/fmask/sentinel.py @@ -0,0 +1,76 @@ +"""Sentinel-2 tasks specific to the standalone "just-Fmask" pipeline. + +Kept out of the production ``sentinel`` package so the main S30 pipeline stays +clean. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import boto3 + +from hls_workflows.base import AssetBundle, MappedTask +from hls_workflows.common import S3Path +from hls_workflows.fmask.products import is_fmask_product +from hls_workflows.sentinel.assets import ( + CONFIG, + UPLOAD_COMPLETE, + EnvConfig, + fmask_bin_asset, + granule_dir_asset, +) +from hls_workflows.sentinel.mapped_tasks import ApplyQualityMask + +if TYPE_CHECKING: + from mypy_boto3_s3 import S3Client + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True, kw_only=True) +class ApplyQualityMaskNoSolar(ApplyQualityMask): + """ApplyQualityMask without the solar zenith check dependency. + + The standalone Fmask pipeline runs Fmask regardless of the solar zenith + angle, so it never runs ``CheckSolarZenith``. + """ + + requires_factory = lambda gid: (granule_dir_asset(gid),) + + +@dataclass(frozen=True, kw_only=True) +class UploadFmaskDebug(MappedTask): + """Upload the Fmask cloud mask to the debug bucket. + + Uploads only the ``*_UPL.tif`` mask, flattened into a single + ``//`` directory. Depends on the Fmask ENVI binary + purely to order after Fmask. + + No-ops (with a warning) when ``DEBUG_BUCKET`` is unset. + """ + + prefix: str + + requires_factory = lambda gid: (CONFIG, fmask_bin_asset(gid)) + provides = (UPLOAD_COMPLETE,) + + def run(self, bundle: AssetBundle) -> AssetBundle: + config: EnvConfig = bundle[CONFIG] + + if not config.debug_bucket: + logger.warning("DEBUG_BUCKET not set; skipping Fmask debug upload") + return {UPLOAD_COMPLETE: True} + + s3: S3Client = boto3.client("s3") + base = S3Path(config.debug_bucket, f"{self.prefix}/{self.granule_id}") + logger.info(f"Uploading Fmask debug files to {base}") + + for f in (config.working_dir / self.granule_id).rglob("*"): + if f.is_file() and is_fmask_product(f.name): + dest = base / f.name + s3.upload_file(str(f), dest.bucket, dest.key) + + return {UPLOAD_COMPLETE: True} diff --git a/src/hls-workflows/src/hls_workflows/fmask/workflow.py b/src/hls-workflows/src/hls_workflows/fmask/workflow.py new file mode 100644 index 00000000..aca92523 --- /dev/null +++ b/src/hls-workflows/src/hls_workflows/fmask/workflow.py @@ -0,0 +1,131 @@ +"""Construct the standalone "just-Fmask" pipeline. + +One unified ``construct_pipeline`` detects whether the granule is Sentinel-2 or +Landsat and assembles the minimal chain needed to reach Fmask, reusing the +existing per-mission tasks: + +- Sentinel-2: ``Download -> GetInnerDir -> ApplyMask -> Fmask -> Upload``. The + ESA quality mask is applied first because it changes the radiances Fmask + reads in production. +- Landsat: ``Download -> Fmask -> Upload``. + +Neither path runs the solar zenith check, so Fmask runs on granules that the +production pipelines would skip. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +from hls_workflows.granules import LandsatGranule, Sentinel2Granule, resolve_granule +from hls_workflows.landsat_ac import tasks as l_tasks +from hls_workflows.metrics import MetricsCollector, MetricSink +from hls_workflows.pipeline import Pipeline, PipelineBuilder +from hls_workflows.sentinel import mapped_tasks as s2_mapped +from hls_workflows.sentinel import tasks as s2_tasks + +from . import landsat as l_fmask +from . import sentinel as s2_fmask + +logger = logging.getLogger(__name__) + +UPLOAD_PREFIX = "fmask" + + +def construct_pipeline( + granule_id: str, + *, + working_dir: Path | None = None, + local_granule: Path | None = None, + metric_sink: MetricSink | None = None, + cleanup_working_dir: bool = True, +) -> Pipeline: + """Construct the standalone Fmask pipeline for one granule. + + Parameters + ---------- + granule_id + Sentinel-2 or Landsat granule ID (auto-detected). + working_dir + Override local processing directory. + local_granule + Pre-downloaded granule path (a .zip for Sentinel-2, a directory for + Landsat). If omitted, the granule is downloaded from S3. + metric_sink + Optional metrics sink. + cleanup_working_dir + Remove the working directory when the pipeline finishes. + """ + match resolve_granule(granule_id): + case Sentinel2Granule(): + builder = _build_sentinel(granule_id, working_dir, local_granule) + case LandsatGranule(): + builder = _build_landsat(granule_id, working_dir, local_granule) + case other: + raise ValueError( + f"Expected a Sentinel-2 or Landsat granule ID, got {type(other).__name__}" + ) + + return builder.build( + metrics=MetricsCollector( + pipeline_dims={"workflow": "fmask-workflow"}, + pipeline_props={"input_granule_id": granule_id}, + sink=metric_sink, + ), + cleanup_working_dir=cleanup_working_dir, + ) + + +def _build_sentinel( + granule_id: str, + working_dir: Path | None, + local_granule: Path | None, +) -> PipelineBuilder: + """Sentinel-2 just-Fmask builder (single granule, no twin consolidation).""" + # The Sentinel EnvSource reads GRANULE_LIST from the environment. + os.environ["GRANULE_LIST"] = granule_id + + download: s2_mapped.LocalSentinelGranule | s2_mapped.DownloadSentinelGranule + if local_granule: + download = s2_mapped.LocalSentinelGranule.map(granule_id)( + "LocalGranule", local_granule_zip=local_granule + ) + else: + download = s2_mapped.DownloadSentinelGranule.map(granule_id)("Download") + + return ( + PipelineBuilder() + .add(s2_tasks.EnvSource("EnvConfig", working_dir=working_dir)) + .add(download) + .add(s2_mapped.GetGranuleDir.map(granule_id)("GetInnerDir")) + .add(s2_fmask.ApplyQualityMaskNoSolar.map(granule_id)(name="ApplyMask")) + .add(s2_mapped.RunFmaskV5.map(granule_id)("Fmask")) + .add(s2_fmask.UploadFmaskDebug.map(granule_id)("Upload", prefix=UPLOAD_PREFIX)) + ) + + +def _build_landsat( + granule_id: str, + working_dir: Path | None, + local_granule: Path | None, +) -> PipelineBuilder: + """Landsat just-Fmask builder.""" + download: l_tasks.LocalGranule | l_tasks.DownloadGranule + if local_granule: + download = l_tasks.LocalGranule("LocalGranule", local_granule_dir=local_granule) + else: + download = l_tasks.DownloadGranule("DownloadGranule") + + return ( + PipelineBuilder() + .add( + l_tasks.EnvSource( + "EnvConfig", granule_id=granule_id, working_dir=working_dir + ) + ) + .add(download) + .add(l_tasks.RunFmaskV5("Fmask")) + .add(l_fmask.UploadFmaskDebug("Upload", prefix=UPLOAD_PREFIX)) + ) diff --git a/src/hls-workflows/tests/fmask/__init__.py b/src/hls-workflows/tests/fmask/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/hls-workflows/tests/fmask/test_upload_debug.py b/src/hls-workflows/tests/fmask/test_upload_debug.py new file mode 100644 index 00000000..041420fa --- /dev/null +++ b/src/hls-workflows/tests/fmask/test_upload_debug.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from pathlib import Path + +from mypy_boto3_s3 import S3Client + +from hls_workflows.fmask import landsat as l_fmask +from hls_workflows.fmask import sentinel as s2_fmask +from hls_workflows.landsat_ac.assets import CONFIG as L_CONFIG +from hls_workflows.landsat_ac.assets import FMASK_BIN +from hls_workflows.landsat_ac.assets import EnvConfig as LandsatEnvConfig +from hls_workflows.sentinel.assets import CONFIG as S2_CONFIG +from hls_workflows.sentinel.assets import EnvConfig as SentinelEnvConfig +from hls_workflows.sentinel.assets import fmask_bin_asset + +DEBUG_BUCKET = "test-debug-bucket" +S2_GRANULE = "S2B_MSIL1C_20250116T141709_N0511_R010_T19HDS_20250116T205310" +L_GRANULE = "LC09_L1TP_001066_20260314_20260314_02_T1" +NON_PRODUCTS = ("fmask.bin", "fmask.hdr", "B01.tif", "not_a_product.txt") + + +def _keys(s3_client: S3Client, bucket: str) -> list[str]: + response = s3_client.list_objects_v2(Bucket=bucket) + return sorted(obj["Key"] for obj in response.get("Contents", [])) + + +def _touch(directory: Path, *names: str) -> None: + directory.mkdir(parents=True, exist_ok=True) + for name in names: + (directory / name).touch() + + +def test_sentinel_upload_only_mask_flattened( + s3_client: S3Client, tmp_path: Path +) -> None: + config = SentinelEnvConfig( + job_id="test-job", + granule_ids=[S2_GRANULE], + input_bucket="test-input-bucket", + output_bucket="test-output-bucket", + gibs_bucket="test-gibs-bucket", + working_dir=tmp_path / "working", + ac_code="LaSRC v3.5.1.8", + cloud_masking_code="Fmask v5.0.1", + debug_bucket=DEBUG_BUCKET, + ) + s3_client.create_bucket(Bucket=DEBUG_BUCKET) + + granule_dir = config.working_dir / S2_GRANULE + _touch(granule_dir / f"{S2_GRANULE}.SAFE", f"{S2_GRANULE}_UPL.tif", *NON_PRODUCTS) + _touch(config.working_dir / "SOME_OTHER_GRANULE", "OTHER_UPL.tif") + + task = s2_fmask.UploadFmaskDebug.map(S2_GRANULE)(name="upload", prefix="fmask") + task.run({S2_CONFIG: config, fmask_bin_asset(S2_GRANULE): Path("unused")}) + + assert _keys(s3_client, DEBUG_BUCKET) == [ + f"fmask/{S2_GRANULE}/{S2_GRANULE}_UPL.tif" + ] + + +def test_landsat_upload_only_mask_flattened( + s3_client: S3Client, tmp_path: Path +) -> None: + config = LandsatEnvConfig( + job_id="test-job", + granule=L_GRANULE, + input_bucket="test-input-bucket", + output_bucket="test-output-bucket", + ac_code="LaSRC", + cloud_masking_code="Fmask v5.0.1", + working_dir=tmp_path / "job", + granule_dir=tmp_path / "job" / L_GRANULE, + debug_bucket=DEBUG_BUCKET, + ) + s3_client.create_bucket(Bucket=DEBUG_BUCKET) + _touch(config.granule_dir, f"{L_GRANULE}_UPL.tif", *NON_PRODUCTS) + + task = l_fmask.UploadFmaskDebug("upload", prefix="fmask") + task.run({L_CONFIG: config, FMASK_BIN: Path("unused")}) + + assert _keys(s3_client, DEBUG_BUCKET) == [f"fmask/{L_GRANULE}/{L_GRANULE}_UPL.tif"] + + +def test_upload_noop_without_debug_bucket(s3_client: S3Client, tmp_path: Path) -> None: + config = LandsatEnvConfig( + job_id="test-job", + granule=L_GRANULE, + input_bucket="test-input-bucket", + output_bucket="test-output-bucket", + ac_code="LaSRC", + cloud_masking_code="Fmask v5.0.1", + working_dir=tmp_path / "job", + granule_dir=tmp_path / "job" / L_GRANULE, + debug_bucket=None, + ) + _touch(config.granule_dir, f"{L_GRANULE}_UPL.tif") + + task = l_fmask.UploadFmaskDebug("upload", prefix="fmask") + task.run({L_CONFIG: config, FMASK_BIN: Path("unused")}) diff --git a/src/hls-workflows/tests/fmask/test_workflow.py b/src/hls-workflows/tests/fmask/test_workflow.py new file mode 100644 index 00000000..7754ddfa --- /dev/null +++ b/src/hls-workflows/tests/fmask/test_workflow.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from hls_workflows.fmask import sentinel as s2_fmask +from hls_workflows.fmask.workflow import construct_pipeline + +S2_GRANULE = "S2B_MSIL1C_20230602T155819_N0510_R097_T18SUJ_20240911T062140" +L_GRANULE = "LC09_L1TP_001066_20260314_20260314_02_T1" + + +@pytest.fixture +def fmask_bins(install_mock_binaries: Callable[[dict[str, str]], Path]) -> Path: + """Binaries the download and Fmask tasks validate at construction.""" + return install_mock_binaries( + {name: "#!/bin/bash\nexit 0\n" for name in ("unzip", "fmask", "gdal_translate")} + ) + + +@pytest.fixture +def base_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("AWS_BATCH_JOB_ID", "fmask-job") + monkeypatch.setenv("INPUT_BUCKET", "in-bucket") + monkeypatch.setenv("OUTPUT_BUCKET", "out-bucket") + monkeypatch.setenv("GIBS_OUTPUT_BUCKET", "gibs-bucket") + monkeypatch.setenv("ACCODE", "LaSRC") + monkeypatch.setenv("SCRATCH_DIR", str(tmp_path / "scratch")) + + +def _classes(granule_id: str, **kwargs: object) -> list[str]: + pipeline = construct_pipeline(granule_id, **kwargs) # type: ignore[arg-type] + return [type(t).__name__ for t in pipeline.execution_order] + + +def test_sentinel_pipeline_shape(fmask_bins: Path, base_env: None) -> None: + assert _classes(S2_GRANULE) == [ + "EnvSource", + f"DownloadSentinelGranule-{S2_GRANULE}", + f"GetGranuleDir-{S2_GRANULE}", + f"ApplyQualityMaskNoSolar-{S2_GRANULE}", + f"RunFmaskV5-{S2_GRANULE}", + f"UploadFmaskDebug-{S2_GRANULE}", + ] + + +def test_landsat_pipeline_shape(fmask_bins: Path, base_env: None) -> None: + assert _classes(L_GRANULE) == [ + "EnvSource", + "DownloadGranule", + "RunFmaskV5", + "UploadFmaskDebug", + ] + + +def test_sentinel_mask_does_not_require_solar(fmask_bins: Path, base_env: None) -> None: + pipeline = construct_pipeline(S2_GRANULE) + mask = next( + t + for t in pipeline.execution_order + if isinstance(t, s2_fmask.ApplyQualityMaskNoSolar) + ) + assert not any("solar" in a.key for a in mask.requires) + + +@pytest.mark.parametrize( + ("granule_id", "match"), + [ + ("HLS.S30.T18TYL.2020001T153621.v2.0", "got HlsGranule"), + ("not-a-granule", "Could not parse"), + ], +) +def test_rejects_non_l1_granule_ids( + base_env: None, granule_id: str, match: str +) -> None: + with pytest.raises(ValueError, match=match): + construct_pipeline(granule_id) + + +@pytest.mark.parametrize("granule_id", [S2_GRANULE, L_GRANULE]) +def test_local_granule_uses_local_tasks( + fmask_bins: Path, base_env: None, tmp_path: Path, granule_id: str +) -> None: + local = tmp_path / "local_granule" + local.mkdir() + classes = _classes(granule_id, local_granule=local) + assert any(c.startswith(("LocalGranule", "LocalSentinelGranule")) for c in classes) + assert not any( + c.startswith(("DownloadGranule", "DownloadSentinel")) for c in classes + ) From cfc840e4cb49aa0027198c1e1a5a8ffce486e753 Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 6 Nov 2026 12:05:35 -0500 Subject: [PATCH 4/5] chore: 'standalone' instead of 'just' in docstrings --- src/hls-workflows/src/hls_workflows/fmask/__init__.py | 2 +- src/hls-workflows/src/hls_workflows/fmask/landsat.py | 2 +- src/hls-workflows/src/hls_workflows/fmask/sentinel.py | 2 +- src/hls-workflows/src/hls_workflows/fmask/workflow.py | 6 +++--- src/hls-workflows/src/hls_workflows/lasrc/__init__.py | 2 +- src/hls-workflows/src/hls_workflows/lasrc/landsat.py | 2 +- src/hls-workflows/src/hls_workflows/lasrc/sentinel.py | 2 +- src/hls-workflows/src/hls_workflows/lasrc/workflow.py | 6 +++--- src/hls-workflows/tests/lasrc/test_workflow.py | 6 +++--- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/hls-workflows/src/hls_workflows/fmask/__init__.py b/src/hls-workflows/src/hls_workflows/fmask/__init__.py index c9befb3e..2739e069 100644 --- a/src/hls-workflows/src/hls_workflows/fmask/__init__.py +++ b/src/hls-workflows/src/hls_workflows/fmask/__init__.py @@ -1,4 +1,4 @@ -"""Standalone "just-Fmask" pipeline for Fmask performance testing. +"""Standalone ``fmask`` pipeline for Fmask performance testing. A debug-oriented pipeline that runs only the Fmask step (Download -> Fmask -> Upload) so changes to Fmask can be measured in isolation from the rest of the diff --git a/src/hls-workflows/src/hls_workflows/fmask/landsat.py b/src/hls-workflows/src/hls_workflows/fmask/landsat.py index 278f6f2a..8cd28190 100644 --- a/src/hls-workflows/src/hls_workflows/fmask/landsat.py +++ b/src/hls-workflows/src/hls_workflows/fmask/landsat.py @@ -1,4 +1,4 @@ -"""Landsat tasks specific to the standalone "just-Fmask" pipeline. +"""Landsat tasks specific to the standalone ``fmask`` pipeline. Kept out of the production ``landsat_ac`` package so the main L30-AC pipeline stays clean. diff --git a/src/hls-workflows/src/hls_workflows/fmask/sentinel.py b/src/hls-workflows/src/hls_workflows/fmask/sentinel.py index d067f758..8814bdde 100644 --- a/src/hls-workflows/src/hls_workflows/fmask/sentinel.py +++ b/src/hls-workflows/src/hls_workflows/fmask/sentinel.py @@ -1,4 +1,4 @@ -"""Sentinel-2 tasks specific to the standalone "just-Fmask" pipeline. +"""Sentinel-2 tasks specific to the standalone ``fmask`` pipeline. Kept out of the production ``sentinel`` package so the main S30 pipeline stays clean. diff --git a/src/hls-workflows/src/hls_workflows/fmask/workflow.py b/src/hls-workflows/src/hls_workflows/fmask/workflow.py index aca92523..11c626b3 100644 --- a/src/hls-workflows/src/hls_workflows/fmask/workflow.py +++ b/src/hls-workflows/src/hls_workflows/fmask/workflow.py @@ -1,4 +1,4 @@ -"""Construct the standalone "just-Fmask" pipeline. +"""Construct the standalone ``fmask`` pipeline. One unified ``construct_pipeline`` detects whether the granule is Sentinel-2 or Landsat and assembles the minimal chain needed to reach Fmask, reusing the @@ -83,7 +83,7 @@ def _build_sentinel( working_dir: Path | None, local_granule: Path | None, ) -> PipelineBuilder: - """Sentinel-2 just-Fmask builder (single granule, no twin consolidation).""" + """Sentinel-2 ``fmask`` pipeline builder (single granule, no twin consolidation).""" # The Sentinel EnvSource reads GRANULE_LIST from the environment. os.environ["GRANULE_LIST"] = granule_id @@ -111,7 +111,7 @@ def _build_landsat( working_dir: Path | None, local_granule: Path | None, ) -> PipelineBuilder: - """Landsat just-Fmask builder.""" + """Landsat ``fmask`` pipeline builder.""" download: l_tasks.LocalGranule | l_tasks.DownloadGranule if local_granule: download = l_tasks.LocalGranule("LocalGranule", local_granule_dir=local_granule) diff --git a/src/hls-workflows/src/hls_workflows/lasrc/__init__.py b/src/hls-workflows/src/hls_workflows/lasrc/__init__.py index ac322ade..87f8e2ea 100644 --- a/src/hls-workflows/src/hls_workflows/lasrc/__init__.py +++ b/src/hls-workflows/src/hls_workflows/lasrc/__init__.py @@ -1,4 +1,4 @@ -"""Standalone "just-LaSRC" pipeline for C vs Rust intercomparison. +"""Standalone ``lasrc`` pipeline for C vs Rust intercomparison. This is a temporary, debug-oriented pipeline that runs only the LaSRC step (Download -> LaSRC -> Upload) so the C LaSRC and the Rust ``lasrc-rs`` port can diff --git a/src/hls-workflows/src/hls_workflows/lasrc/landsat.py b/src/hls-workflows/src/hls_workflows/lasrc/landsat.py index c9941670..d8bd16af 100644 --- a/src/hls-workflows/src/hls_workflows/lasrc/landsat.py +++ b/src/hls-workflows/src/hls_workflows/lasrc/landsat.py @@ -1,4 +1,4 @@ -"""Landsat tasks specific to the standalone "just-LaSRC" pipeline. +"""Landsat tasks specific to the standalone ``lasrc`` pipeline. Kept out of the production ``landsat_ac`` package so the main L30-AC pipeline stays clean; this whole module is expected to be removed once the Rust LaSRC is diff --git a/src/hls-workflows/src/hls_workflows/lasrc/sentinel.py b/src/hls-workflows/src/hls_workflows/lasrc/sentinel.py index 8f5db94d..13f37d7d 100644 --- a/src/hls-workflows/src/hls_workflows/lasrc/sentinel.py +++ b/src/hls-workflows/src/hls_workflows/lasrc/sentinel.py @@ -1,4 +1,4 @@ -"""Sentinel-2 tasks specific to the standalone "just-LaSRC" pipeline. +"""Sentinel-2 tasks specific to the standalone ``lasrc`` pipeline. Kept out of the production ``sentinel`` package so the main S30 pipeline stays clean; this whole module is expected to be removed once the Rust LaSRC is diff --git a/src/hls-workflows/src/hls_workflows/lasrc/workflow.py b/src/hls-workflows/src/hls_workflows/lasrc/workflow.py index 936b54be..0034c67d 100644 --- a/src/hls-workflows/src/hls_workflows/lasrc/workflow.py +++ b/src/hls-workflows/src/hls_workflows/lasrc/workflow.py @@ -1,4 +1,4 @@ -"""Construct the standalone "just-LaSRC" pipeline. +"""Construct the standalone ``lasrc`` pipeline. One unified ``construct_pipeline`` detects whether the granule is Sentinel-2 or Landsat and assembles the minimal chain needed to reach LaSRC, reusing the @@ -91,7 +91,7 @@ def _build_sentinel( working_dir: Path | None, local_granule: Path | None, ) -> PipelineBuilder: - """Sentinel-2 just-LaSRC builder (single granule, no twin consolidation).""" + """Sentinel-2 ``lasrc`` pipeline builder (single granule, no twin consolidation).""" # The Sentinel EnvSource reads GRANULE_LIST from the environment. os.environ["GRANULE_LIST"] = granule_id @@ -141,7 +141,7 @@ def _build_landsat( working_dir: Path | None, local_granule: Path | None, ) -> PipelineBuilder: - """Landsat just-LaSRC builder.""" + """Landsat ``lasrc`` pipeline builder.""" download: l_tasks.LocalGranule | l_tasks.DownloadGranule if local_granule: download = l_tasks.LocalGranule("LocalGranule", local_granule_dir=local_granule) diff --git a/src/hls-workflows/tests/lasrc/test_workflow.py b/src/hls-workflows/tests/lasrc/test_workflow.py index e355c7f3..469cf4a5 100644 --- a/src/hls-workflows/tests/lasrc/test_workflow.py +++ b/src/hls-workflows/tests/lasrc/test_workflow.py @@ -17,7 +17,7 @@ _DOWNLOAD_COMMANDS = ["unzip"] # The C path instantiates the real ESPA/LaSRC tasks, which each validate their -# external command in __post_init__. Note: NO Fmask commands -- the just-LaSRC C +# external command in __post_init__. Note: NO Fmask commands -- the standalone LaSRC C # path does not run Fmask. _C_PATH_COMMANDS = _DOWNLOAD_COMMANDS + [ "gdal_translate", @@ -149,7 +149,7 @@ def test_sentinel_c_pipeline_shape(c_path_bins: Path, base_env: None) -> None: assert any(t.startswith("RunLaSRC-") for t in classes) assert any(t.startswith("PrepareEspaInputNoFmask") for t in classes) assert not any(t.startswith("RunLaSRCRust") for t in classes) - # The just-LaSRC C path does not run Fmask. + # The standalone LaSRC C path does not run Fmask. assert not any(t.startswith("RunFmask") for t in classes) @@ -160,5 +160,5 @@ def test_landsat_c_pipeline_shape(c_path_bins: Path, base_env: None) -> None: assert "ConvertToEspa" in classes assert "ScanlineNoFmask" in classes assert "RunLaSRCRust" not in classes - # The just-LaSRC C path does not run Fmask. + # The standalone LaSRC C path does not run Fmask. assert not any(t.startswith("RunFmask") for t in classes) From 3cf08c4b5ed37af50d35011623ae7ae67a3354bd Mon Sep 17 00:00:00 2001 From: Chris Holden Date: Fri, 6 Nov 2026 13:21:42 -0500 Subject: [PATCH 5/5] feat: fmask output prefix per pkg version --- .../src/hls_workflows/fmask/landsat.py | 14 ++++++----- .../src/hls_workflows/fmask/products.py | 25 ++++++++++++++++--- .../src/hls_workflows/fmask/sentinel.py | 14 ++++++----- .../src/hls_workflows/lasrc/workflow.py | 4 +-- .../tests/fmask/test_upload_debug.py | 21 ++++++++++++++-- 5 files changed, 59 insertions(+), 19 deletions(-) diff --git a/src/hls-workflows/src/hls_workflows/fmask/landsat.py b/src/hls-workflows/src/hls_workflows/fmask/landsat.py index 8cd28190..81d76dbf 100644 --- a/src/hls-workflows/src/hls_workflows/fmask/landsat.py +++ b/src/hls-workflows/src/hls_workflows/fmask/landsat.py @@ -13,8 +13,7 @@ import boto3 from hls_workflows.base import AssetBundle, Task -from hls_workflows.common import S3Path -from hls_workflows.fmask.products import is_fmask_product +from hls_workflows.fmask import products from hls_workflows.landsat_ac.assets import ( CONFIG, FMASK_BIN, @@ -33,8 +32,9 @@ class UploadFmaskDebug(Task): """Upload the Fmask cloud mask to the debug bucket. Uploads only the ``*_UPL.tif`` mask, flattened into a single - ``//`` directory. Depends on the Fmask ENVI binary - purely to order after Fmask. + ``///`` directory (see + ``products.debug_upload_base``). Depends on the Fmask ENVI binary purely to + order after Fmask. No-ops (with a warning) when ``DEBUG_BUCKET`` is unset. """ @@ -52,11 +52,13 @@ def run(self, inputs: AssetBundle) -> AssetBundle: return {UPLOAD_COMPLETE: True} s3: S3Client = boto3.client("s3") - base = S3Path(config.debug_bucket, f"{self.prefix}/{config.granule}") + base = products.debug_upload_base( + config.debug_bucket, self.prefix, config.granule + ) logger.info(f"Uploading Fmask debug files to {base}") for f in config.granule_dir.rglob("*"): - if f.is_file() and is_fmask_product(f.name): + if f.is_file() and products.is_fmask_product(f.name): dest = base / f.name s3.upload_file(str(f), dest.bucket, dest.key) diff --git a/src/hls-workflows/src/hls_workflows/fmask/products.py b/src/hls-workflows/src/hls_workflows/fmask/products.py index e3becbaf..061663e5 100644 --- a/src/hls-workflows/src/hls_workflows/fmask/products.py +++ b/src/hls-workflows/src/hls_workflows/fmask/products.py @@ -1,11 +1,16 @@ -"""Helpers for the standalone Fmask pipeline's outputs. +"""Helpers for the standalone ``fmask`` pipeline's outputs. -Kept dependency-free so both ``fmask.sentinel`` and ``fmask.landsat`` can import -it without an import cycle through ``fmask.workflow``. +Kept free of imports from the rest of the ``fmask`` package so both +``fmask.sentinel`` and ``fmask.landsat`` can import it without an import cycle +through ``fmask.workflow``. """ from __future__ import annotations +from importlib.metadata import version + +from hls_workflows.common import S3Path + # Both the Landsat and Sentinel-2 ``RunFmaskV5`` tasks run the "UPL" model, and # Fmask names its mask "_UPL.tif". The ENVI ``fmask.bin`` is a plain # gdal_translate copy of it, so only the TIF is uploaded. @@ -15,3 +20,17 @@ def is_fmask_product(name: str) -> bool: """True if ``name`` is the Fmask cloud mask GeoTIFF.""" return name.endswith(FMASK_PRODUCT_SUFFIX) + + +def fmask_version() -> str: + """Installed ``fmask`` Python package version (PEP 440 normalized).""" + return version("fmask") + + +def debug_upload_base(bucket: str, prefix: str, granule_id: str) -> S3Path: + """Debug-bucket location for one granule's Fmask outputs. + + Keyed by the installed Fmask version so runs of different Fmask builds do + not overwrite each other: ``///``. + """ + return S3Path(bucket, f"{prefix}/{fmask_version()}/{granule_id}") diff --git a/src/hls-workflows/src/hls_workflows/fmask/sentinel.py b/src/hls-workflows/src/hls_workflows/fmask/sentinel.py index 8814bdde..47107a9f 100644 --- a/src/hls-workflows/src/hls_workflows/fmask/sentinel.py +++ b/src/hls-workflows/src/hls_workflows/fmask/sentinel.py @@ -13,8 +13,7 @@ import boto3 from hls_workflows.base import AssetBundle, MappedTask -from hls_workflows.common import S3Path -from hls_workflows.fmask.products import is_fmask_product +from hls_workflows.fmask import products from hls_workflows.sentinel.assets import ( CONFIG, UPLOAD_COMPLETE, @@ -46,8 +45,9 @@ class UploadFmaskDebug(MappedTask): """Upload the Fmask cloud mask to the debug bucket. Uploads only the ``*_UPL.tif`` mask, flattened into a single - ``//`` directory. Depends on the Fmask ENVI binary - purely to order after Fmask. + ``///`` directory (see + ``products.debug_upload_base``). Depends on the Fmask ENVI binary purely to + order after Fmask. No-ops (with a warning) when ``DEBUG_BUCKET`` is unset. """ @@ -65,11 +65,13 @@ def run(self, bundle: AssetBundle) -> AssetBundle: return {UPLOAD_COMPLETE: True} s3: S3Client = boto3.client("s3") - base = S3Path(config.debug_bucket, f"{self.prefix}/{self.granule_id}") + base = products.debug_upload_base( + config.debug_bucket, self.prefix, self.granule_id + ) logger.info(f"Uploading Fmask debug files to {base}") for f in (config.working_dir / self.granule_id).rglob("*"): - if f.is_file() and is_fmask_product(f.name): + if f.is_file() and products.is_fmask_product(f.name): dest = base / f.name s3.upload_file(str(f), dest.bucket, dest.key) diff --git a/src/hls-workflows/src/hls_workflows/lasrc/workflow.py b/src/hls-workflows/src/hls_workflows/lasrc/workflow.py index 0034c67d..791a878a 100644 --- a/src/hls-workflows/src/hls_workflows/lasrc/workflow.py +++ b/src/hls-workflows/src/hls_workflows/lasrc/workflow.py @@ -130,7 +130,7 @@ def _build_sentinel( ) upload = s2_lasrc.UploadLaSRCDebug.map(granule_id)( - "Upload", prefix=f"lasrc-port/{lasrc_version}" + "Upload", prefix=f"lasrc/{lasrc_version}" ) return builder.add(upload) @@ -168,5 +168,5 @@ def _build_landsat( .add(l_tasks.RunLaSRC("LaSRC")) ) - upload = l_lasrc.UploadLaSRCDebug("Upload", prefix=f"lasrc-port/{lasrc_version}") + upload = l_lasrc.UploadLaSRCDebug("Upload", prefix=f"lasrc/{lasrc_version}") return builder.add(upload) diff --git a/src/hls-workflows/tests/fmask/test_upload_debug.py b/src/hls-workflows/tests/fmask/test_upload_debug.py index 041420fa..7596db18 100644 --- a/src/hls-workflows/tests/fmask/test_upload_debug.py +++ b/src/hls-workflows/tests/fmask/test_upload_debug.py @@ -2,9 +2,11 @@ from pathlib import Path +import pytest from mypy_boto3_s3 import S3Client from hls_workflows.fmask import landsat as l_fmask +from hls_workflows.fmask import products from hls_workflows.fmask import sentinel as s2_fmask from hls_workflows.landsat_ac.assets import CONFIG as L_CONFIG from hls_workflows.landsat_ac.assets import FMASK_BIN @@ -17,6 +19,14 @@ S2_GRANULE = "S2B_MSIL1C_20250116T141709_N0511_R010_T19HDS_20250116T205310" L_GRANULE = "LC09_L1TP_001066_20260314_20260314_02_T1" NON_PRODUCTS = ("fmask.bin", "fmask.hdr", "B01.tif", "not_a_product.txt") +FMASK_VERSION = "5.0.1.post3" +_REAL_FMASK_VERSION = products.fmask_version + + +@pytest.fixture(autouse=True) +def fake_fmask_version(monkeypatch: pytest.MonkeyPatch) -> None: + """Fmask is not installed in every test env, so pin the version lookup.""" + monkeypatch.setattr(products, "fmask_version", lambda: FMASK_VERSION) def _keys(s3_client: S3Client, bucket: str) -> list[str]: @@ -54,7 +64,7 @@ def test_sentinel_upload_only_mask_flattened( task.run({S2_CONFIG: config, fmask_bin_asset(S2_GRANULE): Path("unused")}) assert _keys(s3_client, DEBUG_BUCKET) == [ - f"fmask/{S2_GRANULE}/{S2_GRANULE}_UPL.tif" + f"fmask/{FMASK_VERSION}/{S2_GRANULE}/{S2_GRANULE}_UPL.tif" ] @@ -78,7 +88,9 @@ def test_landsat_upload_only_mask_flattened( task = l_fmask.UploadFmaskDebug("upload", prefix="fmask") task.run({L_CONFIG: config, FMASK_BIN: Path("unused")}) - assert _keys(s3_client, DEBUG_BUCKET) == [f"fmask/{L_GRANULE}/{L_GRANULE}_UPL.tif"] + assert _keys(s3_client, DEBUG_BUCKET) == [ + f"fmask/{FMASK_VERSION}/{L_GRANULE}/{L_GRANULE}_UPL.tif" + ] def test_upload_noop_without_debug_bucket(s3_client: S3Client, tmp_path: Path) -> None: @@ -97,3 +109,8 @@ def test_upload_noop_without_debug_bucket(s3_client: S3Client, tmp_path: Path) - task = l_fmask.UploadFmaskDebug("upload", prefix="fmask") task.run({L_CONFIG: config, FMASK_BIN: Path("unused")}) + + +def test_fmask_version_reads_package_metadata(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(products, "version", lambda name: f"{name}-5.0.2a0") + assert _REAL_FMASK_VERSION() == "fmask-5.0.2a0"