diff --git a/src/satctl/__init__.py b/src/satctl/__init__.py index 8cb8205..64dbb49 100644 --- a/src/satctl/__init__.py +++ b/src/satctl/__init__.py @@ -28,8 +28,6 @@ import os from importlib.resources import files -from satctl.sources import create_source, list_sources - # override the satpy config path, adding our own custom yaml configs # it is non-destructive, i.e. if the variable is already set, we append satpy_config_path = os.getenv("SATPY_CONFIG_PATH", None) @@ -40,4 +38,7 @@ satpy_config_path = str([satpy_config_path, local_config_path]) os.environ["SATPY_CONFIG_PATH"] = satpy_config_path +# need to delay the import, otherwise satpy messes with the env +from satctl.sources import create_source, list_sources # noqa: E402 + __all__ = ["create_source", "list_sources"] diff --git a/src/satctl/_config_data/satpy/readers/slstr_l1b_rev4.yaml b/src/satctl/_config_data/satpy/readers/slstr_l1b_rev4.yaml new file mode 100644 index 0000000..a3a1b92 --- /dev/null +++ b/src/satctl/_config_data/satpy/readers/slstr_l1b_rev4.yaml @@ -0,0 +1,365 @@ + +reader: + name: slstr_l1b_004 + short_name: SLSTR l1b (rev.004) + long_name: Sentinel-3 A and B SLSTR L1B, revision 4 + description: NC Reader for SLSTR data after Jan 2020 + status: Alpha + supports_fsspec: false + sensors: [slstr] + default_channels: [] + reader: !!python/name:satpy.readers.core.yaml_reader.FileYAMLReader + + data_identification_keys: + name: + required: true + wavelength: + type: !!python/name:satpy.dataset.dataid.WavelengthRange + resolution: + transitive: false + calibration: + enum: + - reflectance + - brightness_temperature + - radiance + - counts + transitive: true + view: + enum: + - nadir + - oblique + transitive: true + stripe: + enum: + - a + - b + - i + - f + modifiers: + default: [] + type: !!python/name:satpy.dataset.dataid.ModifierTuple + + coord_identification_keys: + name: + required: true + resolution: + transitive: false + view: + enum: + - nadir + - oblique + transitive: true + stripe: + enum: + - a + - b + - i + - f + +file_types: + esa_l1b_refl: + file_reader: !!python/name:satpy.readers.slstr_l1b.NCSLSTR1B + file_patterns: ['{mission_id:3s}_SL_{processing_level:1s}_{datatype_id:_<6s}_{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3/{dataset_name}_radiance_{stripe:1s}{view:1s}.nc'] + esa_l1b_tir: + file_reader: !!python/name:satpy.readers.slstr_l1b.NCSLSTR1B + file_patterns: ['{mission_id:3s}_SL_{processing_level:1s}_{datatype_id:_<6s}_{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3/{dataset_name}_BT_{stripe:1s}{view:1s}.nc'] + esa_angles: + file_reader: !!python/name:satpy.readers.slstr_l1b.NCSLSTRAngles + file_patterns: ['{mission_id:3s}_SL_{processing_level:1s}_{datatype_id:_<6s}_{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3/geometry_t{view:1s}.nc'] + esa_geo: + file_reader: !!python/name:satpy.readers.slstr_l1b.NCSLSTRGeo + file_patterns: ['{mission_id:3s}_SL_{processing_level:1s}_{datatype_id:_<6s}_{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3/geodetic_{stripe:1s}{view:1s}.nc'] + esa_l1b_flag: + file_reader: !!python/name:satpy.readers.slstr_l1b.NCSLSTRFlag + file_patterns: ['{mission_id:3s}_SL_{processing_level:1s}_{datatype_id:_<6s}_{start_time:%Y%m%dT%H%M%S}_{end_time:%Y%m%dT%H%M%S}_{creation_time:%Y%m%dT%H%M%S}_{duration:4d}_{cycle:3d}_{relative_orbit:3d}_{frame:4s}_{centre:3s}_{mode:1s}_{timeliness:2s}_{collection:3s}.SEN3/flags_{stripe:1s}{view:1s}.nc'] + +datasets: + longitude: + name: longitude + resolution: [500, 1000] + view: [nadir, oblique] + stripe: [a, b, i, f] + file_type: esa_geo + file_key: longitude_{stripe:1s}{view:1s} + standard_name: longitude + units: degree + + latitude: + name: latitude + resolution: [500, 1000] + view: [nadir, oblique] + stripe: [a, b, i, f] + file_type: esa_geo + file_key: latitude_{stripe:1s}{view:1s} + standard_name: latitude + units: degree + + elevation: + name: elevation + resolution: [500, 1000] + view: [nadir, oblique] + stripe: [a, b, i, f] + file_type: esa_geo + file_key: elevation_{stripe:1s}{view:1s} + standard_name: elevation + units: m + + # The channels S1-S3 are available in nadir (default) and oblique view. + S1: + name: S1 + sensor: slstr + wavelength: [0.545,0.555,0.565] + resolution: 500 + view: [nadir, oblique] + stripe: [a, b] + calibration: + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + coordinates: [longitude, latitude] + file_type: esa_l1b_refl + + S2: + name: S2 + sensor: slstr + wavelength: [0.649, 0.659, 0.669] + resolution: 500 + view: [nadir, oblique] + stripe: [a, b] + calibration: + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + coordinates: [longitude, latitude] + file_type: esa_l1b_refl + + S3: + name: S3 + sensor: slstr + wavelength: [0.855, 0.865, 0.875] + resolution: 500 + view: [nadir, oblique] + stripe: [a, b] + calibration: + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + coordinates: [longitude, latitude] + file_type: esa_l1b_refl + + # The channels S4-S6 are available in nadir (default) and oblique view and for both in the + # a,b and c stripes. + S4: + name: S4 + sensor: slstr + wavelength: [1.3675, 1.375, 1.36825] + resolution: 500 + view: [nadir, oblique] + stripe: [a, b] + calibration: + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + coordinates: [longitude, latitude] + file_type: esa_l1b_refl + + S5: + name: S5 + sensor: slstr + wavelength: [1.58, 1.61, 1.64] + resolution: 500 + view: [nadir, oblique] + stripe: [a, b] + calibration: + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + coordinates: [longitude, latitude] + file_type: esa_l1b_refl + + S6: + name: S6 + sensor: slstr + wavelength: [2.225, 2.25, 2.275] + resolution: 500 + view: [nadir, oblique] + stripe: [a, b] + calibration: + reflectance: + standard_name: toa_bidirectional_reflectance + units: "%" + radiance: + standard_name: toa_outgoing_radiance_per_unit_wavelength + units: W m-2 um-1 sr-1 + coordinates: [longitude, latitude] + file_type: esa_l1b_refl + + # The channels S7-S9, F1 and F2 are available in nadir (default) and oblique view. + S7: + name: S7 + sensor: slstr + wavelength: [3.55, 3.74, 3.93] + resolution: 1000 + view: [nadir, oblique] + stripe: i + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + units: "K" + coordinates: [longitude, latitude] + file_type: esa_l1b_tir + + S8: + name: S8 + sensor: slstr + wavelength: [10.4, 10.85, 11.3] + resolution: 1000 + view: [nadir, oblique] + stripe: i + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + units: "K" + coordinates: [longitude, latitude] + file_type: esa_l1b_tir + + S9: + name: S9 + sensor: slstr + wavelength: [11.57, 12.0225, 12.475] + resolution: 1000 + view: [nadir, oblique] + stripe: i + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + units: "K" + coordinates: [longitude, latitude] + file_type: esa_l1b_tir + + F1: + name: F1 + sensor: slstr + wavelength: [3.55, 3.74, 3.93] + resolution: 1000 + view: [nadir, oblique] + stripe: f + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + units: "K" + coordinates: [longitude, latitude] + file_type: esa_l1b_tir + + F2: + name: F2 + sensor: slstr + wavelength: [10.4, 10.85, 11.3] + resolution: 1000 + view: [nadir, oblique] + stripe: i + calibration: + brightness_temperature: + standard_name: toa_brightness_temperature + units: "K" + coordinates: [longitude, latitude] + file_type: esa_l1b_tir + + + solar_zenith_angle: + name: solar_zenith_angle + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + standard_name: solar_zenith_angle + file_type: esa_angles + file_key: solar_zenith_t{view:1s} + + solar_azimuth_angle: + name: solar_azimuth_angle + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + standard_name: solar_azimuth_angle + file_type: esa_angles + file_key: solar_azimuth_t{view:1s} + + satellite_zenith_angle: + name: satellite_zenith_angle + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + standard_name: satellite_zenith_angle + file_type: esa_angles + file_key: sat_zenith_t{view:1s} + + satellite_azimuth_angle: + name: satellite_azimuth_angle + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + standard_name: satellite_azimuth_angle + file_type: esa_angles + file_key: sat_azimuth_t{view:1s} + +# CloudFlags are all bitfields. They are available in nadir (default) and oblique view for +# each of the a,b,c,i stripes. + cloud: + name: cloud + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + stripe: [a, b, i, f] + file_type: esa_l1b_flag + file_key: cloud_{stripe:1s}{view:1s} + + confidence: + name: confidence + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + stripe: [a, b, i, f] + file_type: esa_l1b_flag + file_key: confidence_{stripe:1s}{view:1s} + + + pointing: + name: pointing + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + stripe: [a, b, i, f] + file_type: esa_l1b_flag + file_key: pointing_{stripe:1s}{view:1s} + + bayes: + name: bayes + sensor: slstr + resolution: [500, 1000] + coordinates: [longitude, latitude] + view: [nadir, oblique] + stripe: [a, b, i, f] + file_type: esa_l1b_flag + file_key: bayes_{stripe:1s}{view:1s} diff --git a/src/satctl/sources/base.py b/src/satctl/sources/base.py index e9eb9f1..bbafd1c 100644 --- a/src/satctl/sources/base.py +++ b/src/satctl/sources/base.py @@ -266,16 +266,18 @@ def download( def load_scene( self, item: Granule, + reader: str | None = None, datasets: list[str] | None = None, - generate: bool = False, + lazy: bool = False, **scene_options: Any, ) -> Scene: """Load a satpy Scene from granule files. Args: item (Granule): Granule to load + reader (str | None): Optional custom reader for extra customization. datasets (list[str] | None): List of datasets/composites to load. Defaults to None (uses default_composite). - generate (bool): Whether to generate composites. Defaults to False. + lazy (bool): Whether to lazily return the scene without loading datasets. Defaults to False. **scene_options (Any): Additional keyword arguments passed to Scene reader Returns: @@ -292,10 +294,11 @@ def load_scene( datasets = [self.default_composite] scene = Scene( filenames=self.get_files(item), - reader=self.reader, + reader=reader or self.reader, reader_kwargs=scene_options, ) - scene.load(datasets) + if not lazy: + scene.load(datasets) return scene def resample( diff --git a/src/satctl/sources/mtg.py b/src/satctl/sources/mtg.py index 6d73aff..80d94ee 100644 --- a/src/satctl/sources/mtg.py +++ b/src/satctl/sources/mtg.py @@ -193,38 +193,37 @@ def get_files(self, item: Granule) -> list[Path | str]: def load_scene( self, item: Granule, + reader: str | None = None, datasets: list[str] | None = None, - generate: bool = False, - **scene_options: dict[str, Any], + lazy: bool = False, + **scene_options: Any, ) -> Scene: - """Load MTG data into a Satpy Scene. + """Load a MTG scene with specified calibration. Args: item (Granule): Granule to load - datasets (list[str] | None): List of dataset names to load. Defaults to None. - generate (bool): Whether to generate composites. Defaults to False. - **scene_options (dict[str, Any]): Additional scene options + reader (str | None): Optional custom reader for extra customization. + datasets (list[str] | None): List of datasets/composites to load. Defaults to None (uses default_composite). + lazy (bool): Whether to lazily return the scene without loading datasets. Defaults to False. + **scene_options (Any): Additional keyword arguments passed to Scene reader to Scene reader Returns: - Scene: Loaded Satpy scene with requested datasets + Scene: Loaded satpy Scene object Raises: - ValueError: If datasets is None and no default composite is configured + ValueError: If datasets is None and no default_composite is set """ - if not datasets: - if self.default_composite is None: - raise ValueError( - "Invalid configuration: datasets parameter is required when no default composite is set" - ) - datasets = [self.default_composite] - scene = Scene( - filenames=self.get_files(item), - reader=self.reader, - reader_kwargs=scene_options, + scene = super().load_scene( + item, + reader=reader, + datasets=datasets, + lazy=True, + scene_options=scene_options, ) # note: the data inside the FCI files is stored upside down. # The upper_right_corner='NE' argument flips it automatically in upright position - scene.load(datasets, upper_right_corner="NE") + if not lazy: + scene.load(datasets, upper_right_corner="NE") return scene def validate(self, item: Granule) -> None: diff --git a/src/satctl/sources/sentinel1.py b/src/satctl/sources/sentinel1.py index 82f4893..a1e4599 100644 --- a/src/satctl/sources/sentinel1.py +++ b/src/satctl/sources/sentinel1.py @@ -246,51 +246,6 @@ def validate(self, item: Granule) -> None: "application/xml", ), f"Unexpected media type for asset {name}: {asset.media_type}" - def load_scene( - self, - item: Granule, - datasets: list[str] | None = None, - generate: bool = False, - calibration: str = "counts", - **scene_options: dict[str, Any], - ) -> Scene: - """Load a Sentinel-1 scene into a Satpy Scene object. - - Note: The 'calibration' parameter is currently unused but retained for - API compatibility. SAR calibration is typically specified in the dataset - name or composite definition (e.g., 'sigma_nought', 'beta_nought'). - - Args: - item: Granule to load (must have local_path set) - datasets: List of datasets/composites to load (e.g., ['measurement_vv']) - generate: Whether to generate composites (unused) - calibration: Calibration type - retained for compatibility but not used - (actual calibration is specified in dataset queries) - **scene_options: Additional options passed to Scene reader_kwargs - - Returns: - Loaded satpy Scene object with requested datasets - - Raises: - ValueError: If datasets is None and no default composite is configured - """ - if not datasets: - if self.default_composite is None: - raise ValueError("Please provide the source with a default composite, or provide custom composites") - datasets = [self.default_composite] - - # Create scene with all files in SAFE directory - scene = Scene( - filenames=self.get_files(item), - reader=self.reader, - reader_kwargs=scene_options, - ) - - # Load datasets (calibration is handled by dataset definition, not this parameter) - # TODO: Remove unused calibration parameter in future version - scene.load(datasets, calibration=calibration) - return scene - def download_item(self, item: Granule, destination: Path, downloader: Downloader) -> bool: """Download Sentinel-1 assets and reconstruct SAFE directory structure. diff --git a/src/satctl/sources/sentinel2.py b/src/satctl/sources/sentinel2.py index 08dfca3..9126509 100644 --- a/src/satctl/sources/sentinel2.py +++ b/src/satctl/sources/sentinel2.py @@ -229,19 +229,19 @@ def validate(self, item: Granule) -> None: def load_scene( self, item: Granule, + reader: str | None = None, datasets: list[str] | None = None, - generate: bool = False, - calibration: str = "counts", + lazy: bool = False, **scene_options: Any, ) -> Scene: """Load a Sentinel-2 scene with specified calibration. Args: item (Granule): Granule to load - datasets (list[str] | None): List of datasets/composites to load. Defaults to None. - generate (bool): Whether to generate composites. Defaults to False. - calibration (str): Calibration type - 'counts' (DN 0-10000) or 'reflectance' (percentage 0-100%). Defaults to 'counts'. - **scene_options (Any): Additional keyword arguments passed to Scene reader + reader (str | None): Optional custom reader for extra customization. + datasets (list[str] | None): List of datasets/composites to load. Defaults to None (uses default_composite). + lazy (bool): Whether to lazily return the scene without loading datasets. Defaults to False. + **scene_options (Any): Additional keyword arguments passed to Scene reader to Scene reader Returns: Scene: Loaded satpy Scene object @@ -249,19 +249,16 @@ def load_scene( Raises: ValueError: If datasets is None and no default_composite is set """ - if not datasets: - if self.default_composite is None: - raise ValueError( - "Invalid configuration: datasets parameter is required when no default composite is set" - ) - datasets = [self.default_composite] - scene = Scene( - filenames=self.get_files(item), - reader=self.reader, - reader_kwargs=scene_options, + scene = super().load_scene( + item, + reader=reader, + datasets=datasets, + lazy=True, + scene_options=scene_options, ) # Load with specified calibration - scene.load(datasets, calibration=calibration) + if not lazy: + scene.load(datasets, calibration="counts") return scene def download_item(self, item: Granule, destination: Path, downloader: Downloader) -> bool: diff --git a/src/satctl/sources/sentinel3.py b/src/satctl/sources/sentinel3.py index c2faf56..c5a4bb8 100644 --- a/src/satctl/sources/sentinel3.py +++ b/src/satctl/sources/sentinel3.py @@ -232,18 +232,21 @@ def save_item( Returns: dict[str, list]: Dictionary mapping granule_id to list of output paths """ - # Validate inputs using base class helper self._validate_save_inputs(item, params) - - # Parse datasets using base class helper datasets_dict = self._prepare_datasets(writer, params) - - # Filter existing files using base class helper datasets_dict = self._filter_existing_files(datasets_dict, destination, item.granule_id, writer, force) # Load and resample scene log.debug("Loading and resampling scene") - scene = self.load_scene(item, datasets=list(datasets_dict.values())) + + # workaround patch to fix broker SLSTR reader + # see https://github.com/pytroll/satpy/issues/3251 + # TLDR: SLSTR revision 004 switches band F1 from stripe i to f + # current satpy reader does not allow for missing files + custom_reader = None + if item.info.instrument == "slstr" and item.granule_id.endswith("004"): + custom_reader = f"{self.reader}_rev4" + scene = self.load_scene(item, reader=custom_reader, datasets=list(datasets_dict.values())) # Define area using base class helper area_def = self.define_area(