diff --git a/configs/config.template.yml b/configs/config.template.yml index 4a7710e..eb42b45 100644 --- a/configs/config.template.yml +++ b/configs/config.template.yml @@ -37,54 +37,46 @@ sources: default_authenticator: "s3" default_downloader: "s3" stac_url: "https://stac.dataspace.copernicus.eu/v1" - search_limit: 100 s2-l2a: default_authenticator: "s3" default_downloader: "s3" default_composite: "true_color" stac_url: "https://stac.dataspace.copernicus.eu/v1" - search_limit: 100 s2-l1c: default_authenticator: "s3" default_downloader: "s3" default_composite: "true_color" stac_url: "https://stac.dataspace.copernicus.eu/v1" - search_limit: 100 s3-slstr: default_authenticator: "odata" default_downloader: "http" default_composite: "all_bands_m" stac_url: "https://stac.dataspace.copernicus.eu/v1" - search_limit: 100 s3-olci: default_authenticator: "odata" default_downloader: "http" default_composite: "all_bands" stac_url: "https://stac.dataspace.copernicus.eu/v1" - search_limit: 100 viirs-l1b: default_authenticator: "earthdata" default_downloader: "http" - search_limit: 100 satellite: ["vnp", "jp1", "jp2"] product_type: ["mod", "img"] modis-l1b: default_authenticator: "earthdata" default_downloader: "http" - search_limit: 100 platform: ["mod"] resolution: ["qkm", "hkm", "1km"] mtg-fci-l1c: default_authenticator: "eumetsat" default_downloader: "http" - search_limit: 2 collection_name: "EO:EUM:DAT:0662" reader: "fci_l1c_nc" default_composite: "natural_color" diff --git a/src/satctl/model.py b/src/satctl/model.py index 1d17c99..c7ec7f9 100644 --- a/src/satctl/model.py +++ b/src/satctl/model.py @@ -119,6 +119,7 @@ def area_geometry(self) -> Polygon | None: class SearchParams(AreaParams): start: datetime end: datetime + search_limit: int | None = None @model_validator(mode="after") def validate_dates(self): diff --git a/src/satctl/sources/earthdata.py b/src/satctl/sources/earthdata.py index 1c0d395..c32aefa 100644 --- a/src/satctl/sources/earthdata.py +++ b/src/satctl/sources/earthdata.py @@ -25,8 +25,6 @@ "3": "doi", } -DEFAULT_SEARCH_LIMIT = 100 - class EarthDataAsset(BaseModel): """Asset model for EarthData sources.""" @@ -111,7 +109,6 @@ def __init__( default_downloader: str | None = "http", default_composite: str | None = None, default_resolution: int | None = None, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize EarthData source. @@ -126,7 +123,6 @@ def __init__( version (str | None): Product version. Defaults to None. default_composite (str | None): Default composite name. Defaults to None. default_resolution (int | None): Default resolution in meters. Defaults to None. - search_limit (int): Maximum search results per query. Defaults to 100. """ super().__init__( collection_name, @@ -140,7 +136,6 @@ def __init__( self.reader = reader self.short_name = short_name self.version = version - self.search_limit = search_limit warnings.filterwarnings(action="ignore", category=UserWarning) @abstractmethod @@ -267,9 +262,12 @@ def _search_single_combination( search_kwargs: dict[str, Any] = { "short_name": short_name, "temporal": (params.start.isoformat(), params.end.isoformat()), - "count": self.search_limit, } + # Add search limit if specified (omit for unlimited results) + if params.search_limit is not None: + search_kwargs["count"] = params.search_limit + # Add version if specified if version: search_kwargs["version"] = version diff --git a/src/satctl/sources/modis.py b/src/satctl/sources/modis.py index 8afa2b4..d0d788c 100644 --- a/src/satctl/sources/modis.py +++ b/src/satctl/sources/modis.py @@ -9,7 +9,6 @@ from satctl.downloaders import DownloadBuilder from satctl.model import Granule, ProductInfo, SearchParams from satctl.sources.earthdata import ( - DEFAULT_SEARCH_LIMIT, EarthDataSource, ParsedGranuleId, ) @@ -55,7 +54,6 @@ def __init__( default_downloader: str = "http", default_composite: str | None = None, default_resolution: int | None = None, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize MODIS data source. @@ -70,7 +68,6 @@ def __init__( default_downloader (str): Default downloader name to use when down_builder is None. Defaults to "http". default_composite (str | None): Default composite/band to load. Defaults to None. default_resolution (int | None): Default resolution in meters. Defaults to None. - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( collection_name, @@ -83,7 +80,6 @@ def __init__( default_downloader=default_downloader, default_composite=default_composite, default_resolution=default_resolution, - search_limit=search_limit, ) def _parse_granule_id(self, granule_id: str) -> ParsedGranuleId: @@ -201,7 +197,6 @@ class MODISL1BSource(MODISSource): downloader: HTTP downloader instance platform: List of satellite platforms - ["mod"] (Terra), ["myd"] (Aqua) resolution: List of resolutions - ["qkm"] (250m), ["hkm"] (500m), ["1km"] (1000m) - search_limit: Maximum number of granules to return in search results per combination Examples: # Single combination @@ -222,7 +217,6 @@ def __init__( *, platform: list[Literal["mod", "myd"]], resolution: list[Literal["qkm", "hkm", "1km"]], - search_limit: int = DEFAULT_SEARCH_LIMIT, auth_builder: AuthBuilder | None = None, down_builder: DownloadBuilder | None = None, default_authenticator: str = "earthdata", @@ -235,7 +229,6 @@ def __init__( Args: platform (list[Literal["mod", "myd"]]): List of satellite platforms to search resolution (list[Literal["qkm", "hkm", "1km"]]): List of resolutions to search - search_limit (int): Maximum number of items to return per search. Defaults to 100. auth_builder (AuthBuilder | None): Factory that creates an authenticator object on demand. Defaults to None. down_builder (DownloadBuilder | None): Factory that creates a downloader object on demand. Defaults to None. default_authenticator (str): Default authenticator name to use when auth_builder is None. Defaults to "earthdata". @@ -271,7 +264,6 @@ def __init__( default_authenticator=default_authenticator, default_downloader=default_downloader, version=primary["version"], - search_limit=search_limit, ) def search(self, params: SearchParams) -> list[Granule]: diff --git a/src/satctl/sources/mtg.py b/src/satctl/sources/mtg.py index a13a38b..6d73aff 100644 --- a/src/satctl/sources/mtg.py +++ b/src/satctl/sources/mtg.py @@ -22,9 +22,6 @@ log = logging.getLogger(__name__) -# Constants -DEFAULT_SEARCH_LIMIT = 100 - class MTGAsset(BaseModel): href: str @@ -46,7 +43,6 @@ def __init__( default_downloader: str = "http", default_composite: str | None = None, default_resolution: int | None = None, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize MTG data source. @@ -59,7 +55,6 @@ def __init__( default_downloader (str): Default downloader name to use when down_builder is None. Defaults to "s3". default_composite (str | None): Default composite/band to load. Defaults to None. default_resolution (int | None): Default resolution in meters. Defaults to None. - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( collection_name, @@ -71,7 +66,6 @@ def __init__( default_resolution=default_resolution, ) self.reader = reader - self.search_limit = search_limit warnings.filterwarnings(action="ignore", category=UserWarning) # Use synchronous dask scheduler for processing @@ -140,7 +134,7 @@ def search(self, params: SearchParams) -> list[Granule]: ] ) log.debug("Found %d items", len(items)) - return items[: self.search_limit] + return items if params.search_limit is None else items[: params.search_limit] def get_by_id(self, item_id: str, **kwargs) -> Granule: """Get specific MTG granule by ID. diff --git a/src/satctl/sources/sentinel1.py b/src/satctl/sources/sentinel1.py index 8894529..82f4893 100644 --- a/src/satctl/sources/sentinel1.py +++ b/src/satctl/sources/sentinel1.py @@ -77,7 +77,6 @@ def __init__( default_downloader: str | None = "s3", default_composite: str | None = None, default_resolution: int | None = None, - search_limit: int = 100, ): """Initialize Sentinel-1 data source. @@ -91,7 +90,6 @@ def __init__( default_downloader (str | None): Default downloader name to use when down_builder is None. Defaults to "s3". default_composite (str | None): Default composite/band to load. Defaults to None. default_resolution (int | None): Default resolution in meters. Defaults to None. - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( collection_name, @@ -104,7 +102,6 @@ def __init__( ) self.reader = reader self.stac_url = stac_url - self.search_limit = search_limit @abstractmethod def _parse_item_name(self, name: str) -> ProductInfo: @@ -149,7 +146,7 @@ def search(self, params: SearchParams) -> list[Granule]: collections=self.collections, intersects=params.area_geometry, datetime=(params.start, params.end), - max_items=self.search_limit, + max_items=params.search_limit, ) # Convert STAC items to internal Granule model @@ -490,7 +487,6 @@ def __init__( down_builder: DownloadBuilder | None = None, default_authenticator: str | None = "s3", default_downloader: str | None = "s3", - search_limit: int = 100, ): """Initialize Sentinel-1 GRD data source. @@ -501,7 +497,6 @@ def __init__( down_builder (DownloadBuilder | None): Factory that creates a downloader object on demand. Defaults to None. default_authenticator (str | None): Default authenticator name to use when auth_builder is None. Defaults to "s3". default_downloader (str | None): Default downloader name to use when down_builder is None. Defaults to "s3". - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( "sentinel-1-grd", @@ -513,7 +508,6 @@ def __init__( default_composite=composite, default_resolution=20, # Native GRD resolution in IW mode stac_url=stac_url, - search_limit=search_limit, ) def _parse_item_name(self, name: str) -> ProductInfo: diff --git a/src/satctl/sources/sentinel2.py b/src/satctl/sources/sentinel2.py index 24c9413..08dfca3 100644 --- a/src/satctl/sources/sentinel2.py +++ b/src/satctl/sources/sentinel2.py @@ -17,9 +17,6 @@ log = logging.getLogger(__name__) -# Constants -DEFAULT_SEARCH_LIMIT = 100 - class S2Asset(BaseModel): href: str @@ -67,7 +64,6 @@ def __init__( default_downloader: str | None, default_composite: str | None = None, default_resolution: int | None = None, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize Sentinel-2 source. @@ -81,7 +77,6 @@ def __init__( default_downloader (str | None): Default downloader name to use when down_builder is None. default_composite (str | None): Default composite name. Defaults to None. default_resolution (int | None): Default resolution in meters. Defaults to None. - search_limit (int): Maximum search results. Defaults to 100. """ super().__init__( collection_name, @@ -94,7 +89,6 @@ def __init__( ) self.reader = reader self.stac_url = stac_url - self.search_limit = search_limit @abstractmethod def _parse_item_name(self, name: str) -> ProductInfo: @@ -125,7 +119,7 @@ def search(self, params: SearchParams) -> list[Granule]: collections=self.collections, intersects=params.area_geometry, datetime=(params.start, params.end), - max_items=self.search_limit, + max_items=params.search_limit, ) items = [ Granule( @@ -458,7 +452,6 @@ def __init__( default_composite=default_composite, default_resolution=default_resolution, stac_url=stac_url, - search_limit=search_limit, ) def _parse_item_name(self, name: str) -> ProductInfo: @@ -551,7 +544,6 @@ def __init__( default_composite=default_composite, default_resolution=default_resolution, stac_url=stac_url, - search_limit=search_limit, ) def _parse_item_name(self, name: str) -> ProductInfo: diff --git a/src/satctl/sources/sentinel3.py b/src/satctl/sources/sentinel3.py index fa209c4..c2faf56 100644 --- a/src/satctl/sources/sentinel3.py +++ b/src/satctl/sources/sentinel3.py @@ -17,9 +17,6 @@ log = logging.getLogger(__name__) -# Constants -DEFAULT_SEARCH_LIMIT = 100 - class S3Asset(BaseModel): href: str @@ -41,7 +38,6 @@ def __init__( default_downloader: str | None, default_composite: str | None = None, default_resolution: int | None = None, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize Sentinel-3 data source. @@ -55,7 +51,6 @@ def __init__( default_downloader (str | None): Default downloader name to use when down_builder is None. default_composite (str | None): Default composite/band to load. Defaults to None. default_resolution (int | None): Default resolution in meters. Defaults to None. - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( collection_name, @@ -68,7 +63,6 @@ def __init__( ) self.reader = reader self.stac_url = stac_url - self.search_limit = search_limit @abstractmethod def _parse_item_name(self, name: str) -> ProductInfo: @@ -99,7 +93,7 @@ def search(self, params: SearchParams) -> list[Granule]: collections=self.collections, intersects=params.area_geometry, datetime=(params.start, params.end), - max_items=self.search_limit, + max_items=params.search_limit, ) items = [ Granule( @@ -278,7 +272,6 @@ def __init__( default_downloader: str | None = "http", default_composite: str = "all_bands", default_resolution: int = 1000, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize Sentinel-3 SLSTR data source. @@ -290,7 +283,6 @@ def __init__( default_downloader (str | None): Default downloader name to use when down_builder is None. Defaults to "http". default_composite (str): Default composite/band to load. Defaults to "all_bands". default_resolution (int): Default resolution in meters. Defaults to 1000. - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( "sentinel-3-sl-1-rbt-ntc", @@ -302,7 +294,6 @@ def __init__( default_composite=default_composite, default_resolution=default_resolution, stac_url=stac_url, - search_limit=search_limit, ) def _parse_item_name(self, name: str) -> ProductInfo: @@ -347,7 +338,6 @@ def __init__( default_downloader: str | None = "http", default_composite: str = "all_bands", default_resolution: int = 300, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize Sentinel-3 OLCI data source. @@ -359,7 +349,6 @@ def __init__( default_downloader (str | None): Default downloader name to use when down_builder is None. Defaults to "http". default_composite (str): Default composite/band to load. Defaults to "all_bands". default_resolution (int): Default resolution in meters. Defaults to 300. - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( "sentinel-3-olci-1-efr-ntc", @@ -371,7 +360,6 @@ def __init__( default_composite=default_composite, default_resolution=default_resolution, stac_url=stac_url, - search_limit=search_limit, ) def _parse_item_name(self, name: str) -> ProductInfo: diff --git a/src/satctl/sources/viirs.py b/src/satctl/sources/viirs.py index 308ab35..43820a1 100644 --- a/src/satctl/sources/viirs.py +++ b/src/satctl/sources/viirs.py @@ -9,7 +9,6 @@ from satctl.downloaders import DownloadBuilder from satctl.model import Granule, ProductInfo, SearchParams from satctl.sources.earthdata import ( - DEFAULT_SEARCH_LIMIT, EarthDataSource, ParsedGranuleId, ) @@ -55,7 +54,6 @@ def __init__( default_downloader: str = "http", default_composite: str | None = None, default_resolution: int | None = None, - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize VIIRS data source. @@ -70,7 +68,6 @@ def __init__( default_downloader (str): Default downloader name to use when down_builder is None. Defaults to "http". default_composite (str | None): Default composite/band to load. Defaults to None. default_resolution (int | None): Default resolution in meters. Defaults to None. - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ super().__init__( collection_name, @@ -83,7 +80,6 @@ def __init__( default_downloader=default_downloader, default_composite=default_composite, default_resolution=default_resolution, - search_limit=search_limit, ) def _parse_granule_id(self, granule_id: str) -> ParsedGranuleId: @@ -198,7 +194,6 @@ class VIIRSL1BSource(VIIRSSource): downloader: HTTP downloader instance satellite: List of satellite platforms - ["vnp"] (Suomi-NPP), ["jp1"] (NOAA-20/JPSS-1), ["jp2"] (NOAA-21/JPSS-2) product_type: List of product types - ["mod"] (M-bands, 750m), ["img"] (I-bands, 375m) - search_limit: Maximum number of granules to return in search results per combination Examples: # Single combination @@ -225,7 +220,6 @@ def __init__( default_resolution: int | None = None, satellite: list[Literal["vnp", "jp1", "jp2"]], product_type: list[Literal["mod", "img"]], - search_limit: int = DEFAULT_SEARCH_LIMIT, ): """Initialize VIIRS Level 1B data source. @@ -238,7 +232,6 @@ def __init__( default_resolution (int | None): Default resolution in meters. Defaults to None. satellite (list[Literal["vnp", "jp1", "jp2"]]): List of satellite platforms to search product_type (list[Literal["mod", "img"]]): List of product types to search - search_limit (int): Maximum number of items to return per search. Defaults to 100. """ # Generate all combinations (cartesian product) self.combinations: list[ProductCombination] = [] @@ -269,7 +262,6 @@ def __init__( default_resolution=default_resolution if default_resolution else primary["resolution"], short_name=primary["short_name"], version=primary["version"], - search_limit=search_limit, ) def search(self, params: SearchParams) -> list[Granule]: diff --git a/tests/conftest.py b/tests/conftest.py index 0d8b6bc..4a17674 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,6 +60,7 @@ def test_search_params(): path=geojson_path, start=datetime.strptime("2024-09-01", "%Y-%m-%d"), end=datetime.strptime("2024-09-04", "%Y-%m-%d"), + search_limit=1, # Limit results for testing ) @@ -83,6 +84,7 @@ def test_mtg_search_params(): path=geojson_path, start=datetime.strptime("2024-09-25", "%Y-%m-%d"), end=datetime.strptime("2024-09-26", "%Y-%m-%d"), + search_limit=1, # Limit results for testing ) diff --git a/tests/test_modis.py b/tests/test_modis.py index e3b404f..6dd8613 100644 --- a/tests/test_modis.py +++ b/tests/test_modis.py @@ -42,7 +42,6 @@ def test_auth_and_init(self) -> None: down_builder=configure_downloader("http"), platform=["mod"], # Terra satellite resolution=["1km"], # 1km resolution - search_limit=1, # Limit results for testing ) # Verify source is configured using helper diff --git a/tests/test_mtg.py b/tests/test_mtg.py index 8fbfc78..3ab2319 100644 --- a/tests/test_mtg.py +++ b/tests/test_mtg.py @@ -45,7 +45,6 @@ def test_auth_and_init(self) -> None: reader="fci_l1c_nc", # FCI Level 1C NetCDF reader default_composite="simple_fci_fire_mask", # Or whatever your default composite is default_resolution=2000, # 2km resolution for FCI - search_limit=1, # Limit results for testing ) # Verify source is configured using helper diff --git a/tests/test_olci.py b/tests/test_olci.py index 6c36214..f58f2f6 100644 --- a/tests/test_olci.py +++ b/tests/test_olci.py @@ -38,7 +38,6 @@ def test_auth_and_init(self) -> None: auth_builder=configure_authenticator("odata"), down_builder=configure_downloader("http"), stac_url="https://stac.dataspace.copernicus.eu/v1", - search_limit=1, # Limit results for testing ) # Verify source is configured using helper diff --git a/tests/test_s1.py b/tests/test_s1.py index 1c1edae..0703f30 100644 --- a/tests/test_s1.py +++ b/tests/test_s1.py @@ -44,7 +44,6 @@ def test_auth_and_init(self) -> None: down_builder=configure_downloader("s3"), stac_url="https://stac.dataspace.copernicus.eu/v1", composite="s1_dual_pol", # Or whatever your default SAR composite is - search_limit=1, # Limit results for testing ) # Verify source is configured using helper diff --git a/tests/test_s2.py b/tests/test_s2.py index 2632abb..9e8d688 100644 --- a/tests/test_s2.py +++ b/tests/test_s2.py @@ -38,7 +38,6 @@ def test_auth_and_init(self) -> None: auth_builder=configure_authenticator("s3"), down_builder=configure_downloader("s3"), stac_url="https://stac.dataspace.copernicus.eu/v1", - search_limit=1, # Limit results for testing ) # Verify source is configured using helper @@ -201,7 +200,6 @@ def test_auth_and_init(self) -> None: auth_builder=configure_authenticator("s3"), down_builder=configure_downloader("s3"), stac_url="https://stac.dataspace.copernicus.eu/v1", - search_limit=1, # Limit results for testing ) # Verify source is configured using helper diff --git a/tests/test_slstr.py b/tests/test_slstr.py index 02d7932..445efdb 100644 --- a/tests/test_slstr.py +++ b/tests/test_slstr.py @@ -41,7 +41,6 @@ def test_auth_and_init(self) -> None: auth_builder=configure_authenticator("odata"), down_builder=configure_downloader("http"), stac_url="https://stac.dataspace.copernicus.eu/v1", - search_limit=1, # Limit results for testing ) # Verify source is configured using helper diff --git a/tests/test_viirs.py b/tests/test_viirs.py index 0d3c15e..d8ea241 100644 --- a/tests/test_viirs.py +++ b/tests/test_viirs.py @@ -40,7 +40,6 @@ def test_auth_and_init(self) -> None: down_builder=configure_downloader("http"), satellite=["vnp"], # NPP satellite product_type=["mod"], # M-bands (750m) - search_limit=1, # Limit results for testing ) # Verify source is configured using helper