Skip to content

(#100) moved search_limit parameter to SearchParams instead of source… #105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AndreaBrg merged 1 commit into from
Nov 17, 2025
Merged

(#100) moved search_limit parameter to SearchParams instead of source… #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions configs/config.template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,54 +37,46 @@ sources:
default_authenticator: "s3"
default_downloader: "s3"
stac_url: "https://stac.dataspace.copernicus.eu/v1"
search_limit: 100

s2-l2a:
default_authenticator: "s3"
default_downloader: "s3"
default_composite: "true_color"
stac_url: "https://stac.dataspace.copernicus.eu/v1"
search_limit: 100

s2-l1c:
default_authenticator: "s3"
default_downloader: "s3"
default_composite: "true_color"
stac_url: "https://stac.dataspace.copernicus.eu/v1"
search_limit: 100

s3-slstr:
default_authenticator: "odata"
default_downloader: "http"
default_composite: "all_bands_m"
stac_url: "https://stac.dataspace.copernicus.eu/v1"
search_limit: 100

s3-olci:
default_authenticator: "odata"
default_downloader: "http"
default_composite: "all_bands"
stac_url: "https://stac.dataspace.copernicus.eu/v1"
search_limit: 100

viirs-l1b:
default_authenticator: "earthdata"
default_downloader: "http"
search_limit: 100
satellite: ["vnp", "jp1", "jp2"]
product_type: ["mod", "img"]

modis-l1b:
default_authenticator: "earthdata"
default_downloader: "http"
search_limit: 100
platform: ["mod"]
resolution: ["qkm", "hkm", "1km"]

mtg-fci-l1c:
default_authenticator: "eumetsat"
default_downloader: "http"
search_limit: 2
collection_name: "EO:EUM:DAT:0662"
reader: "fci_l1c_nc"
default_composite: "natural_color"
Expand Down
1 change: 1 addition & 0 deletions src/satctl/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ def area_geometry(self) -> Polygon | None:
class SearchParams(AreaParams):
start: datetime
end: datetime
search_limit: int | None = None

@model_validator(mode="after")
def validate_dates(self):
Expand Down
10 changes: 4 additions & 6 deletions src/satctl/sources/earthdata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@
"3": "doi",
}

DEFAULT_SEARCH_LIMIT = 100


class EarthDataAsset(BaseModel):
"""Asset model for EarthData sources."""
Expand Down Expand Up @@ -111,7 +109,6 @@ def __init__(
default_downloader: str | None = "http",
default_composite: str | None = None,
default_resolution: int | None = None,
search_limit: int = DEFAULT_SEARCH_LIMIT,
):
"""Initialize EarthData source.

Expand All @@ -126,7 +123,6 @@ def __init__(
version (str | None): Product version. Defaults to None.
default_composite (str | None): Default composite name. Defaults to None.
default_resolution (int | None): Default resolution in meters. Defaults to None.
search_limit (int): Maximum search results per query. Defaults to 100.
"""
super().__init__(
collection_name,
Expand All @@ -140,7 +136,6 @@ def __init__(
self.reader = reader
self.short_name = short_name
self.version = version
self.search_limit = search_limit
warnings.filterwarnings(action="ignore", category=UserWarning)

@abstractmethod
Expand Down Expand Up @@ -267,9 +262,12 @@ def _search_single_combination(
search_kwargs: dict[str, Any] = {
"short_name": short_name,
"temporal": (params.start.isoformat(), params.end.isoformat()),
"count": self.search_limit,
}

# Add search limit if specified (omit for unlimited results)
if params.search_limit is not None:
search_kwargs["count"] = params.search_limit

# Add version if specified
if version:
search_kwargs["version"] = version
Expand Down
8 changes: 0 additions & 8 deletions src/satctl/sources/modis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from satctl.downloaders import DownloadBuilder
from satctl.model import Granule, ProductInfo, SearchParams
from satctl.sources.earthdata import (
DEFAULT_SEARCH_LIMIT,
EarthDataSource,
ParsedGranuleId,
)
Expand Down Expand Up @@ -55,7 +54,6 @@ def __init__(
default_downloader: str = "http",
default_composite: str | None = None,
default_resolution: int | None = None,
search_limit: int = DEFAULT_SEARCH_LIMIT,
):
"""Initialize MODIS data source.

Expand All @@ -70,7 +68,6 @@ def __init__(
default_downloader (str): Default downloader name to use when down_builder is None. Defaults to "http".
default_composite (str | None): Default composite/band to load. Defaults to None.
default_resolution (int | None): Default resolution in meters. Defaults to None.
search_limit (int): Maximum number of items to return per search. Defaults to 100.
"""
super().__init__(
collection_name,
Expand All @@ -83,7 +80,6 @@ def __init__(
default_downloader=default_downloader,
default_composite=default_composite,
default_resolution=default_resolution,
search_limit=search_limit,
)

def _parse_granule_id(self, granule_id: str) -> ParsedGranuleId:
Expand Down Expand Up @@ -201,7 +197,6 @@ class MODISL1BSource(MODISSource):
downloader: HTTP downloader instance
platform: List of satellite platforms - ["mod"] (Terra), ["myd"] (Aqua)
resolution: List of resolutions - ["qkm"] (250m), ["hkm"] (500m), ["1km"] (1000m)
search_limit: Maximum number of granules to return in search results per combination

Examples:
# Single combination
Expand All @@ -222,7 +217,6 @@ def __init__(
*,
platform: list[Literal["mod", "myd"]],
resolution: list[Literal["qkm", "hkm", "1km"]],
search_limit: int = DEFAULT_SEARCH_LIMIT,
auth_builder: AuthBuilder | None = None,
down_builder: DownloadBuilder | None = None,
default_authenticator: str = "earthdata",
Expand All @@ -235,7 +229,6 @@ def __init__(
Args:
platform (list[Literal["mod", "myd"]]): List of satellite platforms to search
resolution (list[Literal["qkm", "hkm", "1km"]]): List of resolutions to search
search_limit (int): Maximum number of items to return per search. Defaults to 100.
auth_builder (AuthBuilder | None): Factory that creates an authenticator object on demand. Defaults to None.
down_builder (DownloadBuilder | None): Factory that creates a downloader object on demand. Defaults to None.
default_authenticator (str): Default authenticator name to use when auth_builder is None. Defaults to "earthdata".
Expand Down Expand Up @@ -271,7 +264,6 @@ def __init__(
default_authenticator=default_authenticator,
default_downloader=default_downloader,
version=primary["version"],
search_limit=search_limit,
)

def search(self, params: SearchParams) -> list[Granule]:
Expand Down
8 changes: 1 addition & 7 deletions src/satctl/sources/mtg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@

log = logging.getLogger(__name__)

# Constants
DEFAULT_SEARCH_LIMIT = 100


class MTGAsset(BaseModel):
href: str
Expand All @@ -46,7 +43,6 @@ def __init__(
default_downloader: str = "http",
default_composite: str | None = None,
default_resolution: int | None = None,
search_limit: int = DEFAULT_SEARCH_LIMIT,
):
"""Initialize MTG data source.

Expand All @@ -59,7 +55,6 @@ def __init__(
default_downloader (str): Default downloader name to use when down_builder is None. Defaults to "s3".
default_composite (str | None): Default composite/band to load. Defaults to None.
default_resolution (int | None): Default resolution in meters. Defaults to None.
search_limit (int): Maximum number of items to return per search. Defaults to 100.
"""
super().__init__(
collection_name,
Expand All @@ -71,7 +66,6 @@ def __init__(
default_resolution=default_resolution,
)
self.reader = reader
self.search_limit = search_limit
warnings.filterwarnings(action="ignore", category=UserWarning)

# Use synchronous dask scheduler for processing
Expand Down Expand Up @@ -140,7 +134,7 @@ def search(self, params: SearchParams) -> list[Granule]:
]
)
log.debug("Found %d items", len(items))
return items[: self.search_limit]
return items if params.search_limit is None else items[: params.search_limit]

def get_by_id(self, item_id: str, **kwargs) -> Granule:
"""Get specific MTG granule by ID.
Expand Down
8 changes: 1 addition & 7 deletions src/satctl/sources/sentinel1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ def __init__(
default_downloader: str | None = "s3",
default_composite: str | None = None,
default_resolution: int | None = None,
search_limit: int = 100,
):
"""Initialize Sentinel-1 data source.

Expand All @@ -91,7 +90,6 @@ def __init__(
default_downloader (str | None): Default downloader name to use when down_builder is None. Defaults to "s3".
default_composite (str | None): Default composite/band to load. Defaults to None.
default_resolution (int | None): Default resolution in meters. Defaults to None.
search_limit (int): Maximum number of items to return per search. Defaults to 100.
"""
super().__init__(
collection_name,
Expand All @@ -104,7 +102,6 @@ def __init__(
)
self.reader = reader
self.stac_url = stac_url
self.search_limit = search_limit

@abstractmethod
def _parse_item_name(self, name: str) -> ProductInfo:
Expand Down Expand Up @@ -149,7 +146,7 @@ def search(self, params: SearchParams) -> list[Granule]:
collections=self.collections,
intersects=params.area_geometry,
datetime=(params.start, params.end),
max_items=self.search_limit,
max_items=params.search_limit,
)

# Convert STAC items to internal Granule model
Expand Down Expand Up @@ -490,7 +487,6 @@ def __init__(
down_builder: DownloadBuilder | None = None,
default_authenticator: str | None = "s3",
default_downloader: str | None = "s3",
search_limit: int = 100,
):
"""Initialize Sentinel-1 GRD data source.

Expand All @@ -501,7 +497,6 @@ def __init__(
down_builder (DownloadBuilder | None): Factory that creates a downloader object on demand. Defaults to None.
default_authenticator (str | None): Default authenticator name to use when auth_builder is None. Defaults to "s3".
default_downloader (str | None): Default downloader name to use when down_builder is None. Defaults to "s3".
search_limit (int): Maximum number of items to return per search. Defaults to 100.
"""
super().__init__(
"sentinel-1-grd",
Expand All @@ -513,7 +508,6 @@ def __init__(
default_composite=composite,
default_resolution=20, # Native GRD resolution in IW mode
stac_url=stac_url,
search_limit=search_limit,
)

def _parse_item_name(self, name: str) -> ProductInfo:
Expand Down
10 changes: 1 addition & 9 deletions src/satctl/sources/sentinel2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@

log = logging.getLogger(__name__)

# Constants
DEFAULT_SEARCH_LIMIT = 100


class S2Asset(BaseModel):
href: str
Expand Down Expand Up @@ -67,7 +64,6 @@ def __init__(
default_downloader: str | None,
default_composite: str | None = None,
default_resolution: int | None = None,
search_limit: int = DEFAULT_SEARCH_LIMIT,
):
"""Initialize Sentinel-2 source.

Expand All @@ -81,7 +77,6 @@ def __init__(
default_downloader (str | None): Default downloader name to use when down_builder is None.
default_composite (str | None): Default composite name. Defaults to None.
default_resolution (int | None): Default resolution in meters. Defaults to None.
search_limit (int): Maximum search results. Defaults to 100.
"""
super().__init__(
collection_name,
Expand All @@ -94,7 +89,6 @@ def __init__(
)
self.reader = reader
self.stac_url = stac_url
self.search_limit = search_limit

@abstractmethod
def _parse_item_name(self, name: str) -> ProductInfo:
Expand Down Expand Up @@ -125,7 +119,7 @@ def search(self, params: SearchParams) -> list[Granule]:
collections=self.collections,
intersects=params.area_geometry,
datetime=(params.start, params.end),
max_items=self.search_limit,
max_items=params.search_limit,
)
items = [
Granule(
Expand Down Expand Up @@ -458,7 +452,6 @@ def __init__(
default_composite=default_composite,
default_resolution=default_resolution,
stac_url=stac_url,
search_limit=search_limit,
)

def _parse_item_name(self, name: str) -> ProductInfo:
Expand Down Expand Up @@ -551,7 +544,6 @@ def __init__(
default_composite=default_composite,
default_resolution=default_resolution,
stac_url=stac_url,
search_limit=search_limit,
)

def _parse_item_name(self, name: str) -> ProductInfo:
Expand Down
Loading