diff --git a/.github/workflows/internal-tests.yaml b/.github/workflows/internal-tests.yaml index 87ed820..cd5f8f4 100644 --- a/.github/workflows/internal-tests.yaml +++ b/.github/workflows/internal-tests.yaml @@ -17,6 +17,8 @@ jobs: steps: - name: Clone repo uses: actions/checkout@v2 + with: + submodules: recursive - name: Set up python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/pytest-collect.yaml b/.github/workflows/pytest-collect.yaml index 39a6838..8fdfc44 100644 --- a/.github/workflows/pytest-collect.yaml +++ b/.github/workflows/pytest-collect.yaml @@ -13,6 +13,8 @@ jobs: steps: - name: Clone repo uses: actions/checkout@v2 + with: + submodules: recursive - name: Set up python uses: actions/setup-python@v4 with: diff --git a/.gitignore b/.gitignore index 7a0f22a..4f4d1b8 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,7 @@ venv/ ENV/ env.bak/ venv.bak/ +venv-*/ # Spyder project settings .spyderproject diff --git a/README.md b/README.md index a9f06eb..b6ea11e 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,12 @@ process selection options: - Example: `--process-levels=L1,L2`.` - A level does not imply other levels, so each desired level must be specified explicitly. For example, L2 does **not** include L1 automatically. -- `--experimental`: Enables tests for experimental processes. - By default experimental processes will be skipped. + +If neither `--processes` nor `--process-levels` are specified, all processes are considered. +If both are specified, the union of both will be considered. + +- `--experimental`: By default, experimental processes (or experimental process tests) are ignored. + Enabling this option will consider experimental processes and tests. ### Runner for individual process testing diff --git a/src/openeo_test_suite/lib/internal-tests/__init__.py b/src/openeo_test_suite/lib/internal-tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/openeo_test_suite/lib/internal-tests/test_process_registry.py b/src/openeo_test_suite/lib/internal-tests/test_process_registry.py new file mode 100644 index 0000000..3ced751 --- /dev/null +++ b/src/openeo_test_suite/lib/internal-tests/test_process_registry.py @@ -0,0 +1,122 @@ +import pytest + +from openeo_test_suite.lib.process_registry import ProcessRegistry + + +class TestProcessRegistry: + # Some example processes for some levels + PROCESS_EXAMPLES_L1 = ["add", "divide", "apply_dimension", "reduce_dimension"] + PROCESS_EXAMPLES_L2 = ["aggregate_temporal", "if"] + PROCESS_EXAMPLES_L3 = ["apply_neighborhood", "merge_cubes"] + PROCESS_EXAMPLES_EXPERIMENTAL = ["apply_polygon"] + + @pytest.fixture(scope="class") + def process_registry(self) -> ProcessRegistry: + return ProcessRegistry() + + def test_get_all_processes_basic(self, process_registry): + processes = list(process_registry.get_all_processes()) + assert len(processes) > 0 + + def test_get_all_processes_add(self, process_registry): + (add,) = [ + p for p in process_registry.get_all_processes() if p.process_id == "add" + ] + + assert add.level == "L1" + assert add.experimental is False + assert add.path.name == "add.json5" + assert len(add.tests) + + add00 = {"arguments": {"x": 0, "y": 0}, "returns": 0} + assert add00 in add.tests + + def test_get_all_processes_divide(self, process_registry): + (divide,) = [ + p for p in process_registry.get_all_processes() if p.process_id == "divide" + ] + + assert divide.level == "L1" + assert divide.experimental is False + assert divide.path.name == "divide.json5" + assert len(divide.tests) + + divide0 = { + "arguments": {"x": 1, "y": 0}, + "returns": float("inf"), + "throws": "DivisionByZero", + } + assert divide0 in divide.tests + + def test_get_processes_filtered_default(self, process_registry): + pids = [p.process_id for p in process_registry.get_processes_filtered()] + assert len(pids) > 100 + for pid in ( + self.PROCESS_EXAMPLES_L1 + + self.PROCESS_EXAMPLES_L2 + + self.PROCESS_EXAMPLES_L3 + ): + assert pid in pids + for pid in self.PROCESS_EXAMPLES_EXPERIMENTAL: + assert pid not in pids + + def test_get_processes_filtered_with_process_ids(self, process_registry): + pids = [ + p.process_id + for p in process_registry.get_processes_filtered( + process_ids=["add", "divide"] + ) + ] + assert sorted(pids) == ["add", "divide"] + + def test_get_processes_filtered_with_process_levels(self, process_registry): + pids_l1 = [ + p.process_id + for p in process_registry.get_processes_filtered(process_levels=["L1"]) + ] + pids_l23 = [ + p.process_id + for p in process_registry.get_processes_filtered( + process_levels=["L2", "L3"] + ) + ] + for pid in self.PROCESS_EXAMPLES_L1: + assert pid in pids_l1 + assert pid not in pids_l23 + for pid in self.PROCESS_EXAMPLES_L2: + assert pid not in pids_l1 + assert pid in pids_l23 + for pid in self.PROCESS_EXAMPLES_L3: + assert pid not in pids_l1 + assert pid in pids_l23 + for pid in self.PROCESS_EXAMPLES_EXPERIMENTAL: + assert pid not in pids_l1 + assert pid not in pids_l23 + + def test_get_processes_filtered_with_process_ids_and_levels(self, process_registry): + pids = [ + p.process_id + for p in process_registry.get_processes_filtered( + process_ids=["min", "max"], process_levels=["L2"] + ) + ] + for pid in ["min", "max"] + self.PROCESS_EXAMPLES_L2: + assert pid in pids + for pid in ( + self.PROCESS_EXAMPLES_L1 + + self.PROCESS_EXAMPLES_L3 + + self.PROCESS_EXAMPLES_EXPERIMENTAL + ): + assert pid not in pids + + def test_get_processes_filtered_with_experimental(self, process_registry): + pids = [ + p.process_id + for p in process_registry.get_processes_filtered( + process_ids=["min", "max"], process_levels=["L3"], experimental=True + ) + ] + for pid in ["min", "max"] + self.PROCESS_EXAMPLES_L3: + assert pid in pids + for pid in self.PROCESS_EXAMPLES_EXPERIMENTAL: + assert pid in pids diff --git a/src/openeo_test_suite/lib/internal-tests/test_process_selection.py b/src/openeo_test_suite/lib/internal-tests/test_process_selection.py new file mode 100644 index 0000000..25d30f2 --- /dev/null +++ b/src/openeo_test_suite/lib/internal-tests/test_process_selection.py @@ -0,0 +1,20 @@ +from openeo_test_suite.lib.process_selection import csv_to_list + + +def test_csv_to_list(): + assert csv_to_list() == [] + assert csv_to_list("") == [] + assert csv_to_list(" ") == [] + assert csv_to_list(" , ") == [] + assert csv_to_list("foo") == ["foo"] + assert csv_to_list("foo,bar,baz") == ["foo", "bar", "baz"] + assert csv_to_list(",foo,bar,baz,") == ["foo", "bar", "baz"] + assert csv_to_list(" ,foo , bar, baz , ") == ["foo", "bar", "baz"] + assert csv_to_list(" ,foo ,,, bar, , baz , ") == ["foo", "bar", "baz"] + + +def test_csv_to_list_none_on_empty(): + assert csv_to_list(none_on_empty=True) is None + assert csv_to_list("", none_on_empty=True) is None + assert csv_to_list(" ", none_on_empty=True) is None + assert csv_to_list(" , ", none_on_empty=True) is None diff --git a/src/openeo_test_suite/lib/internal-tests/test_skipping.py b/src/openeo_test_suite/lib/internal-tests/test_skipping.py new file mode 100644 index 0000000..c3ad118 --- /dev/null +++ b/src/openeo_test_suite/lib/internal-tests/test_skipping.py @@ -0,0 +1,44 @@ +import openeo +import pytest +from openeo import DataCube + +from openeo_test_suite.lib.skipping import extract_processes_from_process_graph + + +def test_extract_processes_from_process_graph_basic(): + pg = {"add35": {"process_id": "add", "arguments": {"x": 3, "y": 5}, "result": True}} + assert extract_processes_from_process_graph(pg) == {"add"} + + +@pytest.fixture +def s2_cube() -> openeo.DataCube: + return openeo.DataCube.load_collection( + collection_id="S2", bands=["B02", "B03"], connection=None, fetch_metadata=False + ) + + +def test_extract_processes_from_process_graph_cube_simple(s2_cube): + assert extract_processes_from_process_graph(s2_cube) == {"load_collection"} + + +def test_extract_processes_from_process_graph_cube_reduce_temporal(s2_cube): + cube = s2_cube.reduce_temporal("mean") + assert extract_processes_from_process_graph(cube) == { + "load_collection", + "reduce_dimension", + "mean", + } + + +def test_extract_processes_from_process_graph_cube_reduce_bands(s2_cube): + b2 = s2_cube.band("B02") + b3 = s2_cube.band("B03") + cube = (b3 - b2) / (b3 + b2) + assert extract_processes_from_process_graph(cube) == { + "load_collection", + "reduce_dimension", + "array_element", + "subtract", + "add", + "divide", + } diff --git a/src/openeo_test_suite/lib/process_registry.py b/src/openeo_test_suite/lib/process_registry.py new file mode 100644 index 0000000..b5fb24f --- /dev/null +++ b/src/openeo_test_suite/lib/process_registry.py @@ -0,0 +1,112 @@ +import logging +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Iterator, List, Optional, Union + +import json5 + +import openeo_test_suite + +_log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ProcessData: + """Process data, including profile level and list of tests""" + + process_id: str + level: str + tests: List[dict] # TODO: also make dataclass for each test? + experimental: bool + path: Path + + +class ProcessRegistry: + """ + Registry of processes, metadata (level, experimental flag) + and related tests defined in openeo-processes project + """ + + def __init__(self, root: Optional[Path] = None): + """ + :param root: Root directory of the tests folder in openeo-processes project + """ + self._root = Path( + root + # TODO: eliminate need for this env var? + or os.environ.get("OPENEO_TEST_SUITE_PROCESSES_TEST_ROOT") + or self._guess_root() + ) + # Lazy load cache + self._processes: Union[None, List[ProcessData]] = None + + def _guess_root(self): + # TODO: avoid need for guessing and properly include assets in (installed) package + project_root = Path(openeo_test_suite.__file__).parents[2] + candidates = [ + project_root / "assets/processes/tests", + Path("./assets/processes/tests"), + Path("./openeo-test-suite/assets/processes/tests"), + ] + for candidate in candidates: + if candidate.exists() and candidate.is_dir(): + return candidate + raise ValueError( + f"Could not find valid processes test root directory (tried {candidates})" + ) + + def _load(self) -> Iterator[ProcessData]: + """Collect all processes""" + # TODO: cache or preload this in __init__? Or even reuse across instances? + if not self._root.is_dir(): + raise ValueError(f"Invalid process test root directory: {self._root}") + _log.info(f"Loading process definitions from {self._root}") + for path in self._root.glob("*.json5"): + try: + with path.open() as f: + data = json5.load(f) + assert data["id"] == path.stem + yield ProcessData( + process_id=data["id"], + level=data.get("level"), + tests=data.get("tests", []), + experimental=data.get("experimental", False), + path=path, + ) + except Exception as e: + # TODO: good idea to skip broken definitions? Why not just fail hard? + _log.error(f"Failed to load process data from {path}: {e!r}") + + def get_all_processes(self) -> Iterable[ProcessData]: + if self._processes is None: + self._processes = list(self._load()) + return iter(self._processes) + + def get_processes_filtered( + self, + process_ids: Optional[List[str]] = None, + process_levels: Optional[List[str]] = None, + experimental: bool = False, + ) -> Iterable[ProcessData]: + """ + Collect processes matching with additional filtering: + + :param process_ids: allow list of process ids (empty/None means allow all) + :param process_levels: allow list of process levels (empty/None means allow all) + :param experimental: allow experimental processes or not? + """ + for process_data in self.get_all_processes(): + pid = process_data.process_id + level = process_data.level + + if process_data.experimental and not experimental: + continue + + if process_ids and pid in process_ids: + yield process_data + elif process_levels and level in process_levels: + yield process_data + elif not process_ids and not process_levels: + # No id or level allow lists: no filtering + yield process_data diff --git a/src/openeo_test_suite/lib/process_selection.py b/src/openeo_test_suite/lib/process_selection.py new file mode 100644 index 0000000..e744a68 --- /dev/null +++ b/src/openeo_test_suite/lib/process_selection.py @@ -0,0 +1,72 @@ +import functools +import logging +from dataclasses import dataclass +from typing import Iterable, List, Optional, Union + +import pytest + +from openeo_test_suite.lib.process_registry import ProcessData, ProcessRegistry + +_log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ProcessFilters: + """ + Container for process filters as specified through command line options + `--processes`,`--process-level`, `--experimental` + """ + + process_ids: Optional[List[str]] = None + process_levels: Optional[List[str]] = None + experimental: bool = False + + +# Internal singleton pointing to active set of process filters +# setup happens in `pytest_configure` hook +_process_filters: Union[ProcessFilters, None] = None + + +def set_process_selection_from_config(config: pytest.Config): + """Set up process selection from pytest config (CLI options).""" + global _process_filters + assert _process_filters is None + _process_filters = ProcessFilters( + process_ids=csv_to_list(config.getoption("--processes"), none_on_empty=True), + process_levels=csv_to_list( + config.getoption("--process-levels"), none_on_empty=True + ), + experimental=config.getoption("--experimental"), + ) + + +# TODO: more structural/testable solution for get_selected_processes related caching? +@functools.lru_cache() +def get_selected_processes() -> Iterable[ProcessData]: + """ + Get effective list of processes extracted from the process registry + with filtering based on command line options + `--processes`,`--process-level`, `--experimental` + """ + global _process_filters + assert isinstance(_process_filters, ProcessFilters) + + return ProcessRegistry().get_processes_filtered( + process_ids=_process_filters.process_ids, + process_levels=_process_filters.process_levels, + experimental=_process_filters.experimental, + ) + + +def csv_to_list( + csv: Union[str, None] = None, *, separator: str = ",", none_on_empty: bool = False +) -> Union[List[str], None]: + """ + Convert comma-separated string to list of strings, + properly taking care of trailing whitespace, empty items, ... + """ + # TODO: options to disable stripping, or to allow empty items? + items = [item.strip() for item in (csv or "").split(separator) if item.strip()] + if not items and none_on_empty: + return None + return items diff --git a/src/openeo_test_suite/lib/pytest_plugin.py b/src/openeo_test_suite/lib/pytest_plugin.py index 3d414a9..0a3f5dd 100644 --- a/src/openeo_test_suite/lib/pytest_plugin.py +++ b/src/openeo_test_suite/lib/pytest_plugin.py @@ -8,6 +8,7 @@ get_backend_url, set_backend_under_test, ) +from openeo_test_suite.lib.process_selection import set_process_selection_from_config def pytest_addoption(parser): @@ -24,13 +25,17 @@ def pytest_addoption(parser): "--process-levels", action="store", default="", - help="The openEO process profiles you want to test against, e.g. 'L1,L2,L2A'. Mutually exclusive with --processes.", + help="openEO process selection: " + "the openEO process profiles/levels you want to test against, e.g. 'L1,L2,L2A'. " + "Can be used in combination with `--processes`, in which case the union of both selections will be taken. ", ) parser.addoption( "--processes", action="store", default="", - help="The openEO processes you want to test against, e.g. 'apply,reduce_dimension'. Mutually exclusive with --process-levels.", + help="openEO process selection: " + "the openEO processes you want to test against, e.g. 'apply,reduce_dimension'. " + "Can be used in combination with `--process-levels`, in which case the union of both selections will be taken.", ) parser.addoption( @@ -38,7 +43,9 @@ def pytest_addoption(parser): type=bool, action=argparse.BooleanOptionalAction, default=False, - help="Run tests for experimental functionality or not. By default the tests will be skipped.", + help="openEO process selection: " + "toggle to consider experimental processes (and experimental tests) in the test selection procedure. " + "By default, experimental processes are ignored.", ) parser.addoption( @@ -65,3 +72,5 @@ def pytest_configure(config): connection = openeo.connect(url=backend_url, auto_validate=False) backend = HttpBackend(connection=connection) set_backend_under_test(backend) + + set_process_selection_from_config(config) diff --git a/src/openeo_test_suite/lib/skipping.py b/src/openeo_test_suite/lib/skipping.py index 214f2ab..97d5bbe 100644 --- a/src/openeo_test_suite/lib/skipping.py +++ b/src/openeo_test_suite/lib/skipping.py @@ -1,8 +1,9 @@ import logging -from typing import Iterable, List, Union +from typing import Iterable, Iterator, List, Set, Union import openeo import pytest +from openeo.internal.graph_building import FlatGraphableMixin, as_flat_graph _log = logging.getLogger(__name__) @@ -13,9 +14,16 @@ class Skipper: backend capabilities and configuration. """ - def __init__(self, connection: openeo.Connection, process_levels: List[str]): + def __init__( + self, connection: openeo.Connection, selected_processes: Iterable[str] + ): + """ + :param connection: openeo connection + :param selected_processes: list of active process selection + """ self._connection = connection - self._process_levels = process_levels + + self._selected_processes = set(selected_processes) def _get_output_formats(self) -> set: formats = set( @@ -34,24 +42,67 @@ def skip_if_no_geotiff_support(self): if not output_formats.intersection({"geotiff", "gtiff"}): pytest.skip("GeoTIFF not supported as output file format") - def skip_if_unmatching_process_level(self, level: str): - """Skip test if "process_levels" are set and do not match the given level.""" - if len(self._process_levels) > 0 and level not in self._process_levels: - pytest.skip( - f"Skipping {level} test because the specified levels are: {self._process_levels}" - ) + def _get_processes( + self, processes: Union[str, List[str], Set[str], openeo.DataCube] + ) -> Set[str]: + """ + Generic process id extraction from: + - string (single process id) + - list/set of process ids + - openeo.DataCube: extract process ids from process graph + """ + if isinstance(processes, str): + return {processes} + elif isinstance(processes, (list, set)): + return set(processes) + elif isinstance(processes, openeo.DataCube): + # TODO: wider isinstance check? + return extract_processes_from_process_graph(processes) + else: + raise ValueError(processes) + + def skip_if_unselected_process( + self, processes: Union[str, List[str], Set[str], openeo.DataCube] + ): + """ + Skip test if any of the provided processes is not in the active process selection. + + :param processes: single process id, list/set of process ids or an `openeo.DataCube` to extract process ids from + """ + # TODO: automatically call this skipper from monkey-patched `cube.download()`? + processes = self._get_processes(processes) + unselected_processes = processes.difference(self._selected_processes) + if unselected_processes: + pytest.skip(f"Process selection does not cover: {unselected_processes}") - def skip_if_unsupported_process(self, processes: Union[str, Iterable[str]]): + def skip_if_unsupported_process( + self, processes: Union[str, List[str], Set[str], openeo.DataCube] + ): """ Skip test if any of the provided processes is not supported by the backend. - @param processes: single process id or list of process ids + :param processes: single process id, list/set of process ids or an `openeo.DataCube` to extract process ids from """ - if isinstance(processes, str): - processes = [processes] + processes = self._get_processes(processes) + + # TODO: cache available processes? available_processes = set(p["id"] for p in self._connection.list_processes()) - unsupported_processes = set(processes).difference(available_processes) + unsupported_processes = processes.difference(available_processes) if unsupported_processes: - pytest.skip( - f"Skipping test because backend does not support: {unsupported_processes}" - ) + pytest.skip(f"Backend does not support: {unsupported_processes}") + + +def extract_processes_from_process_graph( + pg: Union[dict, FlatGraphableMixin] +) -> Set[str]: + """Extract process ids from given process graph.""" + pg = as_flat_graph(pg) + + def extract(pg) -> Iterator[str]: + for v in pg.values(): + yield v["process_id"] + for arg in v["arguments"].values(): + if isinstance(arg, dict) and "process_graph" in arg: + yield from extract(arg["process_graph"]) + + return set(extract(pg)) diff --git a/src/openeo_test_suite/tests/conftest.py b/src/openeo_test_suite/tests/conftest.py index 775de9e..77fea17 100644 --- a/src/openeo_test_suite/tests/conftest.py +++ b/src/openeo_test_suite/tests/conftest.py @@ -8,49 +8,12 @@ import pytest from openeo_test_suite.lib.backend_under_test import get_backend_url +from openeo_test_suite.lib.process_selection import get_selected_processes from openeo_test_suite.lib.skipping import Skipper _log = logging.getLogger(__name__) -@pytest.fixture(scope="session") -def skip_experimental(request) -> bool: - """ - Fixture to determine whether experimental functionality should be tested or not. - """ - skip = not request.config.getoption("--experimental") - _log.info(f"Skip experimental functionality {skip=}") - return skip - - -@pytest.fixture(scope="session") -def process_levels(request) -> List[str]: - """ - Fixture to get the desired openEO profiles levels. - """ - levels_str = request.config.getoption("--process-levels") - - if isinstance(levels_str, str) and len(levels_str) > 0: - _log.info(f"Testing process levels {levels_str!r}") - return list(map(lambda l: l.strip(), levels_str.split(","))) - else: - return [] - - -@pytest.fixture(scope="session") -def processes(request) -> List[str]: - """ - Fixture to get the desired processes to test against. - """ - processes_str = request.config.getoption("--processes") - - if isinstance(processes_str, str) and len(processes_str) > 0: - _log.info(f"Testing processes {processes_str!r}") - return list(map(lambda p: p.strip(), processes_str.split(","))) - else: - return [] - - @pytest.fixture(scope="module") def auto_authenticate() -> bool: """ @@ -61,7 +24,9 @@ def auto_authenticate() -> bool: @pytest.fixture(scope="module") -def connection(request, auto_authenticate: bool, pytestconfig) -> openeo.Connection: +def connection( + request, auto_authenticate: bool, pytestconfig: pytest.Config +) -> openeo.Connection: backend_url = get_backend_url(request.config, required=True) con = openeo.connect(backend_url, auto_validate=False) @@ -91,5 +56,8 @@ def connection(request, auto_authenticate: bool, pytestconfig) -> openeo.Connect @pytest.fixture -def skipper(connection, process_levels) -> Skipper: - return Skipper(connection=connection, process_levels=process_levels) +def skipper(connection) -> Skipper: + return Skipper( + connection=connection, + selected_processes=[p.process_id for p in get_selected_processes()], + ) diff --git a/src/openeo_test_suite/tests/processes/processing/test_example.py b/src/openeo_test_suite/tests/processes/processing/test_example.py index c3a505f..aa55cd3 100644 --- a/src/openeo_test_suite/tests/processes/processing/test_example.py +++ b/src/openeo_test_suite/tests/processes/processing/test_example.py @@ -10,50 +10,26 @@ import pytest from deepdiff import DeepDiff -import openeo_test_suite from openeo_test_suite.lib.process_runner.base import ProcessTestRunner from openeo_test_suite.lib.process_runner.util import isostr_to_datetime +from openeo_test_suite.lib.process_selection import get_selected_processes _log = logging.getLogger(__name__) -DEFAULT_EXAMPLES_ROOT = ( - Path(openeo_test_suite.__file__).parents[2] / "assets/processes/tests" -) - - -def _get_prop(prop: str, data: dict, test: dict, *, default=None): - """Get property from example data, first trying test data, then full process data, then general fallback value.""" - # TODO make this function more generic (e.g. `multidict_get(key, *dicts, default=None)`) - if prop in test: - level = test[prop] - elif prop in data: - level = data[prop] - else: - level = default - return level - - -def get_examples( - root: Union[str, Path] = DEFAULT_EXAMPLES_ROOT -) -> List[Tuple[str, dict, Path, str, bool]]: +def get_examples() -> List[Tuple[str, dict, Path, str, bool]]: """Collect process examples/tests from examples root folder containing JSON5 files.""" - # TODO return a list of NamedTuples? - examples = [] - # TODO: it's not recommended use `file` (a built-in) as variable name. `path` would be better. - for file in root.glob("*.json5"): - process_id = file.stem - try: - with file.open() as f: - data = json5.load(f) - for test in data["tests"]: - level = _get_prop("level", data, test, default="L4") - experimental = _get_prop("experimental", data, test, default=False) - examples.append((process_id, test, file, level, experimental)) - except Exception as e: - _log.warning(f"Failed to load {file}: {e}") - - return examples + return [ + ( + process.process_id, + test, + process.path, + test.get("level", process.level), + test.get("experimental", process.experimental), + ) + for process in get_selected_processes() + for test in process.tests + ] @pytest.mark.parametrize( @@ -61,9 +37,6 @@ def get_examples( ) def test_process( connection, - skip_experimental, - process_levels, - processes, process_id, example, file, @@ -71,21 +44,8 @@ def test_process( experimental, skipper, ): - skipper.skip_if_unmatching_process_level(level) - if len(processes) > 0 and process_id not in processes: - pytest.skip( - f"Skipping process {process_id!r} because it is not in the specified processes" - ) - - # check whether the process is available - skipper.skip_if_unsupported_process([process_id]) - - if skip_experimental and experimental: - pytest.skip("Skipping experimental process {}".format(id)) - - # check whether any additionally required processes are available - if "required" in example: - skipper.skip_if_unsupported_process(example["required"]) + # Check whether the process (and additional extra required ones, if any) is supported on the backend + skipper.skip_if_unsupported_process([process_id] + example.get("required", [])) # prepare the arguments from test JSON encoding to internal backend representations # or skip if not supported by the test runner diff --git a/src/openeo_test_suite/tests/workflows/L1/test_apply.py b/src/openeo_test_suite/tests/workflows/L1/test_apply.py index c10e1dc..6880b1e 100644 --- a/src/openeo_test_suite/tests/workflows/L1/test_apply.py +++ b/src/openeo_test_suite/tests/workflows/L1/test_apply.py @@ -2,8 +2,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L1" - def test_apply( skipper, @@ -12,10 +10,10 @@ def test_apply( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_apply.nc" cube = cube_one_day_red.apply(lambda x: x.clip(0, 1)) + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L1/test_apply_dimension_L1.py b/src/openeo_test_suite/tests/workflows/L1/test_apply_dimension_L1.py index 8b24b94..d19260b 100644 --- a/src/openeo_test_suite/tests/workflows/L1/test_apply_dimension_L1.py +++ b/src/openeo_test_suite/tests/workflows/L1/test_apply_dimension_L1.py @@ -2,8 +2,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L1" - def test_apply_dimension_quantiles_0( skipper, @@ -21,7 +19,6 @@ def test_apply_dimension_quantiles_0( number of labels will be equal to the number of values computed by the process. """ skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_apply_dimension_quantiles_0.nc" b_dim = collection_dims["b_dim"] @@ -33,7 +30,7 @@ def test_apply_dimension_quantiles_0( process=lambda d: quantiles(d, probabilities=[0.5, 0.75]), dimension=b_dim, ) - + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() data = load_netcdf_dataarray(filename, band_dim_name=b_dim) @@ -54,7 +51,6 @@ def test_apply_dimension_quantiles_1( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_apply_dimension_quantiles_1.nc" b_dim = collection_dims["b_dim"] @@ -67,7 +63,7 @@ def test_apply_dimension_quantiles_1( dimension=t_dim, target_dimension=b_dim, ) - print(filename) + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() data = load_netcdf_dataarray(filename, band_dim_name=b_dim) @@ -90,7 +86,6 @@ def test_apply_dimension_ndvi( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_apply_dimension_ndvi.nc" b_dim = collection_dims["b_dim"] @@ -104,6 +99,7 @@ def compute_ndvi(data): return array_concat(data, ndvi) ndvi = cube_one_day_red_nir.apply_dimension(dimension=b_dim, process=compute_ndvi) + skipper.skip_if_unselected_process(ndvi) ndvi.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L1/test_load_save.py b/src/openeo_test_suite/tests/workflows/L1/test_load_save.py index 8e67ac2..00fa1ad 100644 --- a/src/openeo_test_suite/tests/workflows/L1/test_load_save.py +++ b/src/openeo_test_suite/tests/workflows/L1/test_load_save.py @@ -9,8 +9,6 @@ bounding_box_32632_10x10, ) -LEVEL = "L1" - def test_load_save_netcdf( skipper, @@ -19,7 +17,6 @@ def test_load_save_netcdf( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_load_save_netcdf.nc" b_dim = collection_dims["b_dim"] @@ -27,6 +24,7 @@ def test_load_save_netcdf( y_dim = collection_dims["y_dim"] t_dim = collection_dims["t_dim"] + skipper.skip_if_unselected_process(cube_red_nir) cube_red_nir.download(filename) assert filename.exists() @@ -64,6 +62,7 @@ def test_load_save_10x10_netcdf( y_dim = collection_dims["y_dim"] t_dim = collection_dims["t_dim"] + skipper.skip_if_unselected_process(cube_red_10x10) cube_red_10x10.download(filename) assert filename.exists() @@ -105,6 +104,7 @@ def test_load_save_geotiff( y_dim = collection_dims["y_dim"] t_dim = collection_dims["t_dim"] + skipper.skip_if_unselected_process(cube_one_day_red) cube_one_day_red.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L1/test_reduce_bands.py b/src/openeo_test_suite/tests/workflows/L1/test_reduce_bands.py index 4f6f2f7..eb7e51c 100644 --- a/src/openeo_test_suite/tests/workflows/L1/test_reduce_bands.py +++ b/src/openeo_test_suite/tests/workflows/L1/test_reduce_bands.py @@ -1,7 +1,5 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L1" - def test_ndvi_index( skipper, @@ -10,7 +8,6 @@ def test_ndvi_index( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_ndvi_index.nc" b_dim = collection_dims["b_dim"] @@ -23,6 +20,7 @@ def compute_ndvi(data): return (nir - red) / (nir + red) ndvi = cube_one_day_red_nir.reduce_dimension(dimension=b_dim, reducer=compute_ndvi) + skipper.skip_if_unselected_process(ndvi) ndvi.download(filename) assert filename.exists() @@ -49,6 +47,7 @@ def compute_ndvi(data): return (nir - red) / (nir + red) ndvi = cube_one_day_red_nir.reduce_dimension(dimension=b_dim, reducer=compute_ndvi) + skipper.skip_if_unselected_process(ndvi) ndvi.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L1/test_reduce_logic_ops.py b/src/openeo_test_suite/tests/workflows/L1/test_reduce_logic_ops.py index 93d7e85..c5eebad 100644 --- a/src/openeo_test_suite/tests/workflows/L1/test_reduce_logic_ops.py +++ b/src/openeo_test_suite/tests/workflows/L1/test_reduce_logic_ops.py @@ -2,8 +2,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L1" - def test_boolean_mask( skipper, @@ -12,7 +10,6 @@ def test_boolean_mask( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_boolean_mask.nc" b_dim = collection_dims["b_dim"] @@ -26,6 +23,7 @@ def compute_ndvi(data): ndvi = cube_one_day_red_nir.reduce_dimension(dimension=b_dim, reducer=compute_ndvi) ndvi_mask = ndvi > 0.75 + skipper.skip_if_unselected_process(ndvi_mask) ndvi_mask.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L1/test_reduce_time.py b/src/openeo_test_suite/tests/workflows/L1/test_reduce_time.py index 6af7e60..4c20d07 100644 --- a/src/openeo_test_suite/tests/workflows/L1/test_reduce_time.py +++ b/src/openeo_test_suite/tests/workflows/L1/test_reduce_time.py @@ -1,7 +1,5 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L1" - def test_reduce_time( skipper, @@ -10,13 +8,13 @@ def test_reduce_time( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_reduce_time.nc" b_dim = collection_dims["b_dim"] t_dim = collection_dims["t_dim"] cube = cube_red_nir.reduce_dimension(dimension=t_dim, reducer="mean") + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L2/test_add_dimension.py b/src/openeo_test_suite/tests/workflows/L2/test_add_dimension.py index 75dba9a..6165266 100644 --- a/src/openeo_test_suite/tests/workflows/L2/test_add_dimension.py +++ b/src/openeo_test_suite/tests/workflows/L2/test_add_dimension.py @@ -1,7 +1,5 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L2" - def test_ndvi_add_dim( skipper, @@ -10,7 +8,6 @@ def test_ndvi_add_dim( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_ndvi_add_dim.nc" b_dim = collection_dims["b_dim"] @@ -22,6 +19,7 @@ def compute_ndvi(data): ndvi = cube_one_day_red_nir.reduce_dimension(dimension=b_dim, reducer=compute_ndvi) ndvi = ndvi.add_dimension(type="bands", name=b_dim, label="NDVI") + skipper.skip_if_unselected_process(ndvi) ndvi.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L2/test_aggregate_temporal.py b/src/openeo_test_suite/tests/workflows/L2/test_aggregate_temporal.py index 82445a1..6d0638a 100644 --- a/src/openeo_test_suite/tests/workflows/L2/test_aggregate_temporal.py +++ b/src/openeo_test_suite/tests/workflows/L2/test_aggregate_temporal.py @@ -1,8 +1,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray from openeo_test_suite.lib.workflows.parameters import temporal_interval_one_day -LEVEL = "L2" - def test_aggregate_temporal( skipper, @@ -11,13 +9,17 @@ def test_aggregate_temporal( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_aggregate_temporal.nc" t_dim = collection_dims["t_dim"] b_dim = collection_dims["b_dim"] - intervals = [ [ "2022-06-01", "2022-06-07" ], [ "2022-06-07", "2022-06-14" ], [ "2022-06-14", "2022-06-21" ]] - cube = cube_red_nir.aggregate_temporal(intervals=intervals,reducer="sd") + intervals = [ + ["2022-06-01", "2022-06-07"], + ["2022-06-07", "2022-06-14"], + ["2022-06-14", "2022-06-21"], + ] + cube = cube_red_nir.aggregate_temporal(intervals=intervals, reducer="sd") + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() @@ -36,13 +38,13 @@ def test_aggregate_temporal_period( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_aggregate_temporal_period.nc" t_dim = collection_dims["t_dim"] b_dim = collection_dims["b_dim"] - cube = cube_red_nir.aggregate_temporal_period(period="week",reducer="sum") + cube = cube_red_nir.aggregate_temporal_period(period="week", reducer="sum") + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() @@ -52,4 +54,3 @@ def test_aggregate_temporal_period( assert len(data.dims) == 4 # 2 spatial + 1 bands + 1 temporal assert t_dim in data.dims assert len(data[t_dim]) == 5 - diff --git a/src/openeo_test_suite/tests/workflows/L2/test_apply_dimension_L2.py b/src/openeo_test_suite/tests/workflows/L2/test_apply_dimension_L2.py index 34fba0b..6e01e71 100644 --- a/src/openeo_test_suite/tests/workflows/L2/test_apply_dimension_L2.py +++ b/src/openeo_test_suite/tests/workflows/L2/test_apply_dimension_L2.py @@ -2,8 +2,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L2" - def test_apply_dimension_ndvi_2( skipper, @@ -12,7 +10,6 @@ def test_apply_dimension_ndvi_2( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_apply_dimension_ndvi_2.nc" b_dim = collection_dims["b_dim"] @@ -26,6 +23,7 @@ def compute_ndvi(data): return array_concat(data, ndvi) ndvi = cube_one_day_red_nir.apply_dimension(dimension=b_dim, process=compute_ndvi) + skipper.skip_if_unselected_process(ndvi) ndvi.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L2/test_drop_dimension.py b/src/openeo_test_suite/tests/workflows/L2/test_drop_dimension.py index 02f096a..ff5deea 100644 --- a/src/openeo_test_suite/tests/workflows/L2/test_drop_dimension.py +++ b/src/openeo_test_suite/tests/workflows/L2/test_drop_dimension.py @@ -1,7 +1,5 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L2" - def test_drop_dimension_time( skipper, @@ -10,13 +8,13 @@ def test_drop_dimension_time( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_drop_dimension_time.nc" t_dim = collection_dims["t_dim"] b_dim = collection_dims["b_dim"] cube = cube_red_10x10.drop_dimension(name=t_dim) + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L2/test_filter_bbox.py b/src/openeo_test_suite/tests/workflows/L2/test_filter_bbox.py index 3af90b8..13e0959 100644 --- a/src/openeo_test_suite/tests/workflows/L2/test_filter_bbox.py +++ b/src/openeo_test_suite/tests/workflows/L2/test_filter_bbox.py @@ -3,8 +3,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray from openeo_test_suite.lib.workflows.parameters import bounding_box_32632_10x10 -LEVEL = "L2" - def test_filter_bbox( skipper, @@ -13,7 +11,6 @@ def test_filter_bbox( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_filter_bbox.nc" b_dim = collection_dims["b_dim"] @@ -22,6 +19,7 @@ def test_filter_bbox( t_dim = collection_dims["t_dim"] cube = cube_one_day_red.filter_bbox(bounding_box_32632_10x10) + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L2/test_filter_temporal.py b/src/openeo_test_suite/tests/workflows/L2/test_filter_temporal.py index b6d3e95..3eb5eca 100644 --- a/src/openeo_test_suite/tests/workflows/L2/test_filter_temporal.py +++ b/src/openeo_test_suite/tests/workflows/L2/test_filter_temporal.py @@ -1,8 +1,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray from openeo_test_suite.lib.workflows.parameters import temporal_interval_one_day -LEVEL = "L2" - def test_filter_temporal( skipper, @@ -11,13 +9,13 @@ def test_filter_temporal( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_filter_temporal.nc" t_dim = collection_dims["t_dim"] b_dim = collection_dims["b_dim"] cube = cube_red_nir.filter_temporal(temporal_interval_one_day) + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L2/test_rename_labels.py b/src/openeo_test_suite/tests/workflows/L2/test_rename_labels.py index 8759900..e15c13a 100644 --- a/src/openeo_test_suite/tests/workflows/L2/test_rename_labels.py +++ b/src/openeo_test_suite/tests/workflows/L2/test_rename_labels.py @@ -1,7 +1,5 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L2" - def test_rename_labels_bands( skipper, @@ -10,7 +8,6 @@ def test_rename_labels_bands( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_rename_labels_bands.nc" b_dim = collection_dims["b_dim"] @@ -18,6 +15,7 @@ def test_rename_labels_bands( renamed_cube = cube_one_day_red_nir.rename_labels( dimension=b_dim, source=["B04", "B08"], target=["red", "nir"] ) + skipper.skip_if_unselected_process(renamed_cube) renamed_cube.download(filename) assert filename.exists() @@ -36,7 +34,6 @@ def test_rename_labels_time( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_rename_labels_time.nc" t_dim = collection_dims["t_dim"] @@ -46,6 +43,7 @@ def test_rename_labels_time( renamed_cube = cube_one_day_red_nir.rename_labels( dimension=t_dim, source=t_labels, target=["first_date"] ) + skipper.skip_if_unselected_process(renamed_cube) renamed_cube.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L2A/test_apply_kernel.py b/src/openeo_test_suite/tests/workflows/L2A/test_apply_kernel.py index d3900d2..a1dbbd5 100644 --- a/src/openeo_test_suite/tests/workflows/L2A/test_apply_kernel.py +++ b/src/openeo_test_suite/tests/workflows/L2A/test_apply_kernel.py @@ -2,8 +2,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L2A" - def test_apply_kernel( skipper, @@ -12,15 +10,16 @@ def test_apply_kernel( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename_0 = tmp_path / "test_apply_kernel_source.nc" filename_1 = tmp_path / "test_apply_kernel.nc" b_dim = collection_dims["b_dim"] - cube_red_10x10.download(filename_0) - blur_cube = cube_red_10x10.apply_kernel(kernel=np.ones((3, 3)), factor=1 / 9) + + skipper.skip_if_unselected_process(cube_red_10x10) + skipper.skip_if_unselected_process(blur_cube) + cube_red_10x10.download(filename_0) blur_cube.download(filename_1) data_source = load_netcdf_dataarray(filename_0, band_dim_name=b_dim) diff --git a/src/openeo_test_suite/tests/workflows/L2A/test_filter_bands.py b/src/openeo_test_suite/tests/workflows/L2A/test_filter_bands.py index ce8a76b..2ba781e 100644 --- a/src/openeo_test_suite/tests/workflows/L2A/test_filter_bands.py +++ b/src/openeo_test_suite/tests/workflows/L2A/test_filter_bands.py @@ -2,8 +2,6 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L2A" - def test_filter_bands( skipper, @@ -12,12 +10,12 @@ def test_filter_bands( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_filter_bbox.nc" b_dim = collection_dims["b_dim"] cube = cube_one_day_red_nir.filter_bands(["B08"]) + skipper.skip_if_unselected_process(cube) cube.download(filename) assert filename.exists() diff --git a/src/openeo_test_suite/tests/workflows/L3/test_apply_dimension_L3.py b/src/openeo_test_suite/tests/workflows/L3/test_apply_dimension_L3.py index 90edca1..a65707b 100644 --- a/src/openeo_test_suite/tests/workflows/L3/test_apply_dimension_L3.py +++ b/src/openeo_test_suite/tests/workflows/L3/test_apply_dimension_L3.py @@ -1,7 +1,5 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L3" - def test_apply_dimension_order( skipper, @@ -10,7 +8,6 @@ def test_apply_dimension_order( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_apply_dimension_order.nc" b_dim = collection_dims["b_dim"] @@ -21,8 +18,9 @@ def test_apply_dimension_order( process=lambda d: order(d), dimension=b_dim, ) - + skipper.skip_if_unselected_process(cube) cube.download(filename) + assert filename.exists() data = load_netcdf_dataarray(filename, band_dim_name=b_dim) diff --git a/src/openeo_test_suite/tests/workflows/L3/test_merge_cubes.py b/src/openeo_test_suite/tests/workflows/L3/test_merge_cubes.py index 67935b2..8448771 100644 --- a/src/openeo_test_suite/tests/workflows/L3/test_merge_cubes.py +++ b/src/openeo_test_suite/tests/workflows/L3/test_merge_cubes.py @@ -1,7 +1,5 @@ from openeo_test_suite.lib.workflows.io import load_netcdf_dataarray -LEVEL = "L3" - def test_reduce_time_merge( skipper, @@ -10,7 +8,6 @@ def test_reduce_time_merge( tmp_path, ): skipper.skip_if_no_netcdf_support() - skipper.skip_if_unmatching_process_level(level=LEVEL) filename = tmp_path / "test_reduce_time_merge.nc" b_dim = collection_dims["b_dim"] @@ -20,6 +17,7 @@ def test_reduce_time_merge( cube_1 = cube_red_nir.reduce_dimension(dimension=t_dim, reducer="median") cube_merged = cube_0.merge_cubes(cube_1, overlap_resolver="subtract") + skipper.skip_if_unselected_process(cube_merged) cube_merged.download(filename) assert filename.exists()