diff --git a/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/deploy.yml b/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/deploy.yml new file mode 100644 index 000000000..eadbae766 --- /dev/null +++ b/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/deploy.yml @@ -0,0 +1,11 @@ +processDescription: + process: + id: Finch_EnsembleGridPointWetdays + jobControlOptions: + - async-execute + outputTransmission: + - reference +executionUnit: + # note: This does not work by itself! The test suite injects the file dynamically. + - href: "tests/functional/application-packages/Finch_EnsembleGridPointWetdays/package.cwl" +deploymentProfileName: "http://www.opengis.net/profiles/eoc/dockerizedApplication" diff --git a/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/execute.yml b/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/execute.yml new file mode 100644 index 000000000..486c6ffc9 --- /dev/null +++ b/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/execute.yml @@ -0,0 +1,13 @@ +lat: "45.35629610945964" +lon: "-73.98748912005094" +start_date: "1950" +end_date: "1960" +ensemble_percentiles: "" +dataset: "candcs-u6" +scenario: "ssp126" +models: "26models" +freq: "YS" +data_validation: "warn" +output_format: "csv" +csv_precision: 0 +thresh: "15 mm\/day" diff --git a/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/package.cwl b/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/package.cwl new file mode 100644 index 000000000..7943d9c32 --- /dev/null +++ b/tests/functional/application-packages/Finch_EnsembleGridPointWetdays/package.cwl @@ -0,0 +1,311 @@ +# NOTE: +# Inspired from (but not an exact equivalent): +# https://finch.crim.ca/wps?service=WPS&request=DescribeProcess&version=1.0.0&identifier=ensemble_grid_point_wetdays +# Outputs are modified to only collect stdout, since we don't have the expected outputs produced by the real process. +# The 'baseCommand' is also added to produce this stdout output. +# All remaining arguments are identical. +cwlVersion: v1.0 +class: CommandLineTool +requirements: + InlineJavascriptRequirement: {} +# NOTE: replaced by 'baseCommand' +#hints: +# WPS1Requirement: +# provider: https://finch.crim.ca/wps +# process: ensemble_grid_point_wetdays +baseCommand: echo +inputs: +- id: lat + type: + - string + - type: array + items: string +- id: lon + type: + - string + - type: array + items: string +- id: start_date + type: + - 'null' + - string +- id: end_date + type: + - 'null' + - string +- id: ensemble_percentiles + type: + - 'null' + - string + default: 10,50,90 +- id: average + type: + - 'null' + - boolean + default: false +- id: dataset + type: + - 'null' + - type: enum + symbols: + - humidex-daily + - candcs-u5 + - candcs-u6 + - bccaqv2 + default: candcs-u5 +# WARNING: +# Following definition combining 'enum' and its corresponding nested definition in 'array' caused a +# schema-salad name resolution error. This CWL is used particularly to validate this *valid* type resolution. +# see https://github.com/common-workflow-language/cwltool/issues/1908 +- id: scenario + type: + - 'null' + - type: enum + symbols: + - ssp126 + - rcp85 + - rcp45 + - rcp26 + - ssp585 + - ssp245 + - type: array + items: + type: enum + symbols: + - ssp126 + - rcp85 + - rcp45 + - rcp26 + - ssp585 + - ssp245 +- id: models + type: + - 'null' + - type: enum + symbols: + - KACE-1-0-G + - CCSM4 + - MIROC5 + - EC-Earth3-Veg + - TaiESM1 + - GFDL-ESM4 + - GFDL-CM3 + - CanESM5 + - HadGEM3-GC31-LL + - INM-CM4-8 + - IPSL-CM5A-MR + - EC-Earth3 + - GFDL-ESM2G + - humidex_models + - GFDL-ESM2M + - MIROC-ESM + - CSIRO-Mk3-6-0 + - MPI-ESM-LR + - NorESM1-M + - CNRM-CM5 + - all + - GISS-E2-1-G + - 24models + - MPI-ESM1-2-HR + - CNRM-ESM2-1 + - CNRM-CM6-1 + - CanESM2 + - FGOALS-g3 + - NorESM1-ME + - IPSL-CM6A-LR + - CMCC-ESM2 + - pcic12 + - EC-Earth3-Veg-LR + - ACCESS-ESM1-5 + - MRI-CGCM3 + - MIROC-ESM-CHEM + - NorESM2-MM + - bcc-csm1-1-m + - BNU-ESM + - UKESM1-0-LL + - CESM1-CAM5 + - MIROC-ES2L + - MRI-ESM2-0 + - HadGEM2-ES + - MIROC6 + - MPI-ESM-MR + - INM-CM5-0 + - bcc-csm1-1 + - BCC-CSM2-MR + - ACCESS-CM2 + - NorESM2-LM + - IPSL-CM5A-LR + - FGOALS-g2 + - HadGEM2-AO + - 26models + - MPI-ESM1-2-LR + - KIOST-ESM + - type: array + items: + type: enum + symbols: + - KACE-1-0-G + - CCSM4 + - MIROC5 + - EC-Earth3-Veg + - TaiESM1 + - GFDL-ESM4 + - GFDL-CM3 + - CanESM5 + - HadGEM3-GC31-LL + - INM-CM4-8 + - IPSL-CM5A-MR + - EC-Earth3 + - GFDL-ESM2G + - humidex_models + - GFDL-ESM2M + - MIROC-ESM + - CSIRO-Mk3-6-0 + - MPI-ESM-LR + - NorESM1-M + - CNRM-CM5 + - all + - GISS-E2-1-G + - 24models + - MPI-ESM1-2-HR + - CNRM-ESM2-1 + - CNRM-CM6-1 + - CanESM2 + - FGOALS-g3 + - NorESM1-ME + - IPSL-CM6A-LR + - CMCC-ESM2 + - pcic12 + - EC-Earth3-Veg-LR + - ACCESS-ESM1-5 + - MRI-CGCM3 + - MIROC-ESM-CHEM + - NorESM2-MM + - bcc-csm1-1-m + - BNU-ESM + - UKESM1-0-LL + - CESM1-CAM5 + - MIROC-ES2L + - MRI-ESM2-0 + - HadGEM2-ES + - MIROC6 + - MPI-ESM-MR + - INM-CM5-0 + - bcc-csm1-1 + - BCC-CSM2-MR + - ACCESS-CM2 + - NorESM2-LM + - IPSL-CM5A-LR + - FGOALS-g2 + - HadGEM2-AO + - 26models + - MPI-ESM1-2-LR + - KIOST-ESM + default: all +- id: thresh + type: + - 'null' + - string + default: 1.0 mm/day +- id: freq + type: + - 'null' + - type: enum + symbols: + - YS + - QS-DEC + - AS-JUL + - MS + default: YS +- id: op + type: + - 'null' + - type: enum + symbols: + - '>=' + - '>' + - gt + - ge + default: '>=' +- id: month + type: + - 'null' + - int + - type: array + items: int + inputBinding: + valueFrom: "\n ${\n const values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];\n if (Array.isArray(self)) {\n \n if (self.every(item => values.includes(item))) {\n return self;\n }\n else {\n throw \"invalid value(s) in [\" + self + \"] are not all allowed values from [\" + values + \"]\";\n }\n \n }\n else {\n \n if (values.includes(self)) {\n return self;\n }\n else {\n throw \"invalid value \" + self + \" is not an allowed value from [\" + values + \"]\";\n }\n \n }\n }\n " +- id: season + type: + - 'null' + - type: enum + symbols: + - SON + - MAM + - JJA + - DJF +- id: check_missing + type: + - 'null' + - type: enum + symbols: + - pct + - at_least_n + - wmo + - skip + - from_context + - any + default: any +- id: missing_options + type: + - 'null' + - File + format: iana:application/json +- id: cf_compliance + type: + - 'null' + - type: enum + symbols: + - raise + - log + - warn + default: warn +- id: data_validation + type: + - 'null' + - type: enum + symbols: + - raise + - log + - warn + default: raise +- id: output_name + type: + - 'null' + - string +- id: output_format + type: + - 'null' + - type: enum + symbols: + - csv + - netcdf + default: netcdf +- id: csv_precision + type: + - 'null' + - int +# NOTE: +# Following structure is permitted in standard CWL, but not supported in Weaver. +# Must use the equivalent 'long form' in the meantime. +#outputs: +#- id: output +# type: stdout +outputs: + - id: output + type: File + outputBinding: + glob: "*/stdout.log" # auto-added by Weaver with corresponding 'stdout: stdout.log' definition +$namespaces: + iana: https://www.iana.org/assignments/media-types/ + edam: http://edamontology.org/ diff --git a/tests/functional/test_wps_package.py b/tests/functional/test_wps_package.py index 589411db7..4deb76af3 100644 --- a/tests/functional/test_wps_package.py +++ b/tests/functional/test_wps_package.py @@ -62,7 +62,7 @@ ) from weaver.processes.types import ProcessType from weaver.status import Status -from weaver.utils import fetch_file, get_any_value, load_file +from weaver.utils import fetch_file, get_any_value, get_path_kvp, load_file from weaver.wps.utils import get_wps_output_dir, get_wps_output_url, map_wps_output_location if TYPE_CHECKING: @@ -3121,6 +3121,32 @@ def test_deploy_enum_array_and_multi_format_inputs_from_wps_xml_reference(self): def test_deploy_multi_outputs_file_from_wps_xml_reference(self): raise NotImplementedError + def test_execute_various_cwl_enum_schema(self): + proc = "Finch_EnsembleGridPointWetdays" + body = self.retrieve_payload(proc, "deploy", local=True) + pkg = self.retrieve_payload(proc, "package", local=True) + body["executionUnit"] = [{"unit": pkg}] + self.deploy_process(body, describe_schema=ProcessSchema.OGC) + + data = self.retrieve_payload(proc, "execute", local=True) + exec_body = { + "mode": ExecuteMode.ASYNC, + "response": ExecuteResponse.DOCUMENT, + "inputs": data, + } + with contextlib.ExitStack() as stack: + for mock_exec in mocked_execute_celery(): + stack.enter_context(mock_exec) + proc_url = f"/processes/{proc}/jobs" + resp = mocked_sub_requests(self.app, "post_json", proc_url, timeout=5, + data=exec_body, headers=self.json_headers, only_local=True) + assert resp.status_code in [200, 201], f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}" + + status_url = resp.json["location"] + results = self.monitor_job(status_url) + + assert results + @pytest.mark.functional class WpsPackageAppWithS3BucketTest(WpsConfigBase, ResourcesUtil): diff --git a/tests/processes/test_convert.py b/tests/processes/test_convert.py index 0f8f1601a..7af12df68 100644 --- a/tests/processes/test_convert.py +++ b/tests/processes/test_convert.py @@ -1,8 +1,6 @@ """ Unit tests of functions within :mod:`weaver.processes.convert`. """ -import copy - # pylint: disable=R1729 # ignore non-generator representation employed for displaying test log results import json @@ -549,6 +547,78 @@ def test_any2cwl_io_enum_validate(test_io, test_input, expect_valid): tool(**inputs) +def test_any2cwl_io_enum_schema_name(): + """ + Ensure that :term:`CWL` ``Enum`` contains a ``name`` to avoid false-positive conflicting schemas. + + When an ``Enum`` is reused multiple times to define an I/O, omitting the ``name`` makes the duplicate definition + to be considered a conflict, since :mod:`cwltool` will automatically apply an auto-generated ``name`` for that + schema. + + .. seealso:: + - https://github.com/common-workflow-language/cwltool/issues/1908 + """ + test_symbols = [str(i) for i in range(100)] + test_input = { + "id": "test", + "data_type": "string", + "allowed_values": test_symbols, + "any_value": False, + "min_occurs": 0, + "max_occurs": 3, + } + cwl_expect = { + "id": "test", + "type": [ + "null", + { + "type": "enum", + "symbols": test_symbols, + }, + { + "type": "array", + "items": { + "type": "enum", + "symbols": test_symbols, + }, + }, + ] + } + cwl_input, _ = any2cwl_io(test_input, IO_INPUT) + assert cwl_input == cwl_expect + + # test it against an actual definition + cwl = { + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "baseCommand": "echo", + "inputs": { + "test": cwl_input, + }, + "outputs": { + "output": "stdout", + } + } + cwl_without_name = deepcopy(cwl) + cwl_without_name["inputs"]["test"].pop("name", None) # FIXME: no None since 'name' required + + factory = CWLFactory() + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as tmp_file: + json.dump(cwl_without_name, tmp_file) + tmp_file.flush() + with pytest.raises(WorkflowException): + tool = factory.make(f"file://{tmp_file.name}") + tool(test=test_symbols[0]) + + tmp_file.seek(0) + json.dump(cwl, tmp_file) + tmp_file.flush() + tool = factory.make(f"file://{tmp_file.name}") + tool(test=None) + tool(test=test_symbols[0]) + tool(test=[test_symbols[0]]) + + @pytest.mark.parametrize( ["test_io", "expect"], [ @@ -781,7 +851,7 @@ def test_get_cwl_io_type_unmodified(io_info, io_def): .. seealso:: - https://github.com/crim-ca/weaver/pull/546 """ - io_copy = copy.deepcopy(io_info) + io_copy = deepcopy(io_info) io_res = get_cwl_io_type(io_info) assert io_res == io_def assert io_info == io_copy, "Argument I/O information should not be modified from parsing." diff --git a/tests/resources/__init__.py b/tests/resources/__init__.py index 00c6968cf..5e4357b10 100644 --- a/tests/resources/__init__.py +++ b/tests/resources/__init__.py @@ -11,6 +11,7 @@ RESOURCES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "")) EXAMPLES_PATH = os.path.join(WEAVER_MODULE_DIR, "wps_restapi/examples") +FUNCTIONAL_APP_PKG = os.path.abspath(os.path.join(RESOURCES_PATH, "../functional/application-packages")) GET_CAPABILITIES_TEMPLATE_URL = "{}?service=WPS&request=GetCapabilities&version=1.0.0" DESCRIBE_PROCESS_TEMPLATE_URL = "{}?service=WPS&request=DescribeProcess&identifier={}&version=1.0.0" diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index fb4b4adca..57acf52f2 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -4619,16 +4619,28 @@ class CWLTypeSymbols(ExtendedSequenceSchema): symbol = CWLTypeSymbolValues() -class CWLTypeArray(ExtendedMappingSchema): - type = ExtendedSchemaNode(String(), example=PACKAGE_ARRAY_BASE, validator=OneOf([PACKAGE_ARRAY_BASE])) - items = CWLTypeString(title="CWLTypeArrayItems", validator=OneOf(PACKAGE_ARRAY_ITEMS)) - - class CWLTypeEnum(ExtendedMappingSchema): type = ExtendedSchemaNode(String(), example=PACKAGE_ENUM_BASE, validator=OneOf(PACKAGE_CUSTOM_TYPES)) symbols = CWLTypeSymbols(summary="Allowed values composing the enum.") +class CWLTypeArrayItemObject(ExtendedMappingSchema): + type = CWLTypeString(validator=OneOf(PACKAGE_ARRAY_ITEMS - PACKAGE_CUSTOM_TYPES)) + + +class CWLTypeArrayItems(OneOfKeywordSchema): + _one_of = [ + CWLTypeString(title="CWLTypeArrayItemsString", validator=OneOf(PACKAGE_ARRAY_ITEMS)), + CWLTypeEnum(summary="CWL type as enum of values."), + CWLTypeArrayItemObject(summary="CWL type in nested object definition."), + ] + + +class CWLTypeArray(ExtendedMappingSchema): + type = ExtendedSchemaNode(String(), example=PACKAGE_ARRAY_BASE, validator=OneOf([PACKAGE_ARRAY_BASE])) + items = CWLTypeArrayItems() + + class CWLTypeBase(OneOfKeywordSchema): _one_of = [ CWLTypeString(summary="CWL type as literal value."),