From aedffab78c7a71340659b72b8708fa38f8753b9b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 10 Aug 2021 12:09:44 -0400 Subject: [PATCH] (wip) adjust tests for file2string_array modified operation + adjust test file-handling mocks to provide more details --- tests/functional/test_builtin.py | 25 +++++++++++--------- tests/functional/test_wps_package.py | 8 +++---- tests/test_utils.py | 2 +- tests/utils.py | 35 +++++++++++++++++++++------- 4 files changed, 45 insertions(+), 25 deletions(-) diff --git a/tests/functional/test_builtin.py b/tests/functional/test_builtin.py index 06bf642d7..d85261f81 100644 --- a/tests/functional/test_builtin.py +++ b/tests/functional/test_builtin.py @@ -5,9 +5,16 @@ from typing import TYPE_CHECKING import pytest +import mock from tests.functional.utils import WpsPackageConfigBase -from tests.utils import get_settings_from_testapp, mocked_execute_process, mocked_sub_requests +from tests.utils import ( + get_settings_from_testapp, + mocked_execute_process, + mocked_http_file, + mocked_reference_test_file, + mocked_sub_requests +) from weaver.execute import EXECUTE_TRANSMISSION_MODE_REFERENCE from weaver.formats import CONTENT_TYPE_APP_JSON, CONTENT_TYPE_APP_NETCDF from weaver.processes.builtin import register_builtin_processes @@ -137,19 +144,15 @@ def test_file2string_array_describe(self): assert len(resp.json["process"]["outputs"][0]["formats"]) == 1 assert resp.json["process"]["outputs"][0]["formats"][0]["mimeType"] == CONTENT_TYPE_APP_JSON # important here - def test_file2string_array_execute(self): - tmp_file = None - dirname = tempfile.gettempdir() + # FIXME: must simulate otherwise tmp-file gets fetched and copied to pywps dir, changing output content (wrong path) + @mock.patch("weaver.processes.wps_package.WpsPackage.must_fetch", return_value=False) + def test_file2string_array_execute(self, __mock): + fake_href = "https://random-server.com/fake-data.txt" with contextlib.ExitStack() as stack_exec: - tmp_text = tempfile.NamedTemporaryFile(dir=dirname, mode="w", suffix=".txt") - tmp_text = stack_exec.enter_context(tmp_text) # noqa - tmp_text.write("Hello World!") - tmp_text.seek(0) - tmp_file = tmp_text.name data = { "mode": "async", "response": "document", - "inputs": [{"id": "input", "href": tmp_file}], + "inputs": [{"id": "input", "href": fake_href}], "outputs": [{"id": "output", "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE}], } @@ -184,7 +187,7 @@ def test_file2string_array_execute(self): assert os.path.isfile(real_path) with open(real_path, "r") as f: out_data = json.load(f) - assert out_data == [tmp_file] + assert out_data == [fake_href] # if everything was valid for results, validate equivalent but differently formatted outputs response output_url = job_url + "/outputs" diff --git a/tests/functional/test_wps_package.py b/tests/functional/test_wps_package.py index 1ad61fd98..c88885c04 100644 --- a/tests/functional/test_wps_package.py +++ b/tests/functional/test_wps_package.py @@ -1028,19 +1028,19 @@ def tmp(value): assert desc["process"] is not None - test_bucket_ref = mocked_aws_s3_bucket_test_file( + test_bucket_ref, _ = mocked_aws_s3_bucket_test_file( "wps-process-test-bucket", "input_file_s3.txt", "This is a generated file for s3 test" ) - test_http_ref = mocked_reference_test_file( + test_http_ref, _ = mocked_reference_test_file( "input_file_http.txt", "http", "This is a generated file for http test" ) - test_file_ref = mocked_reference_test_file( + test_file_ref, _ = mocked_reference_test_file( "input_file_ref.txt", "file", "This is a generated file for file test" @@ -1596,7 +1596,7 @@ def test_execute_application_package_process_with_bucket(self): input_file_s3 = "input-s3.txt" input_file_http = "media-types.txt" # use some random HTTP location that actually exists (will be fetched) test_http_ref = "https://www.iana.org/assignments/media-types/{}".format(input_file_http) - test_bucket_ref = mocked_aws_s3_bucket_test_file("wps-process-test-bucket", input_file_s3) + test_bucket_ref, _ = mocked_aws_s3_bucket_test_file("wps-process-test-bucket", input_file_s3) exec_body = { "mode": EXECUTE_MODE_ASYNC, "response": EXECUTE_RESPONSE_DOCUMENT, diff --git a/tests/test_utils.py b/tests/test_utils.py index 631602ef2..c947dd200 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -552,7 +552,7 @@ def test_fetch_file_remote_s3_bucket(): test_file_name = "test-file.txt" test_file_data = "dummy file" test_bucket_name = "test-fake-bucket" - test_bucket_ref = mocked_aws_s3_bucket_test_file(test_bucket_name, test_file_name, test_file_data) + test_bucket_ref, _ = mocked_aws_s3_bucket_test_file(test_bucket_name, test_file_name, test_file_data) result = fetch_file(test_bucket_ref, tmpdir) assert result == os.path.join(tmpdir, test_file_name) assert os.path.isfile(result) diff --git a/tests/utils.py b/tests/utils.py index 3553362fc..1961da257 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -34,7 +34,7 @@ from weaver.warning import MissingParameterWarning, UnsupportedOperationWarning if TYPE_CHECKING: - from typing import Any, Callable, Iterable, List, Optional, Type, Union + from typing import Any, Callable, Iterable, List, Optional, Tuple, Type, Union import botocore.client # noqa from owslib.wps import Process as ProcessOWSWPS @@ -526,7 +526,7 @@ def wrapped(*args, **kwargs): def mocked_aws_s3_bucket_test_file(bucket_name, file_name, file_content="Test file inside test S3 bucket"): - # type: (str,str, str) -> str + # type: (str,str, str) -> Tuple[str, str] """ Generates a test file reference from dummy data that will be uploaded to the specified S3 bucket name using the provided file key. @@ -548,17 +548,24 @@ def mocked_aws_s3_bucket_test_file(bucket_name, file_name, file_content="Test fi with tempfile.NamedTemporaryFile(mode="w") as tmp_file: tmp_file.write(file_content) tmp_file.flush() - s3.upload_file(Bucket=bucket_name, Filename=tmp_file.name, Key=file_name) - return "s3://{}/{}".format(bucket_name, file_name) + tmp_path = tmp_file.name + s3.upload_file(Bucket=bucket_name, Filename=tmp_path, Key=file_name) + tmp_href = "s3://{}/{}".format(bucket_name, file_name) + return tmp_href, tmp_path def mocked_http_file(test_func): # type: (Callable[[...], Any]) -> Callable """ Creates a mock of the function :func:`fetch_file`, to fetch a generated file locally, for test purposes only. + For instance, calling this function with :func:`mocked_http_file` decorator will effectively employ the mocked :func:`fetch_file` and return a generated local file. + Only file references that employ :py:data:`MOCK_HTTP_REF` as prefix will be mocked during file fetching step. + These kind of references can be obtained using the utility function :func:`mocked_reference_test_file`. + Other references will employ usual fetching methodology. + .. seealso:: - :func:`mocked_reference_test_file` """ @@ -574,16 +581,26 @@ def wrapped(*args, **kwargs): return wrapped -def mocked_reference_test_file(file_name, href_type, file_content="This is a generated file for href test"): - # type: (str,str,str) -> str +def mocked_reference_test_file(file_name, href_type, file_content="test data", stack_context=None): + # type: (str, str, str, Optional[contextlib.ExitStack]) -> Tuple[str, str] """ Generates a test file reference from dummy data for http and file href types. + :param file_name: name of the temporary file to generate. + :param href_type: schema type of href (file, http, etc.) + :param file_content: contents of the temporary file. + :param stack_context: active context for auto-cleanup of temporary directly when closed. + :return: tuple of the generated reference and real temporary file path as (href, path) + .. seealso:: - :func:`mocked_http_file` """ - tmpdir = tempfile.mkdtemp() - path = os.path.join(tmpdir, file_name) + if isinstance(stack_context, contextlib.ExitStack): + tmpdir = stack_context.enter_context(tempfile.TemporaryDirectory()) + else: + tmpdir = tempfile.mkdtemp() + path = os.path.abspath(os.path.join(tmpdir, file_name)) with open(path, "w") as tmp_file: tmp_file.write(file_content) - return "file://{}".format(path) if href_type == "file" else os.path.join(MOCK_HTTP_REF, path) + href = "file://{}".format(path) if href_type == "file" else (MOCK_HTTP_REF + path) + return href, path