remove notification_email job listing filter parameter not supported …

…by aggreation pipeline with non-deterministic decrypt values

CHANGES.rst

@@ -17,11 +17,13 @@ Changes:
 - Modify the ``encrypt_email`` function to use an alternate strategy allowing ``decrypt_email`` on `Job` completed.
 - Add `CLI` ``execute`` options ``--output-public/-oP`` and ``--output-context/-oC OUTPUT_CONTEXT`` that add the
   specified ``X-WPS-Output-Context`` header to request the relevant output storage location of `Job` results.
+- Remove ``notification_email`` from ``GET /jobs`` query parameters.
+  Due to the nature of the encryption strategy, this cannot be supported anymore.
 
 Fixes:
 ------
-- Fix `Job` submitted email encryption not reversible to retrieve the original notification email on completion
-  (fixes `#568 <https://github.com/crim-ca/weaver/issues/568>`_).
+- Fix `Job` submitted with a ``notification_email`` not reversible from its encrypted value to retrieve the original
+  email on `Job` completion to send the notification (fixes `#568 <https://github.com/crim-ca/weaver/issues/568>`_).
 - Fix example Mako Template for email notification using an unavailable property ``${logs}``.
   Instead, the new utility methods ``job.[...]_url`` should be used to retrieve relevant locations.
 

tests/wps_restapi/test_jobs.py

@@ -1,4 +1,5 @@
 import contextlib
+import copy
 import datetime
 import logging
 import os
@@ -34,6 +35,7 @@
 from weaver.datatype import Job, Service
 from weaver.execute import ExecuteMode, ExecuteResponse, ExecuteTransmissionMode
 from weaver.formats import ContentType
+from weaver.notify import decrypt_email
 from weaver.processes.wps_testing import WpsTestProcess
 from weaver.status import JOB_STATUS_CATEGORIES, Status, StatusCategory
 from weaver.utils import get_path_kvp, now
@@ -543,6 +545,7 @@ def test_get_jobs_page_out_of_range(self):
         assert "limit" in str(resp.json["cause"]) and "less than minimum" in str(resp.json["cause"])
         assert "limit" in resp.json["value"] and resp.json["value"]["limit"] == str(0)
 
+    @pytest.mark.skip(reason="Obsolete feature. It is not possible to filter by encrypted notification email anymore.")
     def test_get_jobs_by_encrypted_email(self):
         """
         Verifies that literal email can be used as search criterion although not saved in plain text within db.
@@ -562,13 +565,20 @@ def test_get_jobs_by_encrypted_email(self):
             resp = self.app.post_json(path, params=body, headers=self.json_headers)
             assert resp.status_code == 201
             assert resp.content_type == ContentType.APP_JSON
-        job_id = resp.json["jobID"]
+            job_id = resp.json["jobID"]
+
+            # submit a second job just to make sure email doesn't match it as well
+            other_body = copy.deepcopy(body)
+            other_body["notification_email"] = "[email protected]"
+            resp = self.app.post_json(path, params=other_body, headers=self.json_headers)
+            assert resp.status_code == 201
 
         # verify the email is not in plain text
         job = self.job_store.fetch_by_id(job_id)
         assert job.notification_email != email and job.notification_email is not None
-        assert int(job.notification_email, 16) != 0  # email should be encrypted with hex string
+        assert decrypt_email(job.notification_email, self.settings) == email, "Email should be recoverable."
 
+        # make sure that jobs searched using email are found with encryption transparently for the user
         path = get_path_kvp(sd.jobs_service.path, detail="true", notification_email=email)
         resp = self.app.get(path, headers=self.json_headers)
         assert resp.status_code == 200

weaver/store/base.py

@@ -217,7 +217,6 @@ def find_jobs(self,
                   job_type=None,            # type: Optional[str]
                   tags=None,                # type: Optional[List[str]]
                   access=None,              # type: Optional[str]
-                  notification_email=None,  # type: Optional[str]
                   status=None,              # type: Optional[AnyStatusSearch, List[AnyStatusSearch]]
                   sort=None,                # type: Optional[AnySortType]
                   page=0,                   # type: Optional[int]

weaver/store/mongodb.py

@@ -78,12 +78,19 @@
         AnyVersion,
         ExecutionInputs,
         ExecutionOutputs,
-        JSON
+        JSON,
+        SettingsType
     )
     from weaver.visibility import AnyVisibility
 
     MongodbValue = Union[AnyValueType, datetime.datetime]
-    MongodbAggregateValue = Union[MongodbValue, List[MongodbValue], Dict[str, AnyValueType], List[AnyValueType]]
+    MongodbAggregateValue = Union[
+        MongodbValue,
+        List[MongodbValue],
+        List[AnyValueType],
+        Dict[str, AnyValueType],
+        Dict[str, List[AnyValueType]],
+    ]
     MongodbAggregateSortOrder = Dict[str, int]
     MongodbAggregateSortExpression = TypedDict("MongodbAggregateSortExpression", {
         "$sort": MongodbAggregateSortOrder,
@@ -553,7 +560,7 @@ def list_processes(self,
         if visibility is None:
             visibility = Visibility.values()
         if not isinstance(visibility, list):
-            visibility = [visibility]
+            visibility = [visibility]  # type: List[str]
         for v in visibility:
             vis = Visibility.get(v)
             if vis not in Visibility:
@@ -922,7 +929,6 @@ def find_jobs(self,
                   job_type=None,            # type: Optional[str]
                   tags=None,                # type: Optional[List[str]]
                   access=None,              # type: Optional[str]
-                  notification_email=None,  # type: Optional[str]
                   status=None,              # type: Optional[AnyStatusSearch, List[AnyStatusSearch]]
                   sort=None,                # type: Optional[AnySortType]
                   page=0,                   # type: Optional[int]
@@ -973,7 +979,6 @@ def find_jobs(self,
         :param job_type: filter matching jobs for given type.
         :param tags: list of tags to filter matching jobs.
         :param access: access visibility to filter matching jobs (default: :py:data:`Visibility.PUBLIC`).
-        :param notification_email: notification email to filter matching jobs.
         :param status: status to filter matching jobs.
         :param sort: field which is used for sorting results (default: creation date, descending).
         :param page: page number to return when using result paging (only when not using ``group_by``).
@@ -985,9 +990,6 @@ def find_jobs(self,
         :returns: (list of jobs matching paging OR list of {categories, list of jobs, count}) AND total of matched job.
         """
         search_filters = {}
-        if notification_email is not None:
-            search_filters["notification_email"] = notification_email
-
         search_filters.update(self._apply_status_filter(status))
         search_filters.update(self._apply_ref_or_type_filter(job_type, process, service))
         search_filters.update(self._apply_tags_filter(tags))

weaver/wps_restapi/jobs/jobs.py

@@ -74,10 +74,6 @@ def get_queried_jobs(request):
     detail = filters.pop("detail", False)
     groups = filters.pop("groups", None)
     filters["status"] = filters["status"].split(",") if "status" in filters else None
-    filters["notification_email"] = (
-        encrypt_email(filters["notification_email"], settings)
-        if filters.get("notification_email", False) else None
-    )
     filters["min_duration"] = filters.pop("minDuration", None)
     filters["max_duration"] = filters.pop("maxDuration", None)
     filters["job_type"] = filters.pop("type", None)

weaver/wps_restapi/swagger_definitions.py

@@ -5542,7 +5542,6 @@ class GetJobsQueries(PagingQueries):
                        description="Filter jobs only to matching type (note: 'service' and 'provider' are aliases).")
     sort = JobSortEnum(missing=drop)
     access = JobAccess(missing=drop, default=None)
-    notification_email = ExtendedSchemaNode(String(), missing=drop, validator=Email())
     tags = JobTagsCommaSeparated()