From b769bbc300dd47a9c8c367429be8f0622ff84b3b Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 10:44:19 -0400 Subject: [PATCH 01/22] Added jobs validation and limit parameter against ogcapi standards (fixes #237) --- tests/utils.py | 10 +++ tests/wps_restapi/test_jobs.py | 104 ++++++++++++++++++++-- weaver/store/__init__.py | 27 ++++++ weaver/store/base.py | 1 + weaver/store/mongodb.py | 30 ++++++- weaver/wps_restapi/jobs/jobs.py | 48 +++++----- weaver/wps_restapi/swagger_definitions.py | 28 ++++-- 7 files changed, 213 insertions(+), 35 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 614bf3014..82d06b14e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,6 +2,7 @@ Utility methods for various TestCase setup operations. """ import contextlib +from datetime import date import os import tempfile import uuid @@ -549,3 +550,12 @@ def mocked_aws_s3_bucket_test_file(bucket_name, file_name, file_content="Test fi tmp_file.flush() s3.upload_file(Bucket=bucket_name, Filename=tmp_file.name, Key=file_name) return "s3://{}/{}".format(bucket_name, file_name) + + +def generate_test_datetimes(): + # type: () -> List[str] + """ + Generates a list of dummy datetimes for testing. + """ + year = date.today().year + 1 + return ["{}-0{}-02T03:32:38.487000+00:00".format(year, month) for month in range(1,5)] \ No newline at end of file diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index e7d967da6..49ca9a889 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -1,4 +1,5 @@ import contextlib +from dateutil import parser as dateparser import json import unittest import warnings @@ -18,7 +19,8 @@ setup_config_with_mongodb, setup_mongodb_jobstore, setup_mongodb_processstore, - setup_mongodb_servicestore + setup_mongodb_servicestore, + generate_test_datetimes ) from weaver.datatype import Job, Service from weaver.execute import EXECUTE_MODE_ASYNC, EXECUTE_RESPONSE_DOCUMENT, EXECUTE_TRANSMISSION_MODE_REFERENCE @@ -31,6 +33,7 @@ STATUS_FAILED, STATUS_SUCCEEDED ) +from weaver.store import DATETIME_INTERVAL_CLOSED_SYMBOL, DATETIME_INTERVAL_OPEN_START_SYMBOL, DATETIME_INTERVAL_OPEN_END_SYMBOL from weaver.utils import get_path_kvp from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC from weaver.warning import TimeZoneInfoAlreadySetWarning @@ -39,6 +42,7 @@ if TYPE_CHECKING: from typing import Iterable, List, Tuple, Union +TEST_DATE_INTERVALL = generate_test_datetimes() class WpsRestApiJobsTest(unittest.TestCase): @classmethod @@ -95,21 +99,21 @@ def setUp(self): user_id=self.user_admin_id, status=STATUS_FAILED, progress=55, access=VISIBILITY_PRIVATE) # job public/private service/process combinations self.make_job(task_id="5555-5555-5555-5555", - process=self.process_public.identifier, service=self.service_public.name, + process=self.process_public.identifier, service=self.service_public.name, created_date=TEST_DATE_INTERVALL[0], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) self.make_job(task_id="6666-6666-6666-6666", - process=self.process_private.identifier, service=self.service_public.name, + process=self.process_private.identifier, service=self.service_public.name, created_date=TEST_DATE_INTERVALL[1], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) self.make_job(task_id="7777-7777-7777-7777", - process=self.process_public.identifier, service=self.service_private.name, + process=self.process_public.identifier, service=self.service_private.name, created_date=TEST_DATE_INTERVALL[2], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) self.make_job(task_id="8888-8888-8888-8888", - process=self.process_private.identifier, service=self.service_private.name, + process=self.process_private.identifier, service=self.service_private.name, created_date=TEST_DATE_INTERVALL[3], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) - def make_job(self, task_id, process, service, user_id, status, progress, access): + def make_job(self, task_id, process, service, user_id, status, progress, access, created_date=None): job = self.job_store.save_job(task_id=task_id, process=process, service=service, is_workflow=False, - user_id=user_id, execute_async=True, access=access) + user_id=user_id, execute_async=True, access=access, created_date=created_date) job.status = status if status in JOB_STATUS_CATEGORIES[STATUS_CATEGORY_FINISHED]: job.mark_finished() @@ -503,3 +507,89 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] job_match = all(job in job_ids for job in resp.json["jobs"]) test_values = dict(path=path, access=access, user_id=user_id) assert job_match, self.message_with_jobs_diffs(resp.json["jobs"], job_ids, test_values, index=i) + + def test_jobs_list_with_limit_client(self): + limit_parameter = 20 + path = get_path_kvp(sd.jobs_service.path, limit=limit_parameter) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert "limit" in resp.json and isinstance(resp.json["limit"], int) + assert resp.json["limit"] == limit_parameter + assert len(resp.json["jobs"]) <= limit_parameter + + def test_jobs_list_with_limit_api(self): + resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert "limit" in resp.json and isinstance(resp.json["limit"], int) + assert len(resp.json["jobs"]) <= resp.json["limit"] + + def test_not_required_fields(self): + uri = sd.openapi_json_service.path + resp = self.app.get(uri, headers=self.json_headers) + assert not resp.json["parameters"]["page"]["required"] + assert not resp.json["parameters"]["limit"]["required"] + + def test_jobs_datetime_before(self): + datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL+TEST_DATE_INTERVALL[0] + path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_before) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert len(resp.json["jobs"]) == 4 + for job in resp.json["jobs"]: + base_uri = sd.jobs_service.path + "/{}".format(job) + path = get_path_kvp(base_uri) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert dateparser.parse(resp.json["created"]) <= dateparser.parse(datetime_before.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL,"")) + + def test_jobs_datetime_after(self): + datetime_after = str(TEST_DATE_INTERVALL[2]+DATETIME_INTERVAL_OPEN_END_SYMBOL) + path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_after) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert len(resp.json["jobs"]) == 2 + for job in resp.json["jobs"]: + base_uri = sd.jobs_service.path + "/{}".format(job) + path = get_path_kvp(base_uri) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert dateparser.parse(resp.json["created"]) >= dateparser.parse(datetime_after.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL,"")) + + def test_jobs_datetime_interval(self): + datetime_interval = TEST_DATE_INTERVALL[1]+DATETIME_INTERVAL_CLOSED_SYMBOL+TEST_DATE_INTERVALL[3] + path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_interval) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + + datetime_after, datetime_before = datetime_interval.split(DATETIME_INTERVAL_CLOSED_SYMBOL) + assert len(resp.json["jobs"]) == 3 + for job in resp.json["jobs"]: + base_uri = sd.jobs_service.path + "/{}".format(job) + path = get_path_kvp(base_uri) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert dateparser.parse(resp.json["created"]) >= dateparser.parse(datetime_after) + assert dateparser.parse(resp.json["created"]) <= dateparser.parse(datetime_before) + + def test_jobs_datetime_match(self): + datetime_match = TEST_DATE_INTERVALL[1] + path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_match) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert len(resp.json["jobs"]) == 1 + for job in resp.json["jobs"]: + base_uri = sd.jobs_service.path + "/{}".format(job) + path = get_path_kvp(base_uri) + resp = self.app.get(path, headers=self.json_headers) + assert resp.status_code == 200 + assert resp.content_type == CONTENT_TYPE_APP_JSON + assert dateparser.parse(resp.json["created"]) == dateparser.parse(datetime_match) diff --git a/weaver/store/__init__.py b/weaver/store/__init__.py index e69de29bb..c86a82d0a 100644 --- a/weaver/store/__init__.py +++ b/weaver/store/__init__.py @@ -0,0 +1,27 @@ +from dateutil import parser as dateparser + +DATETIME_INTERVAL_CLOSED_SYMBOL = "/" +DATETIME_INTERVAL_OPEN_START_SYMBOL = "../" +DATETIME_INTERVAL_OPEN_END_SYMBOL = "/.." + + +def datetime_interval_parser(datetime_interval): + + parsed_datetime = {} + + if datetime_interval.startswith(DATETIME_INTERVAL_OPEN_START_SYMBOL): + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL,'') + parsed_datetime["before"] = dateparser.parse(datetime_interval) + + elif datetime_interval.endswith(DATETIME_INTERVAL_OPEN_END_SYMBOL): + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL,'') + parsed_datetime["after"] = dateparser.parse(datetime_interval) + + elif DATETIME_INTERVAL_CLOSED_SYMBOL in datetime_interval: + datetime_interval = datetime_interval.split(DATETIME_INTERVAL_CLOSED_SYMBOL) + parsed_datetime["after"] = dateparser.parse(datetime_interval[0]) + parsed_datetime["before"] = dateparser.parse(datetime_interval[-1]) + else: + parsed_datetime["match"] = dateparser.parse(datetime_interval) + + return parsed_datetime \ No newline at end of file diff --git a/weaver/store/base.py b/weaver/store/base.py index d590c4a90..58a768745 100644 --- a/weaver/store/base.py +++ b/weaver/store/base.py @@ -145,6 +145,7 @@ def find_jobs(self, sort=None, # type: Optional[str] page=0, # type: int limit=10, # type: int + datetime_interval=None, # type: Optional[str] group_by=None, # type: Optional[Union[str, List[str]]] request=None, # type: Optional[Request] ): # type: (...) -> Union[JobListAndCount, JobCategoriesAndCount] diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 52931ec3e..7a2ef3ca4 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -2,6 +2,7 @@ Stores to read/write data to from/to `MongoDB` using pymongo. """ +from dateutil import parser as dateparser import logging from typing import TYPE_CHECKING @@ -42,6 +43,7 @@ SORT_USER ) from weaver.status import JOB_STATUS_CATEGORIES, STATUS_ACCEPTED, map_status +from weaver.store import datetime_interval_parser from weaver.store.base import StoreBills, StoreJobs, StoreProcesses, StoreQuotes, StoreServices from weaver.utils import get_base_url, get_sane_name, get_weaver_url, islambda, now from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC, VISIBILITY_VALUES @@ -405,6 +407,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] + created_date=None, # type: Optional[str] ): # type: (...) -> Job """ Creates a new :class:`Job` and stores it in mongodb. @@ -422,6 +425,9 @@ def save_job(self, tags.append(EXECUTE_MODE_SYNC) if not access: access = VISIBILITY_PRIVATE + + created = dateparser.parse(created_date) if created_date else now() + new_job = Job({ "task_id": task_id, "user_id": user_id, @@ -432,7 +438,7 @@ def save_job(self, "execute_async": execute_async, "is_workflow": is_workflow, "is_local": is_local, - "created": now(), + "created": created, "tags": list(set(tags)), # remove duplicates "access": access, "notification_email": notification_email, @@ -499,6 +505,7 @@ def find_jobs(self, sort=None, # type: Optional[str] page=0, # type: int limit=10, # type: int + datetime_interval=None, # type: Optional[str] group_by=None, # type: Optional[Union[str, List[str]]] request=None, # type: Optional[Request] ): # type: (...) -> Union[JobListAndCount, JobCategoriesAndCount] @@ -515,6 +522,7 @@ def find_jobs(self, :param sort: field which is used for sorting results (default: creation date, descending). :param page: page number to return when using result paging (only when not using ``group_by``). :param limit: number of jobs per page when using result paging (only when not using ``group_by``). + :param datetime_interval: field witch is used for filtering data by creation date with a given date or interval of date. :param group_by: one or many fields specifying categories to form matching groups of jobs (paging disabled). :returns: (list of jobs matching paging OR list of {categories, list of jobs, count}) AND total of matched job @@ -582,6 +590,26 @@ def find_jobs(self, if service is not None: search_filters["service"] = service + + if datetime_interval is not None: + datetime_interval = datetime_interval_parser(datetime_interval) + query = {} + + try: + if datetime_interval.get("after", False): + query["$gte"] = datetime_interval["after"] + + if datetime_interval.get("before", False): + query["$lte"] = datetime_interval["before"] + + if datetime_interval.get("match", False): + query = datetime_interval["match"] + + except Exception as ex: + raise JobRegistrationError("Error occurred during datetime job filtering: [{}]".format(repr(ex))) + + search_filters["created"] = query + if sort is None: sort = SORT_CREATED elif sort == SORT_USER: diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index cc138e183..34794b528 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -194,28 +194,36 @@ def get_queried_jobs(request): """ Retrieve the list of jobs which can be filtered, sorted, paged and categorized using query parameters. """ + settings = get_settings(request) service, process = validate_service_process(request) - detail = asbool(request.params.get("detail", False)) - page = request.params.get("page", "0") - page = int(page) if str.isnumeric(page) else 0 - limit = request.params.get("limit", "10") - limit = int(limit) if str.isnumeric(limit) else 10 - email = request.params.get("notification_email", None) - filters = { - "page": page, - "limit": limit, - # split by comma and filter empty stings - "tags": list(filter(lambda s: s, request.params.get("tags", "").split(","))), - "access": request.params.get("access", None), - "status": request.params.get("status", None), - "sort": request.params.get("sort", sort.SORT_CREATED), - "notification_email": encrypt_email(email, settings) if email else None, - # service and process can be specified by query (short route) or by path (full route) - "process": process, - "service": service, - } - groups = request.params.get("groups", "") + + filters = {**request.params, "process": process} + + filters["detail"] = asbool(request.params.get("detail")) + + if request.params.get("notification_email", False): + filters["notification_email"] = request.params["notification_email"] + + if request.params.get("datetime_interval", False): + filters["datetime_interval"] = request.params["datetime_interval"].replace(' ','+') + + filters = sd.GetJobsQueries().deserialize(filters) + + limit = filters["limit"] + page = filters["page"] + detail = asbool(filters.get("detail", False)) + groups = filters["groups"] + + filters["tags"] = list(filter(lambda s: s, filters.get("tags").split(","))) + filters["notification_email"] = encrypt_email(filters["notification_email"], settings) if filters.get("notification_email", False) else None + filters["service"] = service + filters["access"] = request.params.get("access", None) + filters["status"] = request.params.get("status", None) + + del filters["detail"] + del filters["groups"] + groups = groups.split(",") if groups else None store = get_db(request).get_store(StoreJobs) items, total = store.find_jobs(request=request, group_by=groups, **filters) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 6b8504809..b901677c1 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -228,6 +228,18 @@ class URL(ExtendedSchemaNode): format = "url" +class DATETIME_INTERVAL(ExtendedSchemaNode): + schema_type = String + description = "DateTime name pattern." + + regex_datetime = r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?)" + regex_interval_closed = r"{i}\/{i}".format(i=regex_datetime) + regex_interval_open_start = r"\.\.\/{}".format(regex_datetime) + regex_interval_open_end = r"{}\/\.\.".format(regex_datetime) + + pattern = "^{}|{}|{}|{}$".format(regex_datetime, regex_interval_closed, regex_interval_open_start, regex_interval_open_end) + + class S3Bucket(ExtendedSchemaNode): schema_type = String description = "S3 bucket shorthand URL representation [s3:////.ext]" @@ -1947,7 +1959,7 @@ class CreatedQuotedJobStatusSchema(CreatedJobStatusSchema): class GetPagingJobsSchema(ExtendedMappingSchema): jobs = JobCollection() - limit = ExtendedSchemaNode(Integer(), default=10) + limit = ExtendedSchemaNode(Integer(), missing=10, default=10, validator=Range(min=0, max=10000)) page = ExtendedSchemaNode(Integer(), validator=Range(min=0)) @@ -2788,17 +2800,19 @@ class PostProcessJobsEndpoint(ProcessPath): class GetJobsQueries(ExtendedMappingSchema): detail = ExtendedSchemaNode(Boolean(), description="Provide job details instead of IDs.", - default=False, example=True, missing=drop) + default=False, example=True, missing=False) groups = ExtendedSchemaNode(String(), description="Comma-separated list of grouping fields with which to list jobs.", - default=False, example="process,service", missing=drop) - page = ExtendedSchemaNode(Integer(), missing=drop, default=0, validator=Range(min=0)) - limit = ExtendedSchemaNode(Integer(), missing=drop, default=10) + default=False, example="process,service", missing=False) + page = ExtendedSchemaNode(Integer(), missing=0, default=0, validator=Range(min=0)) + limit = ExtendedSchemaNode(Integer(), missing=10, default=10, validator=Range(min=0, max=10000)) + datetime_interval = DATETIME_INTERVAL(missing=None, default=None) status = JobStatusEnum(missing=drop) process = AnyIdentifier(missing=None) provider = ExtendedSchemaNode(String(), missing=drop, default=None) sort = JobSortEnum(missing=drop) - tags = ExtendedSchemaNode(String(), missing=drop, default=None, + notification_email = ExtendedSchemaNode(String(), missing=drop, validator=Email()) + tags = ExtendedSchemaNode(String(), missing="", default="", description="Comma-separated values of tags assigned to jobs") @@ -2845,7 +2859,7 @@ class ProcessQuoteEndpoint(ProcessPath, QuotePath): class GetQuotesQueries(ExtendedMappingSchema): page = ExtendedSchemaNode(Integer(), missing=drop, default=0) - limit = ExtendedSchemaNode(Integer(), missing=drop, default=10) + limit = ExtendedSchemaNode(Integer(), missing=10, default=10, validator=Range(min=0, max=10000)) process = AnyIdentifier(missing=None) sort = QuoteSortEnum(missing=drop) From ea9c888700b0f13becf8ce45ce1c8cfe99811088 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 10:58:01 -0400 Subject: [PATCH 02/22] pep8 refactor --- tests/utils.py | 2 +- tests/wps_restapi/test_jobs.py | 18 ++++++++++-------- weaver/store/__init__.py | 12 ++++++------ weaver/store/mongodb.py | 5 ++--- weaver/wps_restapi/jobs/jobs.py | 5 +++-- weaver/wps_restapi/swagger_definitions.py | 3 ++- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 82d06b14e..d801a05e6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -558,4 +558,4 @@ def generate_test_datetimes(): Generates a list of dummy datetimes for testing. """ year = date.today().year + 1 - return ["{}-0{}-02T03:32:38.487000+00:00".format(year, month) for month in range(1,5)] \ No newline at end of file + return ["{}-0{}-02T03:32:38.487000+00:00".format(year, month) for month in range(1, 5)] diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index 49ca9a889..705bf69b8 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -540,11 +540,12 @@ def test_jobs_datetime_before(self): assert len(resp.json["jobs"]) == 4 for job in resp.json["jobs"]: base_uri = sd.jobs_service.path + "/{}".format(job) - path = get_path_kvp(base_uri) + path = get_path_kvp(base_uri) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON - assert dateparser.parse(resp.json["created"]) <= dateparser.parse(datetime_before.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL,"")) + assert dateparser.parse(resp.json["created"]) <= dateparser.parse( + datetime_before.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, "")) def test_jobs_datetime_after(self): datetime_after = str(TEST_DATE_INTERVALL[2]+DATETIME_INTERVAL_OPEN_END_SYMBOL) @@ -555,12 +556,13 @@ def test_jobs_datetime_after(self): assert len(resp.json["jobs"]) == 2 for job in resp.json["jobs"]: base_uri = sd.jobs_service.path + "/{}".format(job) - path = get_path_kvp(base_uri) + path = get_path_kvp(base_uri) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON - assert dateparser.parse(resp.json["created"]) >= dateparser.parse(datetime_after.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL,"")) - + assert dateparser.parse(resp.json["created"]) >= dateparser.parse( + datetime_after.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, "")) + def test_jobs_datetime_interval(self): datetime_interval = TEST_DATE_INTERVALL[1]+DATETIME_INTERVAL_CLOSED_SYMBOL+TEST_DATE_INTERVALL[3] path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_interval) @@ -572,13 +574,13 @@ def test_jobs_datetime_interval(self): assert len(resp.json["jobs"]) == 3 for job in resp.json["jobs"]: base_uri = sd.jobs_service.path + "/{}".format(job) - path = get_path_kvp(base_uri) + path = get_path_kvp(base_uri) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON assert dateparser.parse(resp.json["created"]) >= dateparser.parse(datetime_after) assert dateparser.parse(resp.json["created"]) <= dateparser.parse(datetime_before) - + def test_jobs_datetime_match(self): datetime_match = TEST_DATE_INTERVALL[1] path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_match) @@ -588,7 +590,7 @@ def test_jobs_datetime_match(self): assert len(resp.json["jobs"]) == 1 for job in resp.json["jobs"]: base_uri = sd.jobs_service.path + "/{}".format(job) - path = get_path_kvp(base_uri) + path = get_path_kvp(base_uri) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON diff --git a/weaver/store/__init__.py b/weaver/store/__init__.py index c86a82d0a..aff3ec04f 100644 --- a/weaver/store/__init__.py +++ b/weaver/store/__init__.py @@ -1,7 +1,7 @@ from dateutil import parser as dateparser DATETIME_INTERVAL_CLOSED_SYMBOL = "/" -DATETIME_INTERVAL_OPEN_START_SYMBOL = "../" +DATETIME_INTERVAL_OPEN_START_SYMBOL = "../" DATETIME_INTERVAL_OPEN_END_SYMBOL = "/.." @@ -10,13 +10,13 @@ def datetime_interval_parser(datetime_interval): parsed_datetime = {} if datetime_interval.startswith(DATETIME_INTERVAL_OPEN_START_SYMBOL): - datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL,'') + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, '') parsed_datetime["before"] = dateparser.parse(datetime_interval) - + elif datetime_interval.endswith(DATETIME_INTERVAL_OPEN_END_SYMBOL): - datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL,'') + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, '') parsed_datetime["after"] = dateparser.parse(datetime_interval) - + elif DATETIME_INTERVAL_CLOSED_SYMBOL in datetime_interval: datetime_interval = datetime_interval.split(DATETIME_INTERVAL_CLOSED_SYMBOL) parsed_datetime["after"] = dateparser.parse(datetime_interval[0]) @@ -24,4 +24,4 @@ def datetime_interval_parser(datetime_interval): else: parsed_datetime["match"] = dateparser.parse(datetime_interval) - return parsed_datetime \ No newline at end of file + return parsed_datetime diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 7a2ef3ca4..52a25d819 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -590,7 +590,6 @@ def find_jobs(self, if service is not None: search_filters["service"] = service - if datetime_interval is not None: datetime_interval = datetime_interval_parser(datetime_interval) query = {} @@ -601,13 +600,13 @@ def find_jobs(self, if datetime_interval.get("before", False): query["$lte"] = datetime_interval["before"] - + if datetime_interval.get("match", False): query = datetime_interval["match"] except Exception as ex: raise JobRegistrationError("Error occurred during datetime job filtering: [{}]".format(repr(ex))) - + search_filters["created"] = query if sort is None: diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index 34794b528..d5d0df335 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -206,7 +206,7 @@ def get_queried_jobs(request): filters["notification_email"] = request.params["notification_email"] if request.params.get("datetime_interval", False): - filters["datetime_interval"] = request.params["datetime_interval"].replace(' ','+') + filters["datetime_interval"] = request.params["datetime_interval"].replace(' ', '+') filters = sd.GetJobsQueries().deserialize(filters) @@ -216,7 +216,8 @@ def get_queried_jobs(request): groups = filters["groups"] filters["tags"] = list(filter(lambda s: s, filters.get("tags").split(","))) - filters["notification_email"] = encrypt_email(filters["notification_email"], settings) if filters.get("notification_email", False) else None + filters["notification_email"] = encrypt_email( + filters["notification_email"], settings) if filters.get("notification_email", False) else None filters["service"] = service filters["access"] = request.params.get("access", None) filters["status"] = request.params.get("status", None) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index b901677c1..8f02290b1 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -237,7 +237,8 @@ class DATETIME_INTERVAL(ExtendedSchemaNode): regex_interval_open_start = r"\.\.\/{}".format(regex_datetime) regex_interval_open_end = r"{}\/\.\.".format(regex_datetime) - pattern = "^{}|{}|{}|{}$".format(regex_datetime, regex_interval_closed, regex_interval_open_start, regex_interval_open_end) + pattern = "^{}|{}|{}|{}$".format(regex_datetime, regex_interval_closed, + regex_interval_open_start, regex_interval_open_end) class S3Bucket(ExtendedSchemaNode): From 50a4fa7487ac69428c9a781338d15364c9df1bc0 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 11:13:54 -0400 Subject: [PATCH 03/22] pep8 --- tests/wps_restapi/test_jobs.py | 18 +++++++++--------- weaver/store/mongodb.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index 705bf69b8..cc1703386 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -98,17 +98,17 @@ def setUp(self): self.make_job(task_id="4444-4444-4444-4444", process=self.process_public.identifier, service=None, user_id=self.user_admin_id, status=STATUS_FAILED, progress=55, access=VISIBILITY_PRIVATE) # job public/private service/process combinations - self.make_job(task_id="5555-5555-5555-5555", - process=self.process_public.identifier, service=self.service_public.name, created_date=TEST_DATE_INTERVALL[0], + self.make_job(task_id="5555-5555-5555-5555", process=self.process_public.identifier, + service=self.service_public.name, created_date=TEST_DATE_INTERVALL[0], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) - self.make_job(task_id="6666-6666-6666-6666", - process=self.process_private.identifier, service=self.service_public.name, created_date=TEST_DATE_INTERVALL[1], + self.make_job(task_id="6666-6666-6666-6666",process=self.process_private.identifier, + service=self.service_public.name, created_date=TEST_DATE_INTERVALL[1], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) - self.make_job(task_id="7777-7777-7777-7777", - process=self.process_public.identifier, service=self.service_private.name, created_date=TEST_DATE_INTERVALL[2], + self.make_job(task_id="7777-7777-7777-7777",process=self.process_public.identifier, + service=self.service_private.name, created_date=TEST_DATE_INTERVALL[2], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) - self.make_job(task_id="8888-8888-8888-8888", - process=self.process_private.identifier, service=self.service_private.name, created_date=TEST_DATE_INTERVALL[3], + self.make_job(task_id="8888-8888-8888-8888", process=self.process_private.identifier, + service=self.service_private.name, created_date=TEST_DATE_INTERVALL[3], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) def make_job(self, task_id, process, service, user_id, status, progress, access, created_date=None): @@ -524,7 +524,7 @@ def test_jobs_list_with_limit_api(self): assert resp.content_type == CONTENT_TYPE_APP_JSON assert "limit" in resp.json and isinstance(resp.json["limit"], int) assert len(resp.json["jobs"]) <= resp.json["limit"] - + def test_not_required_fields(self): uri = sd.openapi_json_service.path resp = self.app.get(uri, headers=self.json_headers) diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 52a25d819..d720de494 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -522,7 +522,7 @@ def find_jobs(self, :param sort: field which is used for sorting results (default: creation date, descending). :param page: page number to return when using result paging (only when not using ``group_by``). :param limit: number of jobs per page when using result paging (only when not using ``group_by``). - :param datetime_interval: field witch is used for filtering data by creation date with a given date or interval of date. + :param datetime_interval: field used for filtering data by creation date with a given date or interval of date. :param group_by: one or many fields specifying categories to form matching groups of jobs (paging disabled). :returns: (list of jobs matching paging OR list of {categories, list of jobs, count}) AND total of matched job From bed97a9fc7d4f5dd711e27db55d19f65a83534c4 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 11:21:37 -0400 Subject: [PATCH 04/22] pep8 --- tests/wps_restapi/test_jobs.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index cc1703386..d31f46e10 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -33,7 +33,11 @@ STATUS_FAILED, STATUS_SUCCEEDED ) -from weaver.store import DATETIME_INTERVAL_CLOSED_SYMBOL, DATETIME_INTERVAL_OPEN_START_SYMBOL, DATETIME_INTERVAL_OPEN_END_SYMBOL +from weaver.store import ( + DATETIME_INTERVAL_CLOSED_SYMBOL, + DATETIME_INTERVAL_OPEN_START_SYMBOL, + DATETIME_INTERVAL_OPEN_END_SYMBOL +) from weaver.utils import get_path_kvp from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC from weaver.warning import TimeZoneInfoAlreadySetWarning @@ -44,6 +48,7 @@ TEST_DATE_INTERVALL = generate_test_datetimes() + class WpsRestApiJobsTest(unittest.TestCase): @classmethod def setUpClass(cls): @@ -101,10 +106,10 @@ def setUp(self): self.make_job(task_id="5555-5555-5555-5555", process=self.process_public.identifier, service=self.service_public.name, created_date=TEST_DATE_INTERVALL[0], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) - self.make_job(task_id="6666-6666-6666-6666",process=self.process_private.identifier, + self.make_job(task_id="6666-6666-6666-6666", process=self.process_private.identifier, service=self.service_public.name, created_date=TEST_DATE_INTERVALL[1], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) - self.make_job(task_id="7777-7777-7777-7777",process=self.process_public.identifier, + self.make_job(task_id="7777-7777-7777-7777", process=self.process_public.identifier, service=self.service_private.name, created_date=TEST_DATE_INTERVALL[2], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) self.make_job(task_id="8888-8888-8888-8888", process=self.process_private.identifier, From caa8a3cb9a486da1a0f8d7450b1f0b8ff5611481 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 11:34:34 -0400 Subject: [PATCH 05/22] pep8 --- tests/wps_restapi/test_jobs.py | 2 +- weaver/store/__init__.py | 4 ++-- weaver/store/mongodb.py | 2 +- weaver/wps_restapi/jobs/jobs.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index d31f46e10..79994f7ba 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -1,11 +1,11 @@ import contextlib -from dateutil import parser as dateparser import json import unittest import warnings from collections import OrderedDict from distutils.version import LooseVersion from typing import TYPE_CHECKING +from dateutil import parser as dateparser import mock import pyramid.testing diff --git a/weaver/store/__init__.py b/weaver/store/__init__.py index aff3ec04f..75cd7898f 100644 --- a/weaver/store/__init__.py +++ b/weaver/store/__init__.py @@ -10,11 +10,11 @@ def datetime_interval_parser(datetime_interval): parsed_datetime = {} if datetime_interval.startswith(DATETIME_INTERVAL_OPEN_START_SYMBOL): - datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, '') + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, "") parsed_datetime["before"] = dateparser.parse(datetime_interval) elif datetime_interval.endswith(DATETIME_INTERVAL_OPEN_END_SYMBOL): - datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, '') + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, "") parsed_datetime["after"] = dateparser.parse(datetime_interval) elif DATETIME_INTERVAL_CLOSED_SYMBOL in datetime_interval: diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index d720de494..6e62d4f3b 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -2,9 +2,9 @@ Stores to read/write data to from/to `MongoDB` using pymongo. """ -from dateutil import parser as dateparser import logging from typing import TYPE_CHECKING +from dateutil import parser as dateparser import pymongo from pymongo import ASCENDING, DESCENDING diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index d5d0df335..d0fc1c079 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -7,7 +7,7 @@ from pyramid_celery import celery_app as app from notify import encrypt_email -from weaver import sort, status +from weaver import status from weaver.database import get_db from weaver.datatype import Job from weaver.exceptions import ( @@ -206,7 +206,7 @@ def get_queried_jobs(request): filters["notification_email"] = request.params["notification_email"] if request.params.get("datetime_interval", False): - filters["datetime_interval"] = request.params["datetime_interval"].replace(' ', '+') + filters["datetime_interval"] = request.params["datetime_interval"].replace(" ", "+") filters = sd.GetJobsQueries().deserialize(filters) From 7614275b1b37da8da6a1fd3ae7b022dbddd5d6bc Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 11:41:54 -0400 Subject: [PATCH 06/22] pep8 --- weaver/store/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/weaver/store/base.py b/weaver/store/base.py index 58a768745..aefe7d3c9 100644 --- a/weaver/store/base.py +++ b/weaver/store/base.py @@ -111,6 +111,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] + created_date=None, # type: Optional[str] ): # type: (...) -> Job raise NotImplementedError From ffbd4962b6c4547ae63ec28df282ac1f51b12bec Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 11:56:14 -0400 Subject: [PATCH 07/22] pep8 fix-imports --- tests/utils.py | 2 +- tests/wps_restapi/test_jobs.py | 10 +++++----- weaver/store/mongodb.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index d801a05e6..5b8d9f3a1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,12 +2,12 @@ Utility methods for various TestCase setup operations. """ import contextlib -from datetime import date import os import tempfile import uuid import warnings from configparser import ConfigParser +from datetime import date from inspect import isclass from typing import TYPE_CHECKING diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index 79994f7ba..a0a1d148b 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -5,13 +5,14 @@ from collections import OrderedDict from distutils.version import LooseVersion from typing import TYPE_CHECKING -from dateutil import parser as dateparser import mock import pyramid.testing import pytest +from dateutil import parser as dateparser from tests.utils import ( + generate_test_datetimes, get_module_version, get_test_weaver_app, mocked_process_job_runner, @@ -19,8 +20,7 @@ setup_config_with_mongodb, setup_mongodb_jobstore, setup_mongodb_processstore, - setup_mongodb_servicestore, - generate_test_datetimes + setup_mongodb_servicestore ) from weaver.datatype import Job, Service from weaver.execute import EXECUTE_MODE_ASYNC, EXECUTE_RESPONSE_DOCUMENT, EXECUTE_TRANSMISSION_MODE_REFERENCE @@ -35,8 +35,8 @@ ) from weaver.store import ( DATETIME_INTERVAL_CLOSED_SYMBOL, - DATETIME_INTERVAL_OPEN_START_SYMBOL, - DATETIME_INTERVAL_OPEN_END_SYMBOL + DATETIME_INTERVAL_OPEN_END_SYMBOL, + DATETIME_INTERVAL_OPEN_START_SYMBOL ) from weaver.utils import get_path_kvp from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 6e62d4f3b..47aba90ed 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -4,9 +4,9 @@ import logging from typing import TYPE_CHECKING -from dateutil import parser as dateparser import pymongo +from dateutil import parser as dateparser from pymongo import ASCENDING, DESCENDING from pymongo.errors import DuplicateKeyError from pyramid.request import Request From d36c4e18405ba8647f3446b86939a9b8a70fb6ed Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Tue, 25 May 2021 17:16:48 -0400 Subject: [PATCH 08/22] fix reviews part-1 --- tests/wps_restapi/test_jobs.py | 28 +++++++++------- weaver/store/__init__.py | 27 --------------- weaver/store/base.py | 2 +- weaver/store/mongodb.py | 8 ++--- weaver/wps_restapi/jobs/jobs.py | 16 ++++----- weaver/wps_restapi/swagger_definitions.py | 41 ++++++++++++++++++++--- 6 files changed, 61 insertions(+), 61 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index a0a1d148b..1986a9e0a 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -33,7 +33,7 @@ STATUS_FAILED, STATUS_SUCCEEDED ) -from weaver.store import ( +from weaver.wps_restapi.swagger_definitions import ( DATETIME_INTERVAL_CLOSED_SYMBOL, DATETIME_INTERVAL_OPEN_END_SYMBOL, DATETIME_INTERVAL_OPEN_START_SYMBOL @@ -46,8 +46,6 @@ if TYPE_CHECKING: from typing import Iterable, List, Tuple, Union -TEST_DATE_INTERVALL = generate_test_datetimes() - class WpsRestApiJobsTest(unittest.TestCase): @classmethod @@ -60,6 +58,7 @@ def setUpClass(cls): cls.config = setup_config_with_mongodb(settings=settings) cls.app = get_test_weaver_app(config=cls.config) cls.json_headers = {"Accept": CONTENT_TYPE_APP_JSON, "Content-Type": CONTENT_TYPE_APP_JSON} + cls.datetime_interval = generate_test_datetimes() @classmethod def tearDownClass(cls): @@ -104,21 +103,24 @@ def setUp(self): user_id=self.user_admin_id, status=STATUS_FAILED, progress=55, access=VISIBILITY_PRIVATE) # job public/private service/process combinations self.make_job(task_id="5555-5555-5555-5555", process=self.process_public.identifier, - service=self.service_public.name, created_date=TEST_DATE_INTERVALL[0], + service=self.service_public.name, created=self.datetime_interval[0], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) self.make_job(task_id="6666-6666-6666-6666", process=self.process_private.identifier, - service=self.service_public.name, created_date=TEST_DATE_INTERVALL[1], + service=self.service_public.name, created=self.datetime_interval[1], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) self.make_job(task_id="7777-7777-7777-7777", process=self.process_public.identifier, - service=self.service_private.name, created_date=TEST_DATE_INTERVALL[2], + service=self.service_private.name, created=self.datetime_interval[2], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) self.make_job(task_id="8888-8888-8888-8888", process=self.process_private.identifier, - service=self.service_private.name, created_date=TEST_DATE_INTERVALL[3], + service=self.service_private.name, created=self.datetime_interval[3], user_id=self.user_editor1_id, status=STATUS_FAILED, progress=99, access=VISIBILITY_PUBLIC) - def make_job(self, task_id, process, service, user_id, status, progress, access, created_date=None): + def make_job(self, task_id, process, service, user_id, status, progress, access, created=None): + + created = dateparser.parse(created) if created else None + job = self.job_store.save_job(task_id=task_id, process=process, service=service, is_workflow=False, - user_id=user_id, execute_async=True, access=access, created_date=created_date) + user_id=user_id, execute_async=True, access=access, created=created) job.status = status if status in JOB_STATUS_CATEGORIES[STATUS_CATEGORY_FINISHED]: job.mark_finished() @@ -537,7 +539,7 @@ def test_not_required_fields(self): assert not resp.json["parameters"]["limit"]["required"] def test_jobs_datetime_before(self): - datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL+TEST_DATE_INTERVALL[0] + datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL + self.datetime_interval[0] path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_before) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 @@ -553,7 +555,7 @@ def test_jobs_datetime_before(self): datetime_before.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, "")) def test_jobs_datetime_after(self): - datetime_after = str(TEST_DATE_INTERVALL[2]+DATETIME_INTERVAL_OPEN_END_SYMBOL) + datetime_after = str(self.datetime_interval[2] + DATETIME_INTERVAL_OPEN_END_SYMBOL) path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_after) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 @@ -569,7 +571,7 @@ def test_jobs_datetime_after(self): datetime_after.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, "")) def test_jobs_datetime_interval(self): - datetime_interval = TEST_DATE_INTERVALL[1]+DATETIME_INTERVAL_CLOSED_SYMBOL+TEST_DATE_INTERVALL[3] + datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL+self.datetime_interval[3] path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_interval) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 @@ -587,7 +589,7 @@ def test_jobs_datetime_interval(self): assert dateparser.parse(resp.json["created"]) <= dateparser.parse(datetime_before) def test_jobs_datetime_match(self): - datetime_match = TEST_DATE_INTERVALL[1] + datetime_match = self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_match) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 diff --git a/weaver/store/__init__.py b/weaver/store/__init__.py index 75cd7898f..e69de29bb 100644 --- a/weaver/store/__init__.py +++ b/weaver/store/__init__.py @@ -1,27 +0,0 @@ -from dateutil import parser as dateparser - -DATETIME_INTERVAL_CLOSED_SYMBOL = "/" -DATETIME_INTERVAL_OPEN_START_SYMBOL = "../" -DATETIME_INTERVAL_OPEN_END_SYMBOL = "/.." - - -def datetime_interval_parser(datetime_interval): - - parsed_datetime = {} - - if datetime_interval.startswith(DATETIME_INTERVAL_OPEN_START_SYMBOL): - datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, "") - parsed_datetime["before"] = dateparser.parse(datetime_interval) - - elif datetime_interval.endswith(DATETIME_INTERVAL_OPEN_END_SYMBOL): - datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, "") - parsed_datetime["after"] = dateparser.parse(datetime_interval) - - elif DATETIME_INTERVAL_CLOSED_SYMBOL in datetime_interval: - datetime_interval = datetime_interval.split(DATETIME_INTERVAL_CLOSED_SYMBOL) - parsed_datetime["after"] = dateparser.parse(datetime_interval[0]) - parsed_datetime["before"] = dateparser.parse(datetime_interval[-1]) - else: - parsed_datetime["match"] = dateparser.parse(datetime_interval) - - return parsed_datetime diff --git a/weaver/store/base.py b/weaver/store/base.py index aefe7d3c9..95f1919bb 100644 --- a/weaver/store/base.py +++ b/weaver/store/base.py @@ -111,7 +111,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] - created_date=None, # type: Optional[str] + created=None, # type: Optional[str] ): # type: (...) -> Job raise NotImplementedError diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 47aba90ed..799337b84 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -43,7 +43,7 @@ SORT_USER ) from weaver.status import JOB_STATUS_CATEGORIES, STATUS_ACCEPTED, map_status -from weaver.store import datetime_interval_parser +from weaver.wps_restapi.swagger_definitions import datetime_interval_parser from weaver.store.base import StoreBills, StoreJobs, StoreProcesses, StoreQuotes, StoreServices from weaver.utils import get_base_url, get_sane_name, get_weaver_url, islambda, now from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC, VISIBILITY_VALUES @@ -407,7 +407,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] - created_date=None, # type: Optional[str] + created=None, # type: Optional[str] ): # type: (...) -> Job """ Creates a new :class:`Job` and stores it in mongodb. @@ -426,8 +426,6 @@ def save_job(self, if not access: access = VISIBILITY_PRIVATE - created = dateparser.parse(created_date) if created_date else now() - new_job = Job({ "task_id": task_id, "user_id": user_id, @@ -438,7 +436,7 @@ def save_job(self, "execute_async": execute_async, "is_workflow": is_workflow, "is_local": is_local, - "created": created, + "created": created if created else now(), "tags": list(set(tags)), # remove duplicates "access": access, "notification_email": notification_email, diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index d0fc1c079..c90722158 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -202,18 +202,15 @@ def get_queried_jobs(request): filters["detail"] = asbool(request.params.get("detail")) - if request.params.get("notification_email", False): - filters["notification_email"] = request.params["notification_email"] - if request.params.get("datetime_interval", False): + # replace white space with '+' since request.params replaces '+' with whitespaces when parsing filters["datetime_interval"] = request.params["datetime_interval"].replace(" ", "+") filters = sd.GetJobsQueries().deserialize(filters) - limit = filters["limit"] - page = filters["page"] + detail = asbool(filters.get("detail", False)) - groups = filters["groups"] + groups = filters["groups"].split(",") if filters["groups"] else None filters["tags"] = list(filter(lambda s: s, filters.get("tags").split(","))) filters["notification_email"] = encrypt_email( @@ -222,10 +219,9 @@ def get_queried_jobs(request): filters["access"] = request.params.get("access", None) filters["status"] = request.params.get("status", None) - del filters["detail"] - del filters["groups"] + filters.pop("detail", None) + filters.pop("groups", None) - groups = groups.split(",") if groups else None store = get_db(request).get_store(StoreJobs) items, total = store.find_jobs(request=request, group_by=groups, **filters) body = {"total": total} @@ -238,7 +234,7 @@ def _job_list(jobs): grouped_jobs["jobs"] = _job_list(grouped_jobs["jobs"]) body.update({"groups": items}) else: - body.update({"jobs": _job_list(items), "page": page, "limit": limit}) + body.update({"jobs": _job_list(items), "page": filters["page"], "limit": filters["limit"]}) body = sd.GetQueriedJobsSchema().deserialize(body) return HTTPOk(json=body) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 8f02290b1..c0e08d9d0 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -6,6 +6,7 @@ import os from typing import TYPE_CHECKING +from dateutil import parser as dateparser import yaml from colander import DateTime, Email, OneOf, Range, Regex, drop, required @@ -110,6 +111,10 @@ OGC_API_REPO_URL = "https://github.com/opengeospatial/ogcapi-processes" OGC_API_SCHEMA_URL = "https://raw.githubusercontent.com/opengeospatial/ogcapi-processes" +DATETIME_INTERVAL_CLOSED_SYMBOL = "/" +DATETIME_INTERVAL_OPEN_START_SYMBOL = "../" +DATETIME_INTERVAL_OPEN_END_SYMBOL = "/.." + ######################################################### # Examples ######################################################### @@ -228,10 +233,14 @@ class URL(ExtendedSchemaNode): format = "url" -class DATETIME_INTERVAL(ExtendedSchemaNode): +class DateTimeInterval(ExtendedSchemaNode): schema_type = String - description = "DateTime name pattern." - + description = "DateTime format against ogcapi, \ + to get values before a certain date-time use '../' before the date-time \ + to get values after a certain date-time use '/..' after the date-time like the example \ + to get values between two date-times use '/' between the date-times \ + to get values with a specific date-time just pass the datetime" + example = "2022-03-02T03:32:38.487000+00:00/.." regex_datetime = r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?)" regex_interval_closed = r"{i}\/{i}".format(i=regex_datetime) regex_interval_open_start = r"\.\.\/{}".format(regex_datetime) @@ -2807,8 +2816,8 @@ class GetJobsQueries(ExtendedMappingSchema): default=False, example="process,service", missing=False) page = ExtendedSchemaNode(Integer(), missing=0, default=0, validator=Range(min=0)) limit = ExtendedSchemaNode(Integer(), missing=10, default=10, validator=Range(min=0, max=10000)) - datetime_interval = DATETIME_INTERVAL(missing=None, default=None) - status = JobStatusEnum(missing=drop) + datetime_interval = DateTimeInterval(missing=None, default=None) + status = JobStatusEnum(missing=None) process = AnyIdentifier(missing=None) provider = ExtendedSchemaNode(String(), missing=drop, default=None) sort = JobSortEnum(missing=drop) @@ -3501,3 +3510,25 @@ def service_api_route_info(service_api, settings): # type: (Service, SettingsType) -> ViewInfo api_base = wps_restapi_base_path(settings) return {"name": service_api.name, "pattern": "{base}{path}".format(base=api_base, path=service_api.path)} + + +def datetime_interval_parser(datetime_interval): + + parsed_datetime = {} + + if datetime_interval.startswith(DATETIME_INTERVAL_OPEN_START_SYMBOL): + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, "") + parsed_datetime["before"] = dateparser.parse(datetime_interval) + + elif datetime_interval.endswith(DATETIME_INTERVAL_OPEN_END_SYMBOL): + datetime_interval = datetime_interval.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, "") + parsed_datetime["after"] = dateparser.parse(datetime_interval) + + elif DATETIME_INTERVAL_CLOSED_SYMBOL in datetime_interval: + datetime_interval = datetime_interval.split(DATETIME_INTERVAL_CLOSED_SYMBOL) + parsed_datetime["after"] = dateparser.parse(datetime_interval[0]) + parsed_datetime["before"] = dateparser.parse(datetime_interval[-1]) + else: + parsed_datetime["match"] = dateparser.parse(datetime_interval) + + return parsed_datetime From 5db11383e6a18c45599a5924b203a26eb4cdf645 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 10:16:26 -0400 Subject: [PATCH 09/22] fix comments --- tests/wps_restapi/test_jobs.py | 32 +++++++--- weaver/store/base.py | 2 +- weaver/store/mongodb.py | 25 +++----- weaver/wps_restapi/jobs/jobs.py | 77 ++++++++++++++--------- weaver/wps_restapi/swagger_definitions.py | 20 ++++-- 5 files changed, 97 insertions(+), 59 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index 1986a9e0a..df47b99bf 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -33,15 +33,15 @@ STATUS_FAILED, STATUS_SUCCEEDED ) +from weaver.utils import get_path_kvp +from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC +from weaver.warning import TimeZoneInfoAlreadySetWarning +from weaver.wps_restapi import swagger_definitions as sd from weaver.wps_restapi.swagger_definitions import ( DATETIME_INTERVAL_CLOSED_SYMBOL, DATETIME_INTERVAL_OPEN_END_SYMBOL, DATETIME_INTERVAL_OPEN_START_SYMBOL ) -from weaver.utils import get_path_kvp -from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC -from weaver.warning import TimeZoneInfoAlreadySetWarning -from weaver.wps_restapi import swagger_definitions as sd if TYPE_CHECKING: from typing import Iterable, List, Tuple, Union @@ -469,7 +469,7 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] path_jobs_user_req_tests = [ # pylint: disable=C0301,line-too-long # URI ACCESS USER EXPECTED JOBS - (uri_direct_jobs, None, None, public_jobs), # noqa: E241,E501 + (uri_direct_jobs, None, None, public_jobs), # noqa: E241,E501 (uri_direct_jobs, None, self.user_editor1_id, editor1_all_jobs), # noqa: E241,E501 (uri_direct_jobs, None, self.user_admin_id, self.job_info), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PRIVATE, None, public_jobs), # noqa: E241,E501 @@ -540,7 +540,7 @@ def test_not_required_fields(self): def test_jobs_datetime_before(self): datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL + self.datetime_interval[0] - path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_before) + path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON @@ -556,7 +556,7 @@ def test_jobs_datetime_before(self): def test_jobs_datetime_after(self): datetime_after = str(self.datetime_interval[2] + DATETIME_INTERVAL_OPEN_END_SYMBOL) - path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_after) + path = get_path_kvp(sd.jobs_service.path, datetime=datetime_after) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON @@ -571,8 +571,8 @@ def test_jobs_datetime_after(self): datetime_after.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, "")) def test_jobs_datetime_interval(self): - datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL+self.datetime_interval[3] - path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_interval) + datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[3] + path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON @@ -590,7 +590,7 @@ def test_jobs_datetime_interval(self): def test_jobs_datetime_match(self): datetime_match = self.datetime_interval[1] - path = get_path_kvp(sd.jobs_service.path, datetime_interval=datetime_match) + path = get_path_kvp(sd.jobs_service.path, datetime=datetime_match) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON @@ -602,3 +602,15 @@ def test_jobs_datetime_match(self): assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON assert dateparser.parse(resp.json["created"]) == dateparser.parse(datetime_match) + + def test_jobs_datetime_invalid(self): + datetime_invalid = "2022-31-12 23:59:59" + path = get_path_kvp(sd.jobs_service.path, datetime=datetime_invalid) + resp = self.app.get(path, headers=self.json_headers, expect_errors=True) + assert resp.status_code == 422 + + def test_jobs_datetime_interval_invalid(self): + datetime_interval = self.datetime_interval[3] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[1] + path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) + resp = self.app.get(path, headers=self.json_headers, expect_errors=True) + assert resp.status_code == 422 diff --git a/weaver/store/base.py b/weaver/store/base.py index 95f1919bb..6fa98d87f 100644 --- a/weaver/store/base.py +++ b/weaver/store/base.py @@ -146,7 +146,7 @@ def find_jobs(self, sort=None, # type: Optional[str] page=0, # type: int limit=10, # type: int - datetime_interval=None, # type: Optional[str] + datetime=None, # type: Optional[Dict] group_by=None, # type: Optional[Union[str, List[str]]] request=None, # type: Optional[Request] ): # type: (...) -> Union[JobListAndCount, JobCategoriesAndCount] diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 799337b84..a8b880d4f 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import pymongo -from dateutil import parser as dateparser from pymongo import ASCENDING, DESCENDING from pymongo.errors import DuplicateKeyError from pyramid.request import Request @@ -43,7 +42,6 @@ SORT_USER ) from weaver.status import JOB_STATUS_CATEGORIES, STATUS_ACCEPTED, map_status -from weaver.wps_restapi.swagger_definitions import datetime_interval_parser from weaver.store.base import StoreBills, StoreJobs, StoreProcesses, StoreQuotes, StoreServices from weaver.utils import get_base_url, get_sane_name, get_weaver_url, islambda, now from weaver.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC, VISIBILITY_VALUES @@ -503,7 +501,7 @@ def find_jobs(self, sort=None, # type: Optional[str] page=0, # type: int limit=10, # type: int - datetime_interval=None, # type: Optional[str] + datetime=None, # type: Optional[Dict] group_by=None, # type: Optional[Union[str, List[str]]] request=None, # type: Optional[Request] ): # type: (...) -> Union[JobListAndCount, JobCategoriesAndCount] @@ -520,7 +518,7 @@ def find_jobs(self, :param sort: field which is used for sorting results (default: creation date, descending). :param page: page number to return when using result paging (only when not using ``group_by``). :param limit: number of jobs per page when using result paging (only when not using ``group_by``). - :param datetime_interval: field used for filtering data by creation date with a given date or interval of date. + :param datetime: field used for filtering data by creation date with a given date or interval of date. :param group_by: one or many fields specifying categories to form matching groups of jobs (paging disabled). :returns: (list of jobs matching paging OR list of {categories, list of jobs, count}) AND total of matched job @@ -588,22 +586,17 @@ def find_jobs(self, if service is not None: search_filters["service"] = service - if datetime_interval is not None: - datetime_interval = datetime_interval_parser(datetime_interval) + if datetime is not None: query = {} - try: - if datetime_interval.get("after", False): - query["$gte"] = datetime_interval["after"] - - if datetime_interval.get("before", False): - query["$lte"] = datetime_interval["before"] + if datetime.get("after", False): + query["$gte"] = datetime["after"] - if datetime_interval.get("match", False): - query = datetime_interval["match"] + if datetime.get("before", False): + query["$lte"] = datetime["before"] - except Exception as ex: - raise JobRegistrationError("Error occurred during datetime job filtering: [{}]".format(repr(ex))) + if datetime.get("match", False): + query = datetime["match"] search_filters["created"] = query diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index c90722158..9884cfa82 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -1,7 +1,15 @@ from typing import TYPE_CHECKING from celery.utils.log import get_task_logger -from pyramid.httpexceptions import HTTPBadRequest, HTTPNotFound, HTTPOk, HTTPPermanentRedirect, HTTPUnauthorized +from colander import Invalid +from pyramid.httpexceptions import ( + HTTPBadRequest, + HTTPNotFound, + HTTPOk, + HTTPPermanentRedirect, + HTTPUnauthorized, + HTTPUnprocessableEntity +) from pyramid.request import Request from pyramid.settings import asbool from pyramid_celery import celery_app as app @@ -27,6 +35,7 @@ from weaver.visibility import VISIBILITY_PUBLIC from weaver.wps.utils import get_wps_output_url from weaver.wps_restapi import swagger_definitions as sd +from weaver.wps_restapi.swagger_definitions import datetime_interval_parser if TYPE_CHECKING: from typing import List, Optional, Tuple, Union @@ -198,45 +207,57 @@ def get_queried_jobs(request): settings = get_settings(request) service, process = validate_service_process(request) - filters = {**request.params, "process": process} + filters = {**request.params, "process": process, "service": service} filters["detail"] = asbool(request.params.get("detail")) - if request.params.get("datetime_interval", False): + if request.params.get("datetime", False): # replace white space with '+' since request.params replaces '+' with whitespaces when parsing - filters["datetime_interval"] = request.params["datetime_interval"].replace(" ", "+") + filters["datetime"] = request.params["datetime"].replace(" ", "+") - filters = sd.GetJobsQueries().deserialize(filters) + try: + filters = sd.GetJobsQueries().deserialize(filters) + except Invalid as ex: + raise HTTPUnprocessableEntity(json={ + "code": Invalid.__name__, + "description": str(ex) + }) + else: - detail = asbool(filters.get("detail", False)) - groups = filters["groups"].split(",") if filters["groups"] else None + detail = filters.pop("detail", False) + groups = filters.pop("groups", "").split(",") if filters["groups"] else filters.pop("groups", None) - filters["tags"] = list(filter(lambda s: s, filters.get("tags").split(","))) - filters["notification_email"] = encrypt_email( - filters["notification_email"], settings) if filters.get("notification_email", False) else None - filters["service"] = service - filters["access"] = request.params.get("access", None) - filters["status"] = request.params.get("status", None) + filters["tags"] = list(filter(lambda s: s, filters.get("tags").split(","))) + filters["notification_email"] = encrypt_email( + filters["notification_email"], settings) if filters.get("notification_email", False) else None + filters["datetime"] = datetime_interval_parser(filters["datetime"]) if filters.get("datetime", False) else None - filters.pop("detail", None) - filters.pop("groups", None) + if (filters["datetime"] + and filters["datetime"].get("before", False) + and filters["datetime"].get("after", False) + and filters["datetime"]["after"] > filters["datetime"]["before"]): - store = get_db(request).get_store(StoreJobs) - items, total = store.find_jobs(request=request, group_by=groups, **filters) - body = {"total": total} + raise HTTPUnprocessableEntity(json={ + "code": "InvalidDateFormat", + "description": "Datetime at the start of the interval must be less than the datetime at the end." + }) - def _job_list(jobs): - return [j.json(settings) if detail else j.id for j in jobs] + store = get_db(request).get_store(StoreJobs) + items, total = store.find_jobs(request=request, group_by=groups, **filters) + body = {"total": total} - if groups: - for grouped_jobs in items: - grouped_jobs["jobs"] = _job_list(grouped_jobs["jobs"]) - body.update({"groups": items}) - else: - body.update({"jobs": _job_list(items), "page": filters["page"], "limit": filters["limit"]}) - body = sd.GetQueriedJobsSchema().deserialize(body) - return HTTPOk(json=body) + def _job_list(jobs): + return [j.json(settings) if detail else j.id for j in jobs] + + if groups: + for grouped_jobs in items: + grouped_jobs["jobs"] = _job_list(grouped_jobs["jobs"]) + body.update({"groups": items}) + else: + body.update({"jobs": _job_list(items), "page": filters["page"], "limit": filters["limit"]}) + body = sd.GetQueriedJobsSchema().deserialize(body) + return HTTPOk(json=body) @sd.provider_job_service.get(tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROVIDERS], renderer=OUTPUT_FORMAT_JSON, diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index c0e08d9d0..7d08ba81b 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -6,11 +6,11 @@ import os from typing import TYPE_CHECKING -from dateutil import parser as dateparser import yaml from colander import DateTime, Email, OneOf, Range, Regex, drop, required from cornice import Service +from dateutil import parser as dateparser from weaver import __meta__ from weaver.config import WEAVER_CONFIGURATION_EMS @@ -907,6 +907,10 @@ class VisibilityValue(ExtendedSchemaNode): example = VISIBILITY_PUBLIC +class JobAccess(VisibilityValue): + pass + + class Visibility(ExtendedMappingSchema): value = VisibilityValue() @@ -2816,11 +2820,12 @@ class GetJobsQueries(ExtendedMappingSchema): default=False, example="process,service", missing=False) page = ExtendedSchemaNode(Integer(), missing=0, default=0, validator=Range(min=0)) limit = ExtendedSchemaNode(Integer(), missing=10, default=10, validator=Range(min=0, max=10000)) - datetime_interval = DateTimeInterval(missing=None, default=None) - status = JobStatusEnum(missing=None) + datetime = DateTimeInterval(missing=None, default=None) + status = JobStatusEnum(missing=None, default=None) process = AnyIdentifier(missing=None) - provider = ExtendedSchemaNode(String(), missing=drop, default=None) + service = ExtendedSchemaNode(String(), missing=None, default=None) sort = JobSortEnum(missing=drop) + access = JobAccess(missing=None, default=None) notification_email = ExtendedSchemaNode(String(), missing=drop, validator=Email()) tags = ExtendedSchemaNode(String(), missing="", default="", description="Comma-separated values of tags assigned to jobs") @@ -2959,6 +2964,12 @@ class ErrorJsonResponseBodySchema(ExtendedMappingSchema): exception = OWSExceptionResponse(missing=drop) +class UnprocessableEntityResponseSchema(ExtendedMappingSchema): + description = "Wrong format of given parameters." + header = ResponseHeaders() + body = ErrorJsonResponseBodySchema() + + class ForbiddenProcessAccessResponseSchema(ExtendedMappingSchema): description = "Referenced process is not accessible." header = ResponseHeaders() @@ -3379,6 +3390,7 @@ class OkGetJobLogsResponse(ExtendedMappingSchema): "value": EXAMPLES["jobs_listing.json"] } }), + "422": UnprocessableEntityResponseSchema(), "500": InternalServerErrorResponseSchema(), } get_single_job_status_responses = { From a9b34c445d60e9a8b1a2d093fca338d16a9ba5e8 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 10:28:25 -0400 Subject: [PATCH 10/22] fix pep8 --- tests/wps_restapi/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index df47b99bf..f8ba0a601 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -469,7 +469,7 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] path_jobs_user_req_tests = [ # pylint: disable=C0301,line-too-long # URI ACCESS USER EXPECTED JOBS - (uri_direct_jobs, None, None, public_jobs), # noqa: E241,E501 + (uri_direct_jobs, None, None, public_jobs), # noqa: E241,E501 (uri_direct_jobs, None, self.user_editor1_id, editor1_all_jobs), # noqa: E241,E501 (uri_direct_jobs, None, self.user_admin_id, self.job_info), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PRIVATE, None, public_jobs), # noqa: E241,E501 From 734ea9e223de333a85f47c8f787a4d7b00ee0ed9 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 10:46:58 -0400 Subject: [PATCH 11/22] added ogcapi conformance links --- tests/wps_restapi/test_jobs.py | 42 ++++++++++++++++++++++++++++++++++ weaver/wps_restapi/api.py | 2 ++ 2 files changed, 44 insertions(+) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index f8ba0a601..fcfdb7d6e 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -516,6 +516,10 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] assert job_match, self.message_with_jobs_diffs(resp.json["jobs"], job_ids, test_values, index=i) def test_jobs_list_with_limit_client(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ limit_parameter = 20 path = get_path_kvp(sd.jobs_service.path, limit=limit_parameter) resp = self.app.get(path, headers=self.json_headers) @@ -526,6 +530,10 @@ def test_jobs_list_with_limit_client(self): assert len(resp.json["jobs"]) <= limit_parameter def test_jobs_list_with_limit_api(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON @@ -539,6 +547,10 @@ def test_not_required_fields(self): assert not resp.json["parameters"]["limit"]["required"] def test_jobs_datetime_before(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL + self.datetime_interval[0] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) resp = self.app.get(path, headers=self.json_headers) @@ -555,6 +567,10 @@ def test_jobs_datetime_before(self): datetime_before.replace(DATETIME_INTERVAL_OPEN_START_SYMBOL, "")) def test_jobs_datetime_after(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ datetime_after = str(self.datetime_interval[2] + DATETIME_INTERVAL_OPEN_END_SYMBOL) path = get_path_kvp(sd.jobs_service.path, datetime=datetime_after) resp = self.app.get(path, headers=self.json_headers) @@ -571,6 +587,10 @@ def test_jobs_datetime_after(self): datetime_after.replace(DATETIME_INTERVAL_OPEN_END_SYMBOL, "")) def test_jobs_datetime_interval(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[3] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) resp = self.app.get(path, headers=self.json_headers) @@ -589,6 +609,10 @@ def test_jobs_datetime_interval(self): assert dateparser.parse(resp.json["created"]) <= dateparser.parse(datetime_before) def test_jobs_datetime_match(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ datetime_match = self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_match) resp = self.app.get(path, headers=self.json_headers) @@ -604,13 +628,31 @@ def test_jobs_datetime_match(self): assert dateparser.parse(resp.json["created"]) == dateparser.parse(datetime_match) def test_jobs_datetime_invalid(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ datetime_invalid = "2022-31-12 23:59:59" path = get_path_kvp(sd.jobs_service.path, datetime=datetime_invalid) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 422 def test_jobs_datetime_interval_invalid(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ datetime_interval = self.datetime_interval[3] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 422 + + def test_jobs_datetime_before_invalid(self): + """ + .. seealso:: + - `/req/collections/rc-links `_ + """ + datetime_before = "./" + self.datetime_interval[3] + path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) + resp = self.app.get(path, headers=self.json_headers, expect_errors=True) + assert resp.status_code == 422 diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index a0361c6a2..95eed73f3 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -205,6 +205,8 @@ def api_conformance(request): # noqa: F811 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback", # FIXME: https://github.com/crim-ca/weaver/issues/228 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/dismiss", + "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-limit-response.adoc", + "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-time-collections-response.adoc", ]} return HTTPOk(json=conformance) From 5a1b5395b51d81f9a8ae275b4972910ccf1d04d1 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 11:32:17 -0400 Subject: [PATCH 12/22] fix pep8 --- weaver/wps_restapi/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index 95eed73f3..de6cf987e 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -205,8 +205,10 @@ def api_conformance(request): # noqa: F811 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback", # FIXME: https://github.com/crim-ca/weaver/issues/228 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/dismiss", - "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-limit-response.adoc", - "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-time-collections-response.adoc", + "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/ \ + requirements/collections/REQ_rc-limit-response.adoc", + "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/ \ + requirements/collections/REQ_rc-time-collections-response.adoc", ]} return HTTPOk(json=conformance) From 716ee9eaa1646afbbef1697bd3d049e25bcc97aa Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 11:48:15 -0400 Subject: [PATCH 13/22] fix pep8 --- tests/wps_restapi/test_jobs.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index fcfdb7d6e..b10fcb1f5 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -518,7 +518,8 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] def test_jobs_list_with_limit_client(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ limit_parameter = 20 path = get_path_kvp(sd.jobs_service.path, limit=limit_parameter) @@ -532,7 +533,8 @@ def test_jobs_list_with_limit_client(self): def test_jobs_list_with_limit_api(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) assert resp.status_code == 200 @@ -549,7 +551,8 @@ def test_not_required_fields(self): def test_jobs_datetime_before(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL + self.datetime_interval[0] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) @@ -569,7 +572,8 @@ def test_jobs_datetime_before(self): def test_jobs_datetime_after(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ datetime_after = str(self.datetime_interval[2] + DATETIME_INTERVAL_OPEN_END_SYMBOL) path = get_path_kvp(sd.jobs_service.path, datetime=datetime_after) @@ -589,7 +593,8 @@ def test_jobs_datetime_after(self): def test_jobs_datetime_interval(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[3] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) @@ -611,7 +616,8 @@ def test_jobs_datetime_interval(self): def test_jobs_datetime_match(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ datetime_match = self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_match) @@ -630,7 +636,8 @@ def test_jobs_datetime_match(self): def test_jobs_datetime_invalid(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ datetime_invalid = "2022-31-12 23:59:59" path = get_path_kvp(sd.jobs_service.path, datetime=datetime_invalid) @@ -640,7 +647,8 @@ def test_jobs_datetime_invalid(self): def test_jobs_datetime_interval_invalid(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ datetime_interval = self.datetime_interval[3] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) @@ -650,7 +658,8 @@ def test_jobs_datetime_interval_invalid(self): def test_jobs_datetime_before_invalid(self): """ .. seealso:: - - `/req/collections/rc-links `_ + - `/req/collections/rc-links + `_ """ datetime_before = "./" + self.datetime_interval[3] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) From f41fb26917128a923f32fcd938e1a811e146fb5a Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 11:52:22 -0400 Subject: [PATCH 14/22] fix pep8 --- tests/wps_restapi/test_jobs.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index b10fcb1f5..279b5ec9e 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -518,7 +518,7 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] def test_jobs_list_with_limit_client(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ limit_parameter = 20 @@ -533,7 +533,7 @@ def test_jobs_list_with_limit_client(self): def test_jobs_list_with_limit_api(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) @@ -551,7 +551,7 @@ def test_not_required_fields(self): def test_jobs_datetime_before(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL + self.datetime_interval[0] @@ -572,7 +572,7 @@ def test_jobs_datetime_before(self): def test_jobs_datetime_after(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ datetime_after = str(self.datetime_interval[2] + DATETIME_INTERVAL_OPEN_END_SYMBOL) @@ -593,7 +593,7 @@ def test_jobs_datetime_after(self): def test_jobs_datetime_interval(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[3] @@ -616,7 +616,7 @@ def test_jobs_datetime_interval(self): def test_jobs_datetime_match(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ datetime_match = self.datetime_interval[1] @@ -636,7 +636,7 @@ def test_jobs_datetime_match(self): def test_jobs_datetime_invalid(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ datetime_invalid = "2022-31-12 23:59:59" @@ -647,7 +647,7 @@ def test_jobs_datetime_invalid(self): def test_jobs_datetime_interval_invalid(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ datetime_interval = self.datetime_interval[3] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[1] @@ -658,7 +658,7 @@ def test_jobs_datetime_interval_invalid(self): def test_jobs_datetime_before_invalid(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-links `_ """ datetime_before = "./" + self.datetime_interval[3] From b6c42825fa10e58edc8bbcc33e1c54a959f6a31b Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 12:13:16 -0400 Subject: [PATCH 15/22] fix pep8 --- weaver/wps_restapi/api.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index de6cf987e..f2db2c127 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -205,10 +205,8 @@ def api_conformance(request): # noqa: F811 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback", # FIXME: https://github.com/crim-ca/weaver/issues/228 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/dismiss", - "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/ \ - requirements/collections/REQ_rc-limit-response.adoc", - "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/ \ - requirements/collections/REQ_rc-time-collections-response.adoc", + "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-limit-response.adoc", # noqa + "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-time-collections-response.adoc", # noqa ]} return HTTPOk(json=conformance) From 807cf3cd3c2f1ba6a63e09725de3974aa3974ddd Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 12:25:06 -0400 Subject: [PATCH 16/22] fix pep8 --- tests/wps_restapi/test_jobs.py | 18 +++++++++--------- weaver/wps_restapi/api.py | 5 +++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index 279b5ec9e..c999a03d6 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -519,7 +519,7 @@ def test_jobs_list_with_limit_client(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ limit_parameter = 20 path = get_path_kvp(sd.jobs_service.path, limit=limit_parameter) @@ -534,7 +534,7 @@ def test_jobs_list_with_limit_api(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) assert resp.status_code == 200 @@ -552,7 +552,7 @@ def test_jobs_datetime_before(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL + self.datetime_interval[0] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) @@ -573,7 +573,7 @@ def test_jobs_datetime_after(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ datetime_after = str(self.datetime_interval[2] + DATETIME_INTERVAL_OPEN_END_SYMBOL) path = get_path_kvp(sd.jobs_service.path, datetime=datetime_after) @@ -594,7 +594,7 @@ def test_jobs_datetime_interval(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[3] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) @@ -617,7 +617,7 @@ def test_jobs_datetime_match(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ datetime_match = self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_match) @@ -637,7 +637,7 @@ def test_jobs_datetime_invalid(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ datetime_invalid = "2022-31-12 23:59:59" path = get_path_kvp(sd.jobs_service.path, datetime=datetime_invalid) @@ -648,7 +648,7 @@ def test_jobs_datetime_interval_invalid(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ datetime_interval = self.datetime_interval[3] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) @@ -659,7 +659,7 @@ def test_jobs_datetime_before_invalid(self): """ .. seealso:: - `/req/collections/rc-links - `_ + `_ """ datetime_before = "./" + self.datetime_interval[3] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index f2db2c127..1edd519b5 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -178,6 +178,7 @@ def api_conformance(request): # noqa: F811 # type: (Request) -> HTTPException """Weaver specification conformance information.""" # TODO: follow updates with https://github.com/geopython/pygeoapi/issues/198 + ogcapi_common = "https://github.com/opengeospatial/ogcapi-common" conformance = {"conformsTo": [ # "http://www.opengis.net/spec/wfs-1/3.0/req/core", # "http://www.opengis.net/spec/wfs-1/3.0/req/oas30", @@ -205,8 +206,8 @@ def api_conformance(request): # noqa: F811 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback", # FIXME: https://github.com/crim-ca/weaver/issues/228 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/dismiss", - "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-limit-response.adoc", # noqa - "https://github.com/opengeospatial/ogcapi-common/blob/Working/collections/requirements/collections/REQ_rc-time-collections-response.adoc", # noqa + ogcapi_common+"/blob/master/collections/requirements/collections/REQ_rc-limit-response.adoc", + ogcapi_common+"/blob/master/collections/requirements/collections/REQ_rc-time-collections-response.adoc", ]} return HTTPOk(json=conformance) From f2e4e913a6e21f55fc8d245dce3c925f76aee436 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 16:56:29 -0400 Subject: [PATCH 17/22] fix comments --- tests/utils.py | 9 ----- tests/wps_restapi/test_jobs.py | 43 ++++++++++++++++------- weaver/store/base.py | 6 ++-- weaver/store/mongodb.py | 6 ++-- weaver/typedefs.py | 6 ++++ weaver/wps_restapi/api.py | 4 +-- weaver/wps_restapi/jobs/jobs.py | 13 ++++--- weaver/wps_restapi/swagger_definitions.py | 31 ++++++++-------- 8 files changed, 68 insertions(+), 50 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 5b8d9f3a1..7a56dc079 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -550,12 +550,3 @@ def mocked_aws_s3_bucket_test_file(bucket_name, file_name, file_content="Test fi tmp_file.flush() s3.upload_file(Bucket=bucket_name, Filename=tmp_file.name, Key=file_name) return "s3://{}/{}".format(bucket_name, file_name) - - -def generate_test_datetimes(): - # type: () -> List[str] - """ - Generates a list of dummy datetimes for testing. - """ - year = date.today().year + 1 - return ["{}-0{}-02T03:32:38.487000+00:00".format(year, month) for month in range(1, 5)] diff --git a/tests/wps_restapi/test_jobs.py b/tests/wps_restapi/test_jobs.py index c999a03d6..86f3e6b66 100644 --- a/tests/wps_restapi/test_jobs.py +++ b/tests/wps_restapi/test_jobs.py @@ -3,6 +3,7 @@ import unittest import warnings from collections import OrderedDict +from datetime import date from distutils.version import LooseVersion from typing import TYPE_CHECKING @@ -12,7 +13,6 @@ from dateutil import parser as dateparser from tests.utils import ( - generate_test_datetimes, get_module_version, get_test_weaver_app, mocked_process_job_runner, @@ -58,7 +58,7 @@ def setUpClass(cls): cls.config = setup_config_with_mongodb(settings=settings) cls.app = get_test_weaver_app(config=cls.config) cls.json_headers = {"Accept": CONTENT_TYPE_APP_JSON, "Content-Type": CONTENT_TYPE_APP_JSON} - cls.datetime_interval = generate_test_datetimes() + cls.datetime_interval = cls.generate_test_datetimes() @classmethod def tearDownClass(cls): @@ -155,6 +155,15 @@ def get_job_request_auth_mock(self, user_id): mock.patch("{}.has_permission".format(authz_policy_class), return_value=is_admin), ]) + @staticmethod + def generate_test_datetimes(): + # type: () -> List[str] + """ + Generates a list of dummy datetimes for testing. + """ + year = date.today().year + 1 + return ["{}-0{}-02T03:32:38.487000+00:00".format(year, month) for month in range(1, 5)] + @staticmethod def check_job_format(job): assert isinstance(job, dict) @@ -515,10 +524,10 @@ def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] test_values = dict(path=path, access=access, user_id=user_id) assert job_match, self.message_with_jobs_diffs(resp.json["jobs"], job_ids, test_values, index=i) - def test_jobs_list_with_limit_client(self): + def test_jobs_list_with_limit_api(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-limit-response `_ """ limit_parameter = 20 @@ -530,10 +539,10 @@ def test_jobs_list_with_limit_client(self): assert resp.json["limit"] == limit_parameter assert len(resp.json["jobs"]) <= limit_parameter - def test_jobs_list_with_limit_api(self): + def test_jobs_list_with_limit_openapi_schema(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-limit-response `_ """ resp = self.app.get(sd.jobs_service.path, headers=self.json_headers) @@ -551,7 +560,7 @@ def test_not_required_fields(self): def test_jobs_datetime_before(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-time-collections-response `_ """ datetime_before = DATETIME_INTERVAL_OPEN_START_SYMBOL + self.datetime_interval[0] @@ -572,7 +581,7 @@ def test_jobs_datetime_before(self): def test_jobs_datetime_after(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-time-collections-response `_ """ datetime_after = str(self.datetime_interval[2] + DATETIME_INTERVAL_OPEN_END_SYMBOL) @@ -593,7 +602,7 @@ def test_jobs_datetime_after(self): def test_jobs_datetime_interval(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-time-collections-response `_ """ datetime_interval = self.datetime_interval[1] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[3] @@ -616,7 +625,7 @@ def test_jobs_datetime_interval(self): def test_jobs_datetime_match(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-time-collections-response `_ """ datetime_match = self.datetime_interval[1] @@ -636,8 +645,11 @@ def test_jobs_datetime_match(self): def test_jobs_datetime_invalid(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-time-collections-response `_ + + datetime_invalid is not formated against the rfc3339 datetime format, + for more details refer to https://datatracker.ietf.org/doc/html/rfc3339#section-5.6 """ datetime_invalid = "2022-31-12 23:59:59" path = get_path_kvp(sd.jobs_service.path, datetime=datetime_invalid) @@ -647,8 +659,11 @@ def test_jobs_datetime_invalid(self): def test_jobs_datetime_interval_invalid(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-time-collections-response `_ + + datetime_invalid represents a datetime interval where the limit dates are inverted, + the minimun is greather than the maximum datetime limit """ datetime_interval = self.datetime_interval[3] + DATETIME_INTERVAL_CLOSED_SYMBOL + self.datetime_interval[1] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_interval) @@ -658,8 +673,10 @@ def test_jobs_datetime_interval_invalid(self): def test_jobs_datetime_before_invalid(self): """ .. seealso:: - - `/req/collections/rc-links + - `/req/collections/rc-time-collections-response `_ + + datetime_before represents a bad open range datetime interval """ datetime_before = "./" + self.datetime_interval[3] path = get_path_kvp(sd.jobs_service.path, datetime=datetime_before) diff --git a/weaver/store/base.py b/weaver/store/base.py index 6fa98d87f..b395826b8 100644 --- a/weaver/store/base.py +++ b/weaver/store/base.py @@ -6,7 +6,7 @@ from pyramid.request import Request from pywps import Process as ProcessWPS from weaver.datatype import Bill, Job, Process, Quote, Service - from weaver.typedefs import AnyValue + from weaver.typedefs import AnyValue, Datetime, DatetimeIntervalType JobListAndCount = Tuple[List[Job], int] JobCategory = Dict[str, Union[AnyValue, Job]] @@ -111,7 +111,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] - created=None, # type: Optional[str] + created=None, # type: Datetime ): # type: (...) -> Job raise NotImplementedError @@ -146,7 +146,7 @@ def find_jobs(self, sort=None, # type: Optional[str] page=0, # type: int limit=10, # type: int - datetime=None, # type: Optional[Dict] + datetime=None, # type: Optional[DatetimeIntervalType] group_by=None, # type: Optional[Union[str, List[str]]] request=None, # type: Optional[Request] ): # type: (...) -> Union[JobListAndCount, JobCategoriesAndCount] diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index a8b880d4f..6debe43a9 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -51,7 +51,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from pymongo.collection import Collection - from weaver.store.base import JobCategoriesAndCount, JobListAndCount + from weaver.store.base import Datetime, DatetimeIntervalType, JobCategoriesAndCount, JobListAndCount from weaver.typedefs import AnyProcess, AnyProcessType LOGGER = logging.getLogger(__name__) @@ -405,7 +405,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] - created=None, # type: Optional[str] + created=None, # type: Datetime ): # type: (...) -> Job """ Creates a new :class:`Job` and stores it in mongodb. @@ -501,7 +501,7 @@ def find_jobs(self, sort=None, # type: Optional[str] page=0, # type: int limit=10, # type: int - datetime=None, # type: Optional[Dict] + datetime=None, # type: Optional[DatetimeIntervalType] group_by=None, # type: Optional[Union[str, List[str]]] request=None, # type: Optional[Request] ): # type: (...) -> Union[JobListAndCount, JobCategoriesAndCount] diff --git a/weaver/typedefs.py b/weaver/typedefs.py index 19df11656..6a6582364 100644 --- a/weaver/typedefs.py +++ b/weaver/typedefs.py @@ -41,6 +41,7 @@ # pylint: disable=C0103,invalid-name Number = Union[int, float] + Datetime = Optional[str] ValueType = Union[str, Number, bool] AnyValue = Optional[ValueType] AnyValueType = AnyValue # alias @@ -114,3 +115,8 @@ # update_status(provider, message, progress, status) UpdateStatusPartialFunction = Callable[[str, str, int, AnyStatusType], None] + + # others + DatetimeIntervalType = TypedDict("DatetimeIntervalType", + {"before": Datetime, "after": Datetime, + "match": Datetime, }, total=False) diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index 1edd519b5..f81c03b64 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -206,8 +206,8 @@ def api_conformance(request): # noqa: F811 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback", # FIXME: https://github.com/crim-ca/weaver/issues/228 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/dismiss", - ogcapi_common+"/blob/master/collections/requirements/collections/REQ_rc-limit-response.adoc", - ogcapi_common+"/blob/master/collections/requirements/collections/REQ_rc-time-collections-response.adoc", + ogcapi_common + "/req/collections/rc-limit-response", + ogcapi_common + "/req/collections/rc-time-collections-response" ]} return HTTPOk(json=conformance) diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index 9884cfa82..206d29b57 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -207,7 +207,7 @@ def get_queried_jobs(request): settings = get_settings(request) service, process = validate_service_process(request) - filters = {**request.params, "process": process, "service": service} + filters = {**request.params, "process": process, "provider": service} filters["detail"] = asbool(request.params.get("detail")) @@ -226,17 +226,20 @@ def get_queried_jobs(request): else: detail = filters.pop("detail", False) - groups = filters.pop("groups", "").split(",") if filters["groups"] else filters.pop("groups", None) + groups = filters.pop("groups", "").split(",") if filters.get("groups", False) else filters.pop("groups", None) - filters["tags"] = list(filter(lambda s: s, filters.get("tags").split(","))) + filters["tags"] = list(filter(lambda s: s, filters["tags"].split(",") if filters.get("tags", False) else "")) filters["notification_email"] = encrypt_email( filters["notification_email"], settings) if filters.get("notification_email", False) else None filters["datetime"] = datetime_interval_parser(filters["datetime"]) if filters.get("datetime", False) else None + filters["service"] = filters.pop("provider", None) - if (filters["datetime"] + if ( + filters["datetime"] and filters["datetime"].get("before", False) and filters["datetime"].get("after", False) - and filters["datetime"]["after"] > filters["datetime"]["before"]): + and filters["datetime"]["after"] > filters["datetime"]["before"] + ): raise HTTPUnprocessableEntity(json={ "code": "InvalidDateFormat", diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 7d08ba81b..488d801bd 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -74,7 +74,7 @@ from weaver.wps_restapi.utils import wps_restapi_base_path if TYPE_CHECKING: - from weaver.typedefs import SettingsType, TypedDict + from weaver.typedefs import Datetime, DatetimeIntervalType, SettingsType, TypedDict ViewInfo = TypedDict("ViewInfo", {"name": str, "pattern": str}) @@ -235,11 +235,11 @@ class URL(ExtendedSchemaNode): class DateTimeInterval(ExtendedSchemaNode): schema_type = String - description = "DateTime format against ogcapi, \ - to get values before a certain date-time use '../' before the date-time \ - to get values after a certain date-time use '/..' after the date-time like the example \ - to get values between two date-times use '/' between the date-times \ - to get values with a specific date-time just pass the datetime" + description = "DateTime format against OGC-API - Processes,\n\ + to get values before a certain date-time use '../' before the date-time,\n\ + to get values after a certain date-time use '/..' after the date-time like the example,\n\ + to get values between two date-times use '/' between the date-times,\n\ + to get values with a specific date-time just pass the datetime" example = "2022-03-02T03:32:38.487000+00:00/.." regex_datetime = r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?)" regex_interval_closed = r"{i}\/{i}".format(i=regex_datetime) @@ -2814,20 +2814,20 @@ class PostProcessJobsEndpoint(ProcessPath): class GetJobsQueries(ExtendedMappingSchema): detail = ExtendedSchemaNode(Boolean(), description="Provide job details instead of IDs.", - default=False, example=True, missing=False) + default=False, example=True, missing=drop) groups = ExtendedSchemaNode(String(), description="Comma-separated list of grouping fields with which to list jobs.", - default=False, example="process,service", missing=False) + default=False, example="process,service", missing=drop) page = ExtendedSchemaNode(Integer(), missing=0, default=0, validator=Range(min=0)) limit = ExtendedSchemaNode(Integer(), missing=10, default=10, validator=Range(min=0, max=10000)) - datetime = DateTimeInterval(missing=None, default=None) - status = JobStatusEnum(missing=None, default=None) - process = AnyIdentifier(missing=None) - service = ExtendedSchemaNode(String(), missing=None, default=None) + datetime = DateTimeInterval(missing=drop, default=None) + status = JobStatusEnum(missing=drop, default=None) + process = AnyIdentifier(missing=drop) + provider = ExtendedSchemaNode(String(), missing=drop, default=None) sort = JobSortEnum(missing=drop) - access = JobAccess(missing=None, default=None) + access = JobAccess(missing=drop, default=None) notification_email = ExtendedSchemaNode(String(), missing=drop, validator=Email()) - tags = ExtendedSchemaNode(String(), missing="", default="", + tags = ExtendedSchemaNode(String(), missing=drop, default=None, description="Comma-separated values of tags assigned to jobs") @@ -3525,7 +3525,8 @@ def service_api_route_info(service_api, settings): def datetime_interval_parser(datetime_interval): - + # type: (Datetime) -> DatetimeIntervalType + """This function parse a given datetime or interval into a dictionary that will be easy for database process""" parsed_datetime = {} if datetime_interval.startswith(DATETIME_INTERVAL_OPEN_START_SYMBOL): From 12655eacf3cb5ede90d6286d6ec4e27b290f79c2 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Thu, 27 May 2021 17:18:58 -0400 Subject: [PATCH 18/22] fix pep8 --- tests/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 7a56dc079..614bf3014 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -7,7 +7,6 @@ import uuid import warnings from configparser import ConfigParser -from datetime import date from inspect import isclass from typing import TYPE_CHECKING From 0315aac1457c13b0a47334ac35718964104e9807 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Fri, 28 May 2021 13:41:58 -0400 Subject: [PATCH 19/22] fix comments --- weaver/store/base.py | 5 +++-- weaver/store/mongodb.py | 5 +++-- weaver/typedefs.py | 5 ++--- weaver/wps_restapi/api.py | 8 ++++---- weaver/wps_restapi/jobs/jobs.py | 7 ++++--- weaver/wps_restapi/swagger_definitions.py | 14 +++++++------- 6 files changed, 23 insertions(+), 21 deletions(-) diff --git a/weaver/store/base.py b/weaver/store/base.py index b395826b8..9a5b66877 100644 --- a/weaver/store/base.py +++ b/weaver/store/base.py @@ -2,11 +2,12 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + import datetime from typing import Any, Dict, List, Optional, Tuple, Union from pyramid.request import Request from pywps import Process as ProcessWPS from weaver.datatype import Bill, Job, Process, Quote, Service - from weaver.typedefs import AnyValue, Datetime, DatetimeIntervalType + from weaver.typedefs import AnyValue, DatetimeIntervalType JobListAndCount = Tuple[List[Job], int] JobCategory = Dict[str, Union[AnyValue, Job]] @@ -111,7 +112,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] - created=None, # type: Datetime + created=None, # type: Optional[datetime.datetime] ): # type: (...) -> Job raise NotImplementedError diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 6debe43a9..9d55f571d 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -48,10 +48,11 @@ from weaver.wps.utils import get_wps_url if TYPE_CHECKING: + import datetime from typing import Any, Callable, Dict, List, Optional, Tuple, Union from pymongo.collection import Collection - from weaver.store.base import Datetime, DatetimeIntervalType, JobCategoriesAndCount, JobListAndCount + from weaver.store.base import DatetimeIntervalType, JobCategoriesAndCount, JobListAndCount from weaver.typedefs import AnyProcess, AnyProcessType LOGGER = logging.getLogger(__name__) @@ -405,7 +406,7 @@ def save_job(self, access=None, # type: Optional[str] notification_email=None, # type: Optional[str] accept_language=None, # type: Optional[str] - created=None, # type: Datetime + created=None, # type: Optional[datetime.datetime] ): # type: (...) -> Job """ Creates a new :class:`Job` and stores it in mongodb. diff --git a/weaver/typedefs.py b/weaver/typedefs.py index 6a6582364..4e5b57acd 100644 --- a/weaver/typedefs.py +++ b/weaver/typedefs.py @@ -41,7 +41,6 @@ # pylint: disable=C0103,invalid-name Number = Union[int, float] - Datetime = Optional[str] ValueType = Union[str, Number, bool] AnyValue = Optional[ValueType] AnyValueType = AnyValue # alias @@ -118,5 +117,5 @@ # others DatetimeIntervalType = TypedDict("DatetimeIntervalType", - {"before": Datetime, "after": Datetime, - "match": Datetime, }, total=False) + {"before": str, "after": str, + "match": str, }, total=False) diff --git a/weaver/wps_restapi/api.py b/weaver/wps_restapi/api.py index 008e1dc1e..71994c517 100644 --- a/weaver/wps_restapi/api.py +++ b/weaver/wps_restapi/api.py @@ -190,11 +190,11 @@ def api_conformance(request): # noqa: F811 # - https://github.com/opengeospatial/ogcapi-processes/tree/master/extensions/transactions/standard/recommendations # - https://github.com/opengeospatial/ogcapi-processes/tree/master/extensions/workflows/standard/requirements # - https://github.com/opengeospatial/ogcapi-processes/tree/master/extensions/workflows/standard/recommendations - + ows_wps1 = "http://schemas.opengis.net/wps/1.0.0" ows_wps2 = "http://www.opengis.net/spec/WPS/2.0" ogcapi_common = "https://github.com/opengeospatial/ogcapi-common" - ogcapi_processes = "http://www.opengis.net/spec/ogcapi-processes-1/1.0 + ogcapi_processes = "http://www.opengis.net/spec/ogcapi-processes-1/1.0" conformance = {"conformsTo": [ # "http://www.opengis.net/spec/wfs-1/3.0/req/core", # "http://www.opengis.net/spec/wfs-1/3.0/req/oas30", @@ -222,11 +222,11 @@ def api_conformance(request): # noqa: F811 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback", # FIXME: https://github.com/crim-ca/weaver/issues/228 # "http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/dismiss", - ogcapi_common + "/req/collections/rc-limit-response", - ogcapi_common + "/req/collections/rc-time-collections-response" # FIXME: https://github.com/crim-ca/weaver/issues/231 # List all supported requirements, recommendations and abstract tests ogcapi_processes + "/req/core/process", + ogcapi_common + "/req/collections/rc-limit-response", + ogcapi_common + "/req/collections/rc-time-collections-response" ]} return HTTPOk(json=conformance) diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index 206d29b57..0ce17527e 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -229,8 +229,10 @@ def get_queried_jobs(request): groups = filters.pop("groups", "").split(",") if filters.get("groups", False) else filters.pop("groups", None) filters["tags"] = list(filter(lambda s: s, filters["tags"].split(",") if filters.get("tags", False) else "")) - filters["notification_email"] = encrypt_email( - filters["notification_email"], settings) if filters.get("notification_email", False) else None + filters["notification_email"] = ( + encrypt_email(filters["notification_email"], settings) + if filters.get("notification_email", False) else None + ) filters["datetime"] = datetime_interval_parser(filters["datetime"]) if filters.get("datetime", False) else None filters["service"] = filters.pop("provider", None) @@ -240,7 +242,6 @@ def get_queried_jobs(request): and filters["datetime"].get("after", False) and filters["datetime"]["after"] > filters["datetime"]["before"] ): - raise HTTPUnprocessableEntity(json={ "code": "InvalidDateFormat", "description": "Datetime at the start of the interval must be less than the datetime at the end." diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 488d801bd..cdd7d5c2a 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -74,7 +74,7 @@ from weaver.wps_restapi.utils import wps_restapi_base_path if TYPE_CHECKING: - from weaver.typedefs import Datetime, DatetimeIntervalType, SettingsType, TypedDict + from weaver.typedefs import DatetimeIntervalType, SettingsType, TypedDict ViewInfo = TypedDict("ViewInfo", {"name": str, "pattern": str}) @@ -235,11 +235,11 @@ class URL(ExtendedSchemaNode): class DateTimeInterval(ExtendedSchemaNode): schema_type = String - description = "DateTime format against OGC-API - Processes,\n\ - to get values before a certain date-time use '../' before the date-time,\n\ - to get values after a certain date-time use '/..' after the date-time like the example,\n\ - to get values between two date-times use '/' between the date-times,\n\ - to get values with a specific date-time just pass the datetime" + description = "DateTime format against OGC-API - Processes,\ + to get values before a certain date-time use '../' before the date-time,\ + to get values after a certain date-time use '/..' after the date-time like the example,\ + to get values between two date-times use '/' between the date-times,\ + to get values with a specific date-time just pass the datetime." example = "2022-03-02T03:32:38.487000+00:00/.." regex_datetime = r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?)" regex_interval_closed = r"{i}\/{i}".format(i=regex_datetime) @@ -3525,7 +3525,7 @@ def service_api_route_info(service_api, settings): def datetime_interval_parser(datetime_interval): - # type: (Datetime) -> DatetimeIntervalType + # type: (str) -> DatetimeIntervalType """This function parse a given datetime or interval into a dictionary that will be easy for database process""" parsed_datetime = {} From 6c21dbf54836db8e72f56873945832738294589b Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Mon, 31 May 2021 12:00:37 -0400 Subject: [PATCH 20/22] fix comments --- CHANGES.rst | 4 ++++ weaver/wps_restapi/swagger_definitions.py | 14 ++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4e30a124e..46ef9630a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,10 @@ Changes: - Add reference link to ReadTheDocs URL of `Weaver` in API landing page. - Add references to `OGC-API Processes` requirements and recommendations for eventual conformance listing (relates to `#231 `_). +- Add datetime parameters for job searches queries + (relates to `#236 `_). +- Add limit validation and integration for jobs in retrieve queries + (relates to `#237 `_). Fixes: ------ diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index cdd7d5c2a..64b2ad9bc 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -235,11 +235,13 @@ class URL(ExtendedSchemaNode): class DateTimeInterval(ExtendedSchemaNode): schema_type = String - description = "DateTime format against OGC-API - Processes,\ - to get values before a certain date-time use '../' before the date-time,\ - to get values after a certain date-time use '/..' after the date-time like the example,\ - to get values between two date-times use '/' between the date-times,\ - to get values with a specific date-time just pass the datetime." + description = ( + "DateTime format against OGC-API - Processes,\ + to get values before a certain date-time use '../' before the date-time,\ + to get values after a certain date-time use '/..' after the date-time like the example,\ + to get values between two date-times use '/' between the date-times,\ + to get values with a specific date-time just pass the datetime." + ) example = "2022-03-02T03:32:38.487000+00:00/.." regex_datetime = r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?)" regex_interval_closed = r"{i}\/{i}".format(i=regex_datetime) @@ -3526,7 +3528,7 @@ def service_api_route_info(service_api, settings): def datetime_interval_parser(datetime_interval): # type: (str) -> DatetimeIntervalType - """This function parse a given datetime or interval into a dictionary that will be easy for database process""" + """This function parses a given datetime or interval into a dictionary that will be easy for database process.""" parsed_datetime = {} if datetime_interval.startswith(DATETIME_INTERVAL_OPEN_START_SYMBOL): From a48eba6400ee340d8f4e121741073ee445a09bec Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Wed, 2 Jun 2021 12:58:58 -0400 Subject: [PATCH 21/22] fix comments --- CHANGES.rst | 4 ++-- weaver/wps_restapi/swagger_definitions.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 46ef9630a..e57d94c30 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,9 +13,9 @@ Changes: - Add reference link to ReadTheDocs URL of `Weaver` in API landing page. - Add references to `OGC-API Processes` requirements and recommendations for eventual conformance listing (relates to `#231 `_). -- Add datetime parameters for job searches queries +- Add ``datetime`` query parameter for job searches queries (relates to `#236 `_). -- Add limit validation and integration for jobs in retrieve queries +- Add ``limit`` query parameter validation and integration for jobs in retrieve queries (relates to `#237 `_). Fixes: diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 64b2ad9bc..50e40a4e7 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -236,11 +236,11 @@ class URL(ExtendedSchemaNode): class DateTimeInterval(ExtendedSchemaNode): schema_type = String description = ( - "DateTime format against OGC-API - Processes,\ - to get values before a certain date-time use '../' before the date-time,\ - to get values after a certain date-time use '/..' after the date-time like the example,\ - to get values between two date-times use '/' between the date-times,\ - to get values with a specific date-time just pass the datetime." + "DateTime format against OGC-API - Processes," + "to get values before a certain date-time use '../' before the date-time," + "to get values after a certain date-time use '/..' after the date-time like the example," + "to get values between two date-times use '/' between the date-times," + "to get values with a specific date-time just pass the datetime." ) example = "2022-03-02T03:32:38.487000+00:00/.." regex_datetime = r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?)" From 9f2a3ff85e469c0ea49185b4a94fd1da72dd05c1 Mon Sep 17 00:00:00 2001 From: Nadir Trapsida Date: Wed, 2 Jun 2021 13:39:23 -0400 Subject: [PATCH 22/22] fix comments --- weaver/wps_restapi/swagger_definitions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 50e40a4e7..6354f1e3b 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -236,11 +236,11 @@ class URL(ExtendedSchemaNode): class DateTimeInterval(ExtendedSchemaNode): schema_type = String description = ( - "DateTime format against OGC-API - Processes," - "to get values before a certain date-time use '../' before the date-time," - "to get values after a certain date-time use '/..' after the date-time like the example," - "to get values between two date-times use '/' between the date-times," - "to get values with a specific date-time just pass the datetime." + "DateTime format against OGC-API - Processes, " + "to get values before a certain date-time use '../' before the date-time, " + "to get values after a certain date-time use '/..' after the date-time like the example, " + "to get values between two date-times use '/' between the date-times, " + "to get values with a specific date-time just pass the datetime. " ) example = "2022-03-02T03:32:38.487000+00:00/.." regex_datetime = r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(([+-]\d\d:\d\d)|Z)?)"