From a16dc9f2dd4a388976a049a0be4e6ab45cbbe045 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 13 May 2021 15:33:09 -0400 Subject: [PATCH 1/2] apply fix-lint following hack around --- weaver/datatype.py | 1 + weaver/processes/wps_workflow.py | 1 + weaver/store/mongodb.py | 4 ++++ weaver/wps/service.py | 1 + weaver/wps_restapi/colander_extras.py | 7 +++++++ 5 files changed, 14 insertions(+) diff --git a/weaver/datatype.py b/weaver/datatype.py index 784bc6d2f..42be656ff 100644 --- a/weaver/datatype.py +++ b/weaver/datatype.py @@ -56,6 +56,7 @@ class Base(dict): Explicitly overridden ``getter``/``setter`` attributes are called instead of ``dict``-key ``get``/``set``-item to ensure corresponding checks and/or value adjustments are executed before applying it to the sub-``dict``. """ + def __setattr__(self, item, value): # use the existing property setter if defined prop = getattr(type(self), item) diff --git a/weaver/processes/wps_workflow.py b/weaver/processes/wps_workflow.py index 4a1d39e96..514dec3df 100644 --- a/weaver/processes/wps_workflow.py +++ b/weaver/processes/wps_workflow.py @@ -110,6 +110,7 @@ class WpsWorkflow(ProcessCWL): Definition of a ``CWL`` ``workflow`` that can execute ``WPS`` application packages as defined by :class:`weaver.processes.wps_package.WpsPackage` as job steps. """ + def __init__(self, toolpath_object, loading_context, get_job_process_definition): # type: (Dict[Text, Any], LoadingContext, GetJobProcessDefinitionFunction) -> None super(WpsWorkflow, self).__init__(toolpath_object, loading_context) diff --git a/weaver/store/mongodb.py b/weaver/store/mongodb.py index 52931ec3e..34fba84eb 100644 --- a/weaver/store/mongodb.py +++ b/weaver/store/mongodb.py @@ -176,6 +176,7 @@ class MongodbProcessStore(StoreProcesses, MongodbStore): """ Registry for processes. Uses `MongoDB` to store processes and attributes. """ + def __init__(self, *args, **kwargs): db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs) StoreProcesses.__init__(self) @@ -387,6 +388,7 @@ class MongodbJobStore(StoreJobs, MongodbStore): """ Registry for process jobs tracking. Uses `MongoDB` to store job attributes. """ + def __init__(self, *args, **kwargs): db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs) StoreJobs.__init__(self) @@ -637,6 +639,7 @@ class MongodbQuoteStore(StoreQuotes, MongodbStore): """ Registry for quotes. Uses `MongoDB` to store quote attributes. """ + def __init__(self, *args, **kwargs): db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs) StoreQuotes.__init__(self) @@ -708,6 +711,7 @@ class MongodbBillStore(StoreBills, MongodbStore): """ Registry for bills. Uses `MongoDB` to store bill attributes. """ + def __init__(self, *args, **kwargs): db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs) StoreBills.__init__(self) diff --git a/weaver/wps/service.py b/weaver/wps/service.py index 3f1f492df..826bfc658 100644 --- a/weaver/wps/service.py +++ b/weaver/wps/service.py @@ -99,6 +99,7 @@ class WorkerService(ServiceWPS): When receiving ``Execute`` request, convert the XML payload to corresponding JSON and dispatch it to the Celery Worker to actually process it after job setup for monitoring. """ + def __init__(self, *_, is_worker=False, settings=None, **__): super(WorkerService, self).__init__(*_, **__) self.is_worker = is_worker diff --git a/weaver/wps_restapi/colander_extras.py b/weaver/wps_restapi/colander_extras.py index b34cb75c0..f02a436a0 100644 --- a/weaver/wps_restapi/colander_extras.py +++ b/weaver/wps_restapi/colander_extras.py @@ -104,6 +104,7 @@ class OneOfCaseInsensitive(colander.OneOf): """ Validator that ensures the given value matches one of the available choices, but allowing case insensitive values. """ + def __call__(self, node, value): if str(value).lower() not in (choice.lower() for choice in self.choices): return super(OneOfCaseInsensitive, self).__call__(node, value) @@ -113,6 +114,7 @@ class StringRange(colander.Range): """ Validator that provides the same functionalities as :class:`colander.Range` for a numerical string value. """ + def __init__(self, min=None, max=None, **kwargs): try: if isinstance(min, str): @@ -139,6 +141,7 @@ class SchemeURL(colander.Regex): :class:`colander.url` [remote http(s)/ftp(s)] :class:`colander.file_uri` [local file://] """ + def __init__(self, schemes=None, msg=None, flags=re.IGNORECASE): if not schemes: schemes = [""] @@ -861,6 +864,7 @@ class AnyKeyObject(PermissiveMappingSchema): This class is only a shorthand definition of ``unknown`` keyword for convenience. All :class:`colander.MappingSchema` support this natively. """ + def __init__(self, *args, **kwargs): kwargs["unknown"] = "preserve" super(PermissiveMappingSchema, self).__init__(*args, **kwargs) @@ -1478,6 +1482,7 @@ def _deserialize_keyword(self, cstruct): class KeywordTypeConverter(TypeConverter): """Generic keyword converter that builds schema with a list of sub-schemas under the keyword.""" + def convert_type(self, schema_node): keyword = schema_node.get_keyword_name() keyword_schema = { @@ -1500,6 +1505,7 @@ class OneOfKeywordTypeConverter(KeywordTypeConverter): .. seealso:: - :class:`OneOfKeywordSchema` """ + def convert_type(self, schema_node): # type: (OneOfKeywordSchema) -> Dict keyword = schema_node.get_keyword_name() @@ -1568,6 +1574,7 @@ class VariableObjectTypeConverter(ObjectTypeConverter): Updates the mapping object's ``additionalProperties`` for each ``properties`` that a marked as :class:`VariableSchemaNode`. """ + def convert_type(self, schema_node): converted = super(VariableObjectTypeConverter, self).convert_type(schema_node) converted.setdefault("additionalProperties", {}) From cf9be78209a369f2f426d2a1bf222df2fa7e50d1 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 13 May 2021 15:36:59 -0400 Subject: [PATCH 2/2] reapply hack around pycodestyle not respecting noqa E241 via autopep8 (see https://github.com/PyCQA/pycodestyle/issues/996) --- Makefile | 14 +++++++++++++- setup.cfg | 24 ++++++++++++++---------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 1c38638b9..d72be2ae6 100644 --- a/Makefile +++ b/Makefile @@ -469,12 +469,24 @@ fix-imports-only: mkdir-reports ## apply import code checks corrections isort --recursive $(APP_ROOT) \ 1> >(tee "$(REPORTS_DIR)/fixed-imports.txt")' +# FIXME: https://github.com/PyCQA/pycodestyle/issues/996 +# Tool "pycodestyle" doesn't respect "# noqa: E241" locally, but "flake8" and other tools do. +# Because "autopep8" uses "pycodestyle", it is impossible to disable locally extra spaces (as in tests to align values). +# Override the codes here from "setup.cfg" because "autopep8" also uses the "flake8" config, and we want to preserve +# global detection of those errors (typos, bad indents), unless explicitly added and excluded for readability purposes. +# WARNING: this will cause inconsistencies between what 'check-lint' detects and what 'fix-lint' can actually fix +_DEFAULT_SETUP_ERROR := E126,E226,E402,F401,W503,W504 +_EXTRA_SETUP_ERROR := E241 + .PHONY: fix-lint-only fix-lint-only: mkdir-reports ## fix some PEP8 code style problems automatically @echo "Fixing PEP8 code style problems..." @-rm -fr "$(REPORTS_DIR)/fixed-lint.txt" @bash -c '$(CONDA_CMD) \ - autopep8 -v -j 0 -i -r $(APP_ROOT) \ + autopep8 \ + --global-config "$(APP_ROOT)/setup.cfg" \ + --ignore "$(_DEFAULT_SETUP_ERROR),$(_EXTRA_SETUP_ERROR)" \ + -v -j 0 -i -r $(APP_ROOT) \ 1> >(tee "$(REPORTS_DIR)/fixed-lint.txt")' # FIXME: move parameters to setup.cfg when implemented (https://github.com/myint/docformatter/issues/10) diff --git a/setup.cfg b/setup.cfg index 060fd0ac7..3561a9852 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,21 +5,21 @@ tag = True tag_name = {new_version} [bumpversion:file:CHANGES.rst] -search = +search = `Unreleased `_ (latest) ======================================================================== -replace = +replace = `Unreleased `_ (latest) ======================================================================== - + Changes: -------- - No change. - + Fixes: ------ - No change. - + `{new_version} `_ ({now:%%Y-%%m-%%d}) ======================================================================== @@ -40,12 +40,12 @@ search = LABEL version="{current_version}" replace = LABEL version="{new_version}" [tool:pytest] -addopts = +addopts = --strict-markers --tb=native weaver/ python_files = test_*.py -markers = +markers = testbed14: mark test as 'testbed14' validation functional: mark test as functionality validation workflow: mark test as workflow execution (E2E) @@ -70,9 +70,12 @@ exclude = *.egg-info,build,dist,env,tests,./tests,test_* targets = . [flake8] +# FIXME: https://github.com/PyCQA/pycodestyle/issues/996 +# see Makefile for some specific overrides to avoid false-positive during 'fix-lint' +# new codes must be synced between below list and corresponding one in Makefile ignore = E126,E226,E402,F401,W503,W504 max-line-length = 120 -exclude = +exclude = src, .git, __pycache__, @@ -88,18 +91,19 @@ max-line-length = 120 ignore-path = docs/build,docs/source/autoapi [pylint] +# use .pylintrc [coverage:run] branch = true source = ./ include = weaver/* -omit = +omit = setup.py docs/* tests/* [coverage:report] -exclude_lines = +exclude_lines = pragma: no cover raise AssertionError raise NotImplementedError