fix schema references and properties in rendered JSON schema/contents

CHANGES.rst

@@ -12,11 +12,17 @@ Changes
 
 Changes:
 --------
-- No change.
+- Add schema validation and reference to the `API` landing page, with additional parameters to respect `OGC` schema.
+- Add multiple `JSON` schema references for schema classes that are represented by corresponding `OGC` definitions.
 
 Fixes:
 ------
-- No change.
+- Fix auto-insertion of ``$schema`` and ``$id`` URI references into `JSON` schema and their data content representation.
+  When in `OpenAPI` context, schemas now correctly report their ``$id`` as the reference schema they represent (usually
+  from external `OGC` schema references), and ``$schema`` as the `JSON` meta-schema. When representing `JSON` data
+  contents validated against a `JSON` schema, the ``$schema`` property is used instead to refer to that schema.
+  All auto-insertions of these references can be enabled or disabled with options depending on what is more sensible
+  for presenting results from various `API` responses.
 
 .. _changes_4.33.0:
 

tests/wps_restapi/test_colander_extras.py

@@ -13,7 +13,7 @@
 from weaver.wps_restapi import colander_extras as ce, swagger_definitions as sd
 
 if TYPE_CHECKING:
-    from typing import List, Tuple, Type, Union
+    from typing import List, Optional, Tuple, Type, Union
 
     from weaver.typedefs import JSON
 
@@ -946,3 +946,158 @@ def test_media_type_pattern():
             pass
         else:
             pytest.fail(f"Expected valid format from [{test_schema.__name__}] with: '{test_value}'")
+
+
+@pytest.mark.parametrize(
+    [
+        "schema",
+        "schema_include",
+        "schema_include_deserialize",
+        "schema_include_convert_type",
+        "schema_expected_deserialize",
+        "schema_expected_convert_type",
+        "schema_meta",
+        "schema_meta_include",
+        "schema_meta_include_convert_type",
+        "schema_meta_expected_convert_type",
+    ],
+    [
+        (
+            "https://schema.com/item", True, True, True, "https://schema.com/item", "https://schema.com/item",
+            "https://json-schema.com#", True, True, "https://json-schema.com#",
+        ),
+        (
+            "https://schema.com/item", False, True, True, None, None,
+            "https://json-schema.com#", True, True, "https://json-schema.com#",
+        ),
+        (
+            "https://schema.com/item", True, True, True, "https://schema.com/item", "https://schema.com/item",
+            "https://json-schema.com#", False, True, None,
+        ),
+        (
+            "https://schema.com/item", False, True, True, None, None,
+            "https://json-schema.com#", False, True, None,
+        ),
+        (
+            None, True, True, True, None, None,
+            "https://json-schema.com#", True, True, "https://json-schema.com#",
+        ),
+        (
+            "https://schema.com/item", True, True, True, "https://schema.com/item", "https://schema.com/item",
+            None, True, True, None,
+        ),
+        (
+            "https://schema.com/item", True, False, True, None, "https://schema.com/item",
+            "https://json-schema.com#", True, True, "https://json-schema.com#",
+        ),
+        (
+            "https://schema.com/item", True, True, False, "https://schema.com/item", None,
+            "https://json-schema.com#", True, False, None,
+        ),
+        (
+            None, True, True, True, None, None,
+            None, True, True, None,
+        ),
+        (
+            # even when provided by attribute/argument, invalid URIs are ignored
+            # this is to avoid injecting them and generate invalid JSON schema/data instances
+            "--not-an-uri!", True, True, True, None, None,
+            "://not_an_uri", True, True, None,
+        ),
+    ]
+)
+def test_schema_ref_resolution(
+    schema,                             # type: Optional[str]
+    schema_include,                     # type: bool
+    schema_include_deserialize,         # type: bool
+    schema_include_convert_type,        # type: bool
+    schema_expected_deserialize,        # type: Optional[str]
+    schema_expected_convert_type,       # type: Optional[str]
+    schema_meta,                        # type: Optional[str]
+    schema_meta_include,                # type: bool
+    schema_meta_include_convert_type,   # type: bool
+    schema_meta_expected_convert_type,  # type: Optional[str]
+):                                      # type: (...) -> None
+    class MapByAttribute(ce.ExtendedMappingSchema):
+        title = "Item"
+        child = ce.ExtendedSchemaNode(ce.ExtendedString())
+
+    # do the same as if _schema, _schema_meta, etc. were set at the same place as 'title'
+    # but use 'setattr' here to allow combinations with/without them being set as needed
+    for schema_field, schema_value in [
+        ("_schema", schema),
+        ("_schema_include", schema_include),
+        ("_schema_include_deserialize", schema_include_deserialize),
+        ("_schema_include_convert_type", schema_include_convert_type),
+        ("_schema_meta", schema_meta),
+        ("_schema_meta_include", schema_meta_include),
+        ("_schema_meta_include_convert_type", schema_meta_include_convert_type),
+    ]:
+        setattr(MapByAttribute, schema_field, schema_value)
+
+    class MapByArgument(ce.ExtendedMappingSchema):
+        title = "Item"
+        child = ce.ExtendedSchemaNode(ce.ExtendedString())
+
+    # must be valid for both mappings above
+    test_data = {"child": "test"}
+
+    for schema_node in [
+        MapByAttribute(),
+        MapByArgument(
+            schema=schema,
+            schema_include=schema_include,
+            schema_include_deserialize=schema_include_deserialize,
+            schema_include_convert_type=schema_include_convert_type,
+            schema_meta=schema_meta,
+            schema_meta_include=schema_meta_include,
+            schema_meta_include_convert_type=schema_meta_include_convert_type,
+        ),
+    ]:
+        dispatcher = ce.OAS3TypeConversionDispatcher()
+        schema_json = dispatcher(schema_node)
+        schema_data = schema_node.deserialize(test_data)
+
+        err_msg = f"Failed using {schema_node}"
+        assert "$id" not in schema_data, err_msg
+        if schema_expected_convert_type:
+            assert "$id" in schema_json, err_msg
+            assert schema_json["$id"] == schema_expected_convert_type, err_msg
+        else:
+            assert "$id" not in schema_json, err_msg
+        if schema_expected_deserialize:
+            assert "$schema" in schema_data, err_msg
+            assert schema_data["$schema"] == schema_expected_deserialize, err_msg
+        else:
+            assert "$schema" not in schema_data, err_msg
+        if schema_meta_expected_convert_type:
+            assert "$schema" in schema_json, err_msg
+            assert schema_json["$schema"] == schema_meta_expected_convert_type, err_msg
+        else:
+            assert "$schema" not in schema_json, err_msg
+
+        # check that the remaining of the resolution is as expected
+        schema_json.pop("$id", None)
+        schema_data.pop("$id", None)
+        schema_json.pop("$schema", None)
+        schema_data.pop("$schema", None)
+        assert schema_data == test_data, err_msg
+        assert schema_json == {
+            "type": "object",
+            "title": "Item",
+            "required": ["child"],
+            "properties": {
+                "child": {"type": "string", "title": "child"}
+            },
+            "additionalProperties": {}
+        }, err_msg
+
+
+def test_schema_ref_resolution_include():
+    schema_node = sd.ProviderSummarySchema()
+    assert schema_node._schema_meta_include is True, "Cannot run test without pre-condition"
+    assert schema_node._schema, "Cannot run test without pre-condition"
+
+    def_handler = ce.OAS3DefinitionHandler
+    schema_json = ce.OAS3BodyParameterConverter().convert(schema_node, def_handler)
+    print(schema_json)

weaver/wps_restapi/api.py

@@ -519,17 +519,21 @@ def api_frontpage_body(settings):
              "rel": "wps-schema-2", "type": ContentType.TEXT_XML,
              "title": "WPS 2.0 XML validation schemas entrypoint."},
         ])
-    return {
-        "message": "Weaver Information",
-        "configuration": weaver_config,
-        "description": __meta__.__description__,
-        "parameters": [
-            {"name": "api", "enabled": weaver_api, "url": weaver_api_url, "api": weaver_api_oas_ui},
-            {"name": "vault", "enabled": weaver_vault},
-            {"name": "wps", "enabled": weaver_wps, "url": weaver_wps_url},
-        ],
-        "links": weaver_links,
-    }
+    body = sd.FrontpageSchema().deserialize(
+        {
+            "message": "Weaver Information",
+            "configuration": weaver_config,
+            "description": __meta__.__description__,
+            "attribution": __meta__.__author__,
+            "parameters": [
+                {"name": "api", "enabled": weaver_api, "url": weaver_api_url, "api": weaver_api_oas_ui},
+                {"name": "vault", "enabled": weaver_vault},
+                {"name": "wps", "enabled": weaver_wps, "url": weaver_wps_url},
+            ],
+            "links": weaver_links,
+        }
+    )
+    return body
 
 
 @sd.api_versions_service.get(tags=[sd.TAG_API], renderer=OutputFormat.JSON,

weaver/wps_restapi/colander_extras.py

@@ -123,10 +123,12 @@
 NO_DOUBLE_SLASH_PATTERN = r"(?!.*//.*$)"
 URL_REGEX = colander.URL_REGEX.replace(r"://)?", rf"://)?{NO_DOUBLE_SLASH_PATTERN}")
 URL = colander.Regex(URL_REGEX, msg=colander._("Must be a URL"), flags=re.IGNORECASE)
-URI_REGEX = colander.URI_REGEX.replace(r"://", r"://(?!//)")
-FILE_URI = colander.Regex(URI_REGEX, msg=colander._("Must be a file:// URI scheme"), flags=re.IGNORECASE)
+FILE_URL_REGEX = colander.URI_REGEX.replace(r"://", r"://(?!//)")
+FILE_URI = colander.Regex(FILE_URL_REGEX, msg=colander._("Must be a file:// URI scheme"), flags=re.IGNORECASE)
+URI_REGEX = rf"{colander.URL_REGEX[:-1]}(?:#?|[#?]\S+)$"
+URI = colander.Regex(URI_REGEX, msg=colander._("Must be a URI"), flags=re.IGNORECASE)
 STRING_FORMATTERS.update({
-    "uri": {"converter": BaseStringTypeConverter, "validator": URL},
+    "uri": {"converter": BaseStringTypeConverter, "validator": URI},
     "url": {"converter": BaseStringTypeConverter, "validator": URL},
     "file": {"converter": BaseStringTypeConverter, "validator": FILE_URI},
 })
@@ -1180,31 +1182,52 @@ class SchemaRefMappingSchema(ExtendedNodeInterface, ExtendedSchemaBase):
     """
     Mapping schema that supports auto-insertion of JSON-schema references provided in the definition.
 
-    When the :class:`colander.MappingSchema` defines ``_schema = "<URL>"`` with a valid URL,
-    all validations will automatically insert the corresponding ``$schema`` or ``$id`` field with this URL to
-    the deserialized :term:`OpenAPI` schema using :class:`SchemaRefConverter`, and to the deserialized :term:`JSON`
-    content, respectively. When injecting the ``$id`` reference into the :term:`JSON` object, the ``$schema`` will
-    instead refer to the ``schema_meta`` attribute that default to the :term:`JSON` meta-schema.
+    Schema references are resolved under two distinct contexts:
+
+    1. When generating the :term:`JSON` schema representation of the current schema node, for :term:`OpenAPI`
+       representation, the ``_schema`` attribute will indicate the ``$id`` value that identifies this schema,
+       while the ``_schema_meta`` will provide the ``$schema`` property that refers to the :term:`JSON` meta-schema
+       used by default to define it.
+
+    2. When deserializing :term:`JSON` data that should be validated against the current schema node, the generated
+       :term:`JSON` data will include the ``$schema`` property using the ``_schema`` attribute. In this case,
+       the ``$id`` is omitted as that :term:`JSON` represents an instance of the schema, but not its identity.
 
     Alternatively, the parameters ``schema`` and ``schema_meta`` can be passed as keyword arguments when instantiating
-    the schema node. The references injection can be disabled with ``schema_meta_include`` and ``schema_include``.
+    the schema node. The references injection in the :term:`JSON` schema and data can be disabled with parameters
+    ``schema_include`` and ``schema_meta_include``, or the corresponding class attributes. Furthermore, options
+    ``schema_include_deserialize``, ``schema_include_convert_type`` and ``schema_meta_include_convert_type`` can be
+    used to control individually each schema inclusion during either the type conversion context (:term:`JSON` schema)
+    or the deserialization context (:term:`JSON` data validation).
     """
     _extension = "_ext_schema_ref"
-    _ext_schema_options = ["_schema_meta", "_schema_meta_include", "_schema", "_schema_include"]
+    _ext_schema_options = [
+        "_schema_meta",
+        "_schema_meta_include",
+        "_schema_meta_include_convert_type",
+        "_schema",
+        "_schema_include",
+        "_schema_include_deserialize",
+        "_schema_include_convert_type",
+    ]
     _ext_schema_fields = ["_id", "_schema"]
 
     # typings and attributes to help IDEs flag that the field is available/overridable
 
     _schema_meta = Draft7Validator.META_SCHEMA["$schema"]  # type: str
-    _schema_meta_include = False    # type: bool
-    _schema = None                  # type: str
-    _schema_include = True          # type: bool
+    _schema_meta_include = True                 # type: bool
+    _schema_meta_include_convert_type = True    # type: bool
+
+    _schema = None                              # type: str
+    _schema_include = True                      # type: bool
+    _schema_include_deserialize = True          # type: bool
+    _schema_include_convert_type = True         # type: bool
 
     def __init__(self, *args, **kwargs):
         for schema_key in self._schema_options:
             schema_field = schema_key[1:]
-            schema_value = kwargs.pop(schema_field, None)
-            if schema_value not in ["", None]:
+            schema_value = kwargs.pop(schema_field, object)
+            if schema_value is not object:
                 setattr(self, schema_key, schema_value)
         super(SchemaRefMappingSchema, self).__init__(*args, **kwargs)
         setattr(self, SchemaRefMappingSchema._extension, True)
@@ -1219,7 +1242,7 @@ def __init__(self, *args, **kwargs):
     @staticmethod
     def _is_schema_ref(schema_ref):
         # type: (Any) -> bool
-        return isinstance(schema_ref, str) and URL.match_object.match(schema_ref)
+        return isinstance(schema_ref, str) and URI.match_object.match(schema_ref)
 
     @property
     def _schema_options(self):
@@ -1231,6 +1254,9 @@ def _schema_fields(self):
 
     def _schema_deserialize(self, cstruct, schema_meta, schema_id):
         # type: (OpenAPISchema, Optional[str], Optional[str]) -> OpenAPISchema
+        """
+        Applies the relevant schema references and properties depending on :term:`JSON` schema/data conversion context.
+        """
         if not isinstance(cstruct, dict):
             return cstruct
         if not getattr(self, SchemaRefMappingSchema._extension, False):
@@ -1255,20 +1281,38 @@ def _schema_deserialize(self, cstruct, schema_meta, schema_id):
         return schema_result
 
     def _deserialize_impl(self, cstruct):  # pylint: disable=W0222,signature-differs
+        """
+        Converts the data using validation against the :term:`JSON` schema definition.
+        """
+        # meta-schema always disabled in this context since irrelevant
+        # refer to the "id" of the parent schema representing this data using "$schema"
+        # this is not "official" JSON requirement, but very common in practice
+        schema_id = None
+        schema_id_include = getattr(self, "_schema_include", False)
+        schema_id_include_deserialize = getattr(self, "_schema_include_deserialize", False)
+        if schema_id_include and schema_id_include_deserialize:
+            schema_id = getattr(self, "_schema", None)
+        if schema_id:
+            return self._schema_deserialize(cstruct, schema_id, None)
+        return cstruct
+
+    def convert_type(self, cstruct):  # pylint: disable=W0222,signature-differs
+        """
+        Converts the node to obtain the :term:`JSON` schema definition.
+        """
         schema_id = schema_meta = None
         schema_id_include = getattr(self, "_schema_include", False)
+        schema_id_include_convert_type = getattr(self, "_schema_include_convert_type", False)
         schema_meta_include = getattr(self, "_schema_meta_include", False)
-        if schema_meta_include:
-            schema_meta = getattr(self, "_schema_meta", None)
-        if schema_id_include:
+        schema_meta_include_convert_type = getattr(self, "_schema_meta_include_convert_type", False)
+        if schema_id_include and schema_id_include_convert_type:
             schema_id = getattr(self, "_schema", None)
+        if schema_meta_include and schema_meta_include_convert_type:
+            schema_meta = getattr(self, "_schema_meta", None)
         if schema_id or schema_meta:
             return self._schema_deserialize(cstruct, schema_meta, schema_id)
         return cstruct
 
-    def convert_type(self, cstruct):  # pylint: disable=W0222,signature-differs
-        return SchemaRefMappingSchema._deserialize_impl(self, cstruct)
-
     @staticmethod
     @abstractmethod
     def schema_type():