Merge pull request #23208 from pgellert/sr/verbose-compat-4

CORE-6861 Schema Registry: verbose compat checks for JSON

src/v/pandaproxy/schema_registry/compatibility.cc

@@ -148,6 +148,250 @@ ss::sstring proto_incompatibility::describe() const {
       fmt::arg("path", _path.string()));
 }
 
+std::ostream& operator<<(std::ostream& os, const json_incompatibility_type& t) {
+    switch (t) {
+    case json_incompatibility_type::type_narrowed:
+        return os << "TYPE_NARROWED";
+    case json_incompatibility_type::type_changed:
+        return os << "TYPE_CHANGED";
+    case json_incompatibility_type::max_length_added:
+        return os << "MAX_LENGTH_ADDED";
+    case json_incompatibility_type::max_length_decreased:
+        return os << "MAX_LENGTH_DECREASED";
+    case json_incompatibility_type::min_length_added:
+        return os << "MIN_LENGTH_ADDED";
+    case json_incompatibility_type::min_length_increased:
+        return os << "MIN_LENGTH_INCREASED";
+    case json_incompatibility_type::pattern_added:
+        return os << "PATTERN_ADDED";
+    case json_incompatibility_type::pattern_changed:
+        return os << "PATTERN_CHANGED";
+    case json_incompatibility_type::maximum_added:
+        return os << "MAXIMUM_ADDED";
+    case json_incompatibility_type::maximum_decreased:
+        return os << "MAXIMUM_DECREASED";
+    case json_incompatibility_type::minimum_added:
+        return os << "MINIMUM_ADDED";
+    case json_incompatibility_type::minimum_increased:
+        return os << "MINIMUM_INCREASED";
+    case json_incompatibility_type::exclusive_maximum_added:
+        return os << "EXCLUSIVE_MAXIMUM_ADDED";
+    case json_incompatibility_type::exclusive_maximum_decreased:
+        return os << "EXCLUSIVE_MAXIMUM_DECREASED";
+    case json_incompatibility_type::exclusive_minimum_added:
+        return os << "EXCLUSIVE_MINIMUM_ADDED";
+    case json_incompatibility_type::exclusive_minimum_increased:
+        return os << "EXCLUSIVE_MINIMUM_INCREASED";
+    case json_incompatibility_type::multiple_of_added:
+        return os << "MULTIPLE_OF_ADDED";
+    case json_incompatibility_type::multiple_of_expanded:
+        return os << "MULTIPLE_OF_EXPANDED";
+    case json_incompatibility_type::multiple_of_changed:
+        return os << "MULTIPLE_OF_CHANGED";
+    case json_incompatibility_type::required_attribute_added:
+        return os << "REQUIRED_ATTRIBUTE_ADDED";
+    case json_incompatibility_type::max_properties_added:
+        return os << "MAX_PROPERTIES_ADDED";
+    case json_incompatibility_type::max_properties_decreased:
+        return os << "MAX_PROPERTIES_DECREASED";
+    case json_incompatibility_type::min_properties_added:
+        return os << "MIN_PROPERTIES_ADDED";
+    case json_incompatibility_type::min_properties_increased:
+        return os << "MIN_PROPERTIES_INCREASED";
+    case json_incompatibility_type::additional_properties_removed:
+        return os << "ADDITIONAL_PROPERTIES_REMOVED";
+    case json_incompatibility_type::additional_properties_narrowed:
+        return os << "ADDITIONAL_PROPERTIES_NARROWED";
+    case json_incompatibility_type::dependency_array_added:
+        return os << "DEPENDENCY_ARRAY_ADDED";
+    case json_incompatibility_type::dependency_array_extended:
+        return os << "DEPENDENCY_ARRAY_EXTENDED";
+    case json_incompatibility_type::dependency_array_changed:
+        return os << "DEPENDENCY_ARRAY_CHANGED";
+    case json_incompatibility_type::dependency_schema_added:
+        return os << "DEPENDENCY_SCHEMA_ADDED";
+    case json_incompatibility_type::property_added_to_open_content_model:
+        return os << "PROPERTY_ADDED_TO_OPEN_CONTENT_MODEL";
+    case json_incompatibility_type::
+      required_property_added_to_unopen_content_model:
+        return os << "REQUIRED_PROPERTY_ADDED_TO_UNOPEN_CONTENT_MODEL";
+    case json_incompatibility_type::property_removed_from_closed_content_model:
+        return os << "PROPERTY_REMOVED_FROM_CLOSED_CONTENT_MODEL";
+    case json_incompatibility_type::
+      property_removed_not_covered_by_partially_open_content_model:
+        return os << "PROPERTY_REMOVED_NOT_COVERED_BY_PARTIALLY_OPEN_CONTENT_"
+                     "MODEL";
+    case json_incompatibility_type::
+      property_added_not_covered_by_partially_open_content_model:
+        return os
+               << "PROPERTY_ADDED_NOT_COVERED_BY_PARTIALLY_OPEN_CONTENT_MODEL";
+    case json_incompatibility_type::reserved_property_removed:
+        return os << "RESERVED_PROPERTY_REMOVED";
+    case json_incompatibility_type::reserved_property_conflicts_with_property:
+        return os << "RESERVED_PROPERTY_CONFLICTS_WITH_PROPERTY";
+    case json_incompatibility_type::max_items_added:
+        return os << "MAX_ITEMS_ADDED";
+    case json_incompatibility_type::max_items_decreased:
+        return os << "MAX_ITEMS_DECREASED";
+    case json_incompatibility_type::min_items_added:
+        return os << "MIN_ITEMS_ADDED";
+    case json_incompatibility_type::min_items_increased:
+        return os << "MIN_ITEMS_INCREASED";
+    case json_incompatibility_type::unique_items_added:
+        return os << "UNIQUE_ITEMS_ADDED";
+    case json_incompatibility_type::additional_items_removed:
+        return os << "ADDITIONAL_ITEMS_REMOVED";
+    case json_incompatibility_type::additional_items_narrowed:
+        return os << "ADDITIONAL_ITEMS_NARROWED";
+    case json_incompatibility_type::item_added_to_open_content_model:
+        return os << "ITEM_ADDED_TO_OPEN_CONTENT_MODEL";
+    case json_incompatibility_type::item_removed_from_closed_content_model:
+        return os << "ITEM_REMOVED_FROM_CLOSED_CONTENT_MODEL";
+    case json_incompatibility_type::
+      item_removed_not_covered_by_partially_open_content_model:
+        return os << "ITEM_REMOVED_NOT_COVERED_BY_PARTIALLY_OPEN_CONTENT_MODEL";
+    case json_incompatibility_type::
+      item_added_not_covered_by_partially_open_content_model:
+        return os << "ITEM_ADDED_NOT_COVERED_BY_PARTIALLY_OPEN_CONTENT_MODEL";
+    case json_incompatibility_type::enum_array_narrowed:
+        return os << "ENUM_ARRAY_NARROWED";
+    case json_incompatibility_type::enum_array_changed:
+        return os << "ENUM_ARRAY_CHANGED";
+    case json_incompatibility_type::combined_type_changed:
+        return os << "COMBINED_TYPE_CHANGED";
+    case json_incompatibility_type::product_type_extended:
+        return os << "PRODUCT_TYPE_EXTENDED";
+    case json_incompatibility_type::sum_type_extended:
+        return os << "SUM_TYPE_EXTENDED";
+    case json_incompatibility_type::sum_type_narrowed:
+        return os << "SUM_TYPE_NARROWED";
+    case json_incompatibility_type::combined_type_subschemas_changed:
+        return os << "COMBINED_TYPE_SUBSCHEMAS_CHANGED";
+    case json_incompatibility_type::not_type_extended:
+        return os << "NOT_TYPE_EXTENDED";
+    case json_incompatibility_type::unknown:
+        return os << "UNKNOWN";
+    }
+    __builtin_unreachable();
+}
+
+std::string_view description_for_type(json_incompatibility_type t) {
+    switch (t) {
+    case json_incompatibility_type::maximum_added:
+    case json_incompatibility_type::minimum_added:
+    case json_incompatibility_type::exclusive_maximum_added:
+    case json_incompatibility_type::exclusive_minimum_added:
+    case json_incompatibility_type::multiple_of_added:
+    case json_incompatibility_type::max_length_added:
+    case json_incompatibility_type::min_length_added:
+    case json_incompatibility_type::pattern_added:
+    case json_incompatibility_type::required_attribute_added:
+    case json_incompatibility_type::max_properties_added:
+    case json_incompatibility_type::min_properties_added:
+    case json_incompatibility_type::dependency_array_added:
+    case json_incompatibility_type::dependency_schema_added:
+    case json_incompatibility_type::max_items_added:
+    case json_incompatibility_type::min_items_added:
+    case json_incompatibility_type::unique_items_added:
+    case json_incompatibility_type::additional_items_removed:
+    case json_incompatibility_type::additional_properties_removed:
+        return "The keyword at path '{path}' in the {{reader}} schema is not "
+               "present in the {{writer}} schema";
+    case json_incompatibility_type::min_length_increased:
+    case json_incompatibility_type::minimum_increased:
+    case json_incompatibility_type::exclusive_minimum_increased:
+    case json_incompatibility_type::min_properties_increased:
+    case json_incompatibility_type::multiple_of_expanded:
+    case json_incompatibility_type::min_items_increased:
+        return "The value at path '{path}' in the {{reader}} schema is more "
+               "than its value in the {{writer}} schema";
+    case json_incompatibility_type::max_length_decreased:
+    case json_incompatibility_type::maximum_decreased:
+    case json_incompatibility_type::max_items_decreased:
+    case json_incompatibility_type::exclusive_maximum_decreased:
+    case json_incompatibility_type::max_properties_decreased:
+        return "The value at path '{path}' in the {{reader}} schema is less "
+               "than its value in the {{writer}} schema";
+    case json_incompatibility_type::additional_items_narrowed:
+    case json_incompatibility_type::enum_array_narrowed:
+    case json_incompatibility_type::sum_type_narrowed:
+    case json_incompatibility_type::additional_properties_narrowed:
+        return "An array or combined type at path '{path}' has fewer elements "
+               "in the {{reader}} schema than the {{writer}} schema";
+    case json_incompatibility_type::pattern_changed:
+    case json_incompatibility_type::multiple_of_changed:
+    case json_incompatibility_type::dependency_array_changed:
+        return "The value at path '{path}' is different between the {{reader}} "
+               "and {{writer}} schema";
+    case json_incompatibility_type::type_changed:
+    case json_incompatibility_type::type_narrowed:
+    case json_incompatibility_type::combined_type_changed:
+    case json_incompatibility_type::combined_type_subschemas_changed:
+    case json_incompatibility_type::enum_array_changed:
+        return "A type at path '{path}' is different between the {{reader}} "
+               "schema and the {{writer}} schema";
+    case json_incompatibility_type::dependency_array_extended:
+    case json_incompatibility_type::product_type_extended:
+    case json_incompatibility_type::sum_type_extended:
+    case json_incompatibility_type::not_type_extended:
+        return "An array or combined type at path '{path}' has more elements "
+               "in the {{reader}} schema than the {{writer}} schema";
+    case json_incompatibility_type::property_added_to_open_content_model:
+    case json_incompatibility_type::item_added_to_open_content_model:
+        return "The {{reader}} schema has an open content model and has a "
+               "property or item at path '{path}' which is missing in the "
+               "{{writer}} schema";
+    case json_incompatibility_type::
+      required_property_added_to_unopen_content_model:
+        return "The {{reader}} schema has an unopen content model and has a "
+               "required property at path '{path}' which is missing in the "
+               "{{writer}} schema";
+    case json_incompatibility_type::property_removed_from_closed_content_model:
+    case json_incompatibility_type::item_removed_from_closed_content_model:
+        return "The {{reader}} has a closed content model and is missing a "
+               "property or item present at path '{path}' in the {{writer}} "
+               "schema";
+    case json_incompatibility_type::
+      property_removed_not_covered_by_partially_open_content_model:
+    case json_incompatibility_type::
+      item_removed_not_covered_by_partially_open_content_model:
+        return "A property or item is missing in the {{reader}} schema but "
+               "present at path '{path}' in the {{writer}} schema and is not "
+               "covered by its partially open content model";
+    case json_incompatibility_type::
+      property_added_not_covered_by_partially_open_content_model:
+    case json_incompatibility_type::
+      item_added_not_covered_by_partially_open_content_model:
+        return "The {{reader}} schema has a property or item at path '{path}' "
+               "which is missing in the {{writer}} schema and is not covered "
+               "by its partially open content model";
+    case json_incompatibility_type::reserved_property_removed:
+        return "The {{reader}} schema has reserved property '{path}' removed "
+               "from its metadata which is present in the {{writer}} schema.";
+    case json_incompatibility_type::reserved_property_conflicts_with_property:
+        return "The {{reader}} schema has property at path '{path}' that "
+               "conflicts with the reserved properties which is missing in the "
+               "{{writer}} schema.";
+
+    case json_incompatibility_type::unknown:
+        return "{{reader}} schema is not compatible with {{writer}} schema: "
+               "check '{path}'";
+    }
+    __builtin_unreachable();
+}
+
+std::ostream& operator<<(std::ostream& os, const json_incompatibility& v) {
+    fmt::print(
+      os, R"({{errorType:"{}", description:"{}"}})", v._type, v.describe());
+    return os;
+}
+
+ss::sstring json_incompatibility::describe() const {
+    return fmt::format(
+      fmt::runtime(description_for_type(_type)),
+      fmt::arg("path", _path.string()));
+}
+
 compatibility_result
 raw_compatibility_result::operator()(verbose is_verbose) && {
     compatibility_result result = {.is_compat = !has_error()};

src/v/pandaproxy/schema_registry/compatibility.h

@@ -136,6 +136,105 @@ class proto_incompatibility {
     Type _type;
 };
 
+enum class json_incompatibility_type {
+    type_narrowed = 0,
+    type_changed,
+    max_length_added,
+    max_length_decreased,
+    min_length_added,
+    min_length_increased,
+    pattern_added,
+    pattern_changed,
+    maximum_added,
+    maximum_decreased,
+    minimum_added,
+    minimum_increased,
+    exclusive_maximum_added,
+    exclusive_maximum_decreased,
+    exclusive_minimum_added,
+    exclusive_minimum_increased,
+    multiple_of_added,
+    multiple_of_expanded,
+    multiple_of_changed,
+    required_attribute_added,
+    max_properties_added,
+    max_properties_decreased,
+    min_properties_added,
+    min_properties_increased,
+    additional_properties_removed,
+    additional_properties_narrowed,
+    dependency_array_added,
+    dependency_array_extended,
+    dependency_array_changed,
+    dependency_schema_added,
+    property_added_to_open_content_model,
+    required_property_added_to_unopen_content_model,
+    property_removed_from_closed_content_model,
+    property_removed_not_covered_by_partially_open_content_model,
+    property_added_not_covered_by_partially_open_content_model,
+    reserved_property_removed,
+    reserved_property_conflicts_with_property,
+    max_items_added,
+    max_items_decreased,
+    min_items_added,
+    min_items_increased,
+    unique_items_added,
+    additional_items_removed,
+    additional_items_narrowed,
+    item_added_to_open_content_model,
+    item_removed_from_closed_content_model,
+    item_removed_not_covered_by_partially_open_content_model,
+    item_added_not_covered_by_partially_open_content_model,
+    enum_array_narrowed,
+    enum_array_changed,
+    combined_type_changed,
+    product_type_extended,
+    sum_type_extended,
+    sum_type_narrowed,
+    combined_type_subschemas_changed,
+    not_type_extended,
+    unknown,
+};
+
+/**
+ * json_incompatibility - A single incompatibility between JSON schemas.
+ *
+ * Encapsulates:
+ *   - the path to the location of the incompatibility in the _writer_ schema
+ *   - the type of incompatibility
+ *
+ * Primary interface is `describe`, which combines the contained info into
+ * a format string which can then be interpolated with identifying info for
+ * the reader and writer schemas in the request handler.
+ */
+class json_incompatibility {
+public:
+    using Type = json_incompatibility_type;
+    json_incompatibility(std::filesystem::path path, Type type)
+      : _path(std::move(path))
+      , _type(type) {}
+
+    ss::sstring describe() const;
+    Type type() const { return _type; }
+
+    friend std::ostream&
+    operator<<(std::ostream& os, const json_incompatibility& v);
+
+    friend bool
+    operator==(const json_incompatibility&, const json_incompatibility&)
+      = default;
+
+    // Helpful for unit testing
+    template<typename H>
+    friend H AbslHashValue(H h, const json_incompatibility& e) {
+        return H::combine(std::move(h), e._path.string(), e._type);
+    }
+
+private:
+    std::filesystem::path _path;
+    Type _type;
+};
+
 /**
  * raw_compatibility_result - A collection of unformatted proto or avro
  * incompatibilities. Its purpose is twofold:
@@ -146,12 +245,23 @@ class proto_incompatibility {
  *     incompatibilities into formatted error messages.
  */
 class raw_compatibility_result {
-    using schema_incompatibility
-      = std::variant<avro_incompatibility, proto_incompatibility>;
+    using schema_incompatibility = std::variant<
+      avro_incompatibility,
+      proto_incompatibility,
+      json_incompatibility>;
 
 public:
     raw_compatibility_result() = default;
 
+    template<typename T, typename... Args>
+    requires std::constructible_from<T, Args&&...>
+             && std::convertible_to<T, schema_incompatibility>
+    static auto of(Args&&... args) {
+        raw_compatibility_result res;
+        res.emplace<T>(std::forward<Args>(args)...);
+        return res;
+    }
+
     template<typename T, typename... Args>
     requires std::constructible_from<T, Args&&...>
              && std::convertible_to<T, schema_incompatibility>