chore: move report schema to a share location; add a schema step

Author: Trong Nhan Mai

tests/integration/README.md

@@ -191,6 +191,7 @@ In case you want to debug the utility script itself, there is the verbose mode f
   * `"verify"`: runs the `macaron verify-policy` command.
   * `"compare"`: compares an output file with an expected output file.
   * `"shell"`: runs an arbitrary shell command.
+  * `"schema"`: validates an output file with a schema.
 * `options`: Configuration options for the step. These options are specific to the step kind. See their schema below.
 * `env` (`dict[string, string | null]`, optional): Key value pairs of environment variables being modified during the step after inheriting the environment in which the utility is executed within. Each value can be a string if you want to set a value to the environment variable, or null if you want to "unset" the variable.
 * `expect_fail` (`bool`, optional, default is `false`): If `true`, assert that the step must exit with non-zero code. This should be used for cases where we expect a command to fail.
@@ -214,10 +215,17 @@ In case you want to debug the utility script itself, there is the verbose mode f
 
 ### Compare step options Schema
 
-* `kind` (`"analysis_report_json_schema" | "analysis_report" | "policy_report" | "deps_report" | "vsa"`, required): The kind of JSON report to compare.
+* `kind` (`"analysis_report" | "policy_report" | "deps_report" | "vsa"`, required): The kind of JSON report to compare.
 * `result` (`string`, required): The output file (a relative path from test case directory).
 * `expected` (`string`, required): The expected output file (a relative path from test case directory).
 
+### Schema step options Schema
+
+* `kind` (`"json_schema"`, required): The kind of schema validation to perform. For now, only json-schema is supported.
+* `result` (`string`, required): The output file (a relative path from test case directory).
+* `schema_type` (`output_json_report`, required): The type of schema for the validation. These are the default schemas available for the integration test.
+* `custom_schema_path` (`string`, optional): The path to the custom schema (a relative path from test case directory). If it is provided, the validation will use this schema and ignore the schema corresponding with `schema_type`.
+
 ### Shell step options Schema
 
 * `cmd` (`string`, required): The shell command to run.

tests/integration/cases/micronaut-projects_micronaut-test/test.yaml

@@ -15,12 +15,12 @@ steps:
     - -c
     - micronaut_test_config.yaml
     - --skip-deps
-- name: Compare JSON report schema
-  kind: compare
+- name: Validate JSON report schema
+  kind: schema
   options:
-    kind: analysis_report_json_schema
+    kind: json_schema
+    schema_type: output_json_report
     result: output/reports/github_com/micronaut-projects/micronaut-test/micronaut-test.json
-    expected: ./report_schema.json
 - name: Compare dependency report
   kind: compare
   options:

tests/integration/run.py

@@ -75,13 +75,20 @@ def configure_logging(verbose: bool) -> None:
 
 
 COMPARE_SCRIPTS: dict[str, Sequence[str]] = {
-    "analysis_report_json_schema": ["tests", "analyze_report_json_schema", "schema_validate.py"],
     "analysis_report": ["tests", "analyze_json_output", "compare_analyze_json_output.py"],
     "policy_report": ["tests", "policy_engine", "compare_policy_reports.py"],
     "deps_report": ["tests", "dependency_analyzer", "compare_dependencies.py"],
     "vsa": ["tests", "vsa", "compare_vsa.py"],
 }
 
+VALIDATE_SCHEMA_SCRIPTS: dict[str, Sequence[str]] = {
+    "json_schema": ["tests", "schema_validation", "json_schema_validate.py"],
+}
+
+DEFAULT_SCHEMAS: dict[str, Sequence[str]] = {
+    "output_json_report": ["tests", "schema_validation", "report_schema.json"],
+}
+
 
 def check_required_file(cwd: str) -> Callable[[str], None]:
     """Check for a required file of a test case."""
@@ -209,6 +216,72 @@ def cmd(self, macaron_cmd: str) -> list[str]:
         return self.options["cmd"].strip().split()
 
 
+class SchemaStepOptions(TypedDict):
+    """The configuration options of a schema validation step."""
+
+    kind: str
+    result: str
+    schema_type: str
+    custom_schema_path: str | None
+
+
+@dataclass
+class SchemaStep(Step[SchemaStepOptions]):
+    """A schema validation step in a test case, which allows for validating a file against a schema."""
+
+    @staticmethod
+    def options_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map:
+        """Generate the schema of a schema validation step."""
+        if check_expected_result_files:
+            check_file = check_required_file(cwd)
+        else:
+            check_file = cfgv.check_string
+
+        return cfgv.Map(
+            "schema options",
+            None,
+            *[
+                cfgv.Required(
+                    key="kind",
+                    check_fn=cfgv.check_one_of(tuple(VALIDATE_SCHEMA_SCRIPTS.keys())),
+                ),
+                cfgv.Required(
+                    key="result",
+                    check_fn=cfgv.check_string,
+                ),
+                cfgv.Required(
+                    key="schema_type",
+                    check_fn=cfgv.check_one_of(tuple(DEFAULT_SCHEMAS.keys())),
+                ),
+                cfgv.Optional(
+                    key="custom_schema_path",
+                    default=None,
+                    check_fn=check_file,
+                ),
+            ],
+        )
+
+    def cmd(self, macaron_cmd: str) -> list[str]:
+        kind = self.options["kind"]
+        result_file = self.options["result"]
+        schema_type = self.options["schema_type"]
+        custom_schema_path = self.options["custom_schema_path"]
+
+        if custom_schema_path is None:
+            return [
+                "python",
+                os.path.abspath(os.path.join(*VALIDATE_SCHEMA_SCRIPTS[kind])),
+                *[result_file, os.path.abspath(os.path.join(*DEFAULT_SCHEMAS[schema_type]))],
+            ]
+
+        logger.info("A custom schema path at %s is given, using that instead.", custom_schema_path)
+        return [
+            "python",
+            os.path.abspath(os.path.join(*VALIDATE_SCHEMA_SCRIPTS[kind])),
+            *[result_file, custom_schema_path],
+        ]
+
+
 class CompareStepOptions(TypedDict):
     """Configuration of a compare step."""
 
@@ -474,6 +547,7 @@ def gen_step_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map:
                         "compare",
                         "analyze",
                         "verify",
+                        "schema",
                     ),
                 ),
             ),
@@ -483,6 +557,15 @@ def gen_step_schema(cwd: str, check_expected_result_files: bool) -> cfgv.Map:
                 key="options",
                 schema=ShellStep.options_schema(),
             ),
+            cfgv.ConditionalRecurse(
+                condition_key="kind",
+                condition_value="schema",
+                key="options",
+                schema=SchemaStep.options_schema(
+                    cwd=cwd,
+                    check_expected_result_files=check_expected_result_files,
+                ),
+            ),
             cfgv.ConditionalRecurse(
                 condition_key="kind",
                 condition_value="compare",
@@ -700,6 +783,7 @@ def parse_step_config(step_id: int, step_config: Mapping) -> Step:
         "verify": VerifyStep,
         "shell": ShellStep,
         "compare": CompareStep,
+        "schema": SchemaStep,
     }[kind]
     return step_cls(  # type: ignore  # https://github.com/python/mypy/issues/3115
         step_id=step_id,