Skip to content

Commit 3d576b9

Browse files
author
Trong Nhan Mai
committed
test: add analysis report JSON schema validation
Signed-off-by: Trong Nhan Mai <[email protected]>
1 parent e19c4b2 commit 3d576b9

File tree

6 files changed

+175
-1
lines changed

6 files changed

+175
-1
lines changed

‎tests/analyze_report_json_schema/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

‎tests/analyze_report_json_schema/schema_validate.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
3+
4+
"""This module validates the result JSON files against a JSON schema."""
5+
6+
import json
7+
import os
8+
import sys
9+
from collections.abc import Sequence
10+
11+
import jsonschema
12+
13+
14+
def main(argv: Sequence[str] | None = None) -> int:
15+
"""Run main logic."""
16+
if not argv or not len(argv) == 3:
17+
print("Usage: python3 schema_validate.py <json_path> <schema_path>")
18+
return os.EX_USAGE
19+
20+
data_path = sys.argv[1]
21+
schema_path = sys.argv[2]
22+
23+
schema = None
24+
with open(schema_path, encoding="utf-8") as file:
25+
try:
26+
schema = json.load(file)
27+
except json.JSONDecodeError as error:
28+
print(f"Failed to load schema at {schema_path}, err:\n{error}\n")
29+
return os.EX_DATAERR
30+
31+
data = None
32+
with open(data_path, encoding="utf-8") as file:
33+
try:
34+
data = json.load(file)
35+
except json.JSONDecodeError as error:
36+
print(f"Failed to load JSON data at {data_path}, err:\n{error}\n")
37+
return os.EX_DATAERR
38+
39+
try:
40+
jsonschema.validate(
41+
schema=schema,
42+
instance=data,
43+
)
44+
print(f"JSON data at {data_path} PASSED schema {schema_path}.")
45+
return os.EX_OK
46+
except jsonschema.ValidationError as error:
47+
print(f"JSON data at {data_path} FAILED schema {schema_path}, err:\n{error}\n")
48+
return os.EX_DATAERR
49+
except jsonschema.SchemaError as error:
50+
print(f"The schema at {schema_path} is not valid, err:\n{error}\n")
51+
return os.EX_DATAERR
52+
53+
54+
if __name__ == "__main__":
55+
raise SystemExit(main(sys.argv))

‎tests/integration/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ In case you want to debug the utility script itself, there is the verbose mode f
214214
215215
### Compare step options Schema
216216
217-
* `kind` (`"analysis_report" | "policy_report" | "deps_report" | "vsa"`, required): The kind of JSON report to compare.
217+
* `kind` (`"analysis_report_json_schema" | "analysis_report" | "policy_report" | "deps_report" | "vsa"`, required): The kind of JSON report to compare.
218218
* `result` (`string`, required): The output file (a relative path from test case directory).
219219
* `expected` (`string`, required): The expected output file (a relative path from test case directory).
220220

‎tests/integration/cases/micronaut-projects_micronaut-test/report_schema.json‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "macaron-json-report-schema",
4+
"title": "Report",
5+
"type": "object",
6+
"properties": {
7+
"metadata": {
8+
"type": "object"
9+
},
10+
"target": {
11+
"type": "object",
12+
"properties": {
13+
"info": {
14+
"type": "object"
15+
},
16+
"provenances": {
17+
"type": "object"
18+
},
19+
"checks": {
20+
"type": "object",
21+
"properties": {
22+
"summary": {
23+
"type": "object",
24+
"items": {
25+
"type": "integer"
26+
}
27+
},
28+
"results": {
29+
"type": "array",
30+
"items": {
31+
"type": "object",
32+
"properties": {
33+
"check_id": {
34+
"type": "string"
35+
},
36+
"check_description": {
37+
"type": "string"
38+
},
39+
"slsa_requirements": {
40+
"type": "array",
41+
"items": {
42+
"type": "string"
43+
}
44+
},
45+
"justification": {
46+
"type": "array",
47+
"items": {
48+
"oneOf": [
49+
{ "type": "string" },
50+
{ "type": "object" }
51+
]
52+
}
53+
},
54+
"result_type": {
55+
"type": "string"
56+
}
57+
}
58+
}
59+
}
60+
}
61+
}
62+
}
63+
},
64+
"dependencies": {
65+
"type": "object",
66+
"properties": {
67+
"analyzed_deps": {
68+
"type": "integer"
69+
},
70+
"unique_dep_repos": {
71+
"type": "integer"
72+
},
73+
"checks_summary": {
74+
"type": "array",
75+
"items": {
76+
"type": "object",
77+
"properties": {
78+
"check_id": {
79+
"type": "string"
80+
},
81+
"num_deps_pass": {
82+
"type": "integer"
83+
}
84+
}
85+
}
86+
},
87+
"dep_status": {
88+
"type": "array",
89+
"items": {
90+
"type": "object",
91+
"properties": {
92+
"id": {
93+
"type": "string"
94+
},
95+
"description": {
96+
"type": "string"
97+
},
98+
"report": {
99+
"type": "string"
100+
},
101+
"repo_url_status": {
102+
"type": "string"
103+
}
104+
}
105+
}
106+
}
107+
}
108+
}
109+
}
110+
}

‎tests/integration/cases/micronaut-projects_micronaut-test/test.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ steps:
1515
- -c
1616
- micronaut_test_config.yaml
1717
- --skip-deps
18+
- name: Compare JSON report schema
19+
kind: compare
20+
options:
21+
kind: analysis_report_json_schema
22+
result: output/reports/github_com/micronaut-projects/micronaut-test/micronaut-test.json
23+
expected: ./report_schema.json
1824
- name: Compare dependency report
1925
kind: compare
2026
options:

‎tests/integration/run.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ def configure_logging(verbose: bool) -> None:
7575

7676

7777
COMPARE_SCRIPTS: dict[str, Sequence[str]] = {
78+
"analysis_report_json_schema": ["tests", "analyze_report_json_schema", "schema_validate.py"],
7879
"analysis_report": ["tests", "analyze_json_output", "compare_analyze_json_output.py"],
7980
"policy_report": ["tests", "policy_engine", "compare_policy_reports.py"],
8081
"deps_report": ["tests", "dependency_analyzer", "compare_dependencies.py"],

0 commit comments

Comments
 (0)