From 261e4da13406c0c94724d2a30c3c2bdf172191a2 Mon Sep 17 00:00:00 2001 From: Alejandro Villar Date: Thu, 19 Oct 2023 15:17:17 +0200 Subject: [PATCH] Squashed commit of the following: commit 3d16db89c9abb58109d10254c8d795ee53f6448e Author: Alejandro Villar Date: Thu Oct 19 15:17:04 2023 +0200 Implement global HTML validation report commit 80accb2081dc300b15a214d06ef0c12b50152cd3 Merge: b33c633 659eb08 Author: Alejandro Villar Date: Thu Oct 19 12:30:05 2023 +0200 Merge branch 'master' into report-html commit b33c63357ef206450d332ce0aa88455f0c377955 Author: Alejandro Villar Date: Fri Oct 13 11:52:06 2023 +0200 WIP - HTML report --- ogc/bblocks/postprocess.py | 21 +++- ogc/bblocks/validate.py | 152 ++++++++++++++---------- ogc/bblocks/validation/report.html.mako | 120 +++++++++++++++++++ 3 files changed, 228 insertions(+), 65 deletions(-) create mode 100644 ogc/bblocks/validation/report.html.mako diff --git a/ogc/bblocks/postprocess.py b/ogc/bblocks/postprocess.py index 855b50d..03b9421 100644 --- a/ogc/bblocks/postprocess.py +++ b/ogc/bblocks/postprocess.py @@ -17,7 +17,7 @@ from ogc.bblocks.generate_docs import DocGenerator from ogc.bblocks.util import write_superbblocks_schemas, annotate_schema, BuildingBlock, \ write_jsonld_context, BuildingBlockRegister, ImportedBuildingBlocks -from ogc.bblocks.validate import validate_test_resources +from ogc.bblocks.validate import validate_test_resources, report_to_html from ogc.bblocks.transform import apply_transforms, transformers ANNOTATED_ITEM_CLASSES = ('schema', 'datatype') @@ -38,6 +38,9 @@ def postprocess(registered_items_path: str | Path = 'registereditems', cwd = Path().resolve() + if not isinstance(test_outputs_path, Path): + test_outputs_path = Path(test_outputs_path) + if base_url and base_url[-1] != '/': base_url += '/' @@ -64,6 +67,8 @@ def postprocess(registered_items_path: str | Path = 'registereditems', annotated_path=annotated_path, imported_bblocks=imported_bblocks) + validation_reports = [] + def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: try: @@ -119,10 +124,13 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: if not light: print(f" > Running tests for {bblock.identifier}", file=sys.stderr) - validation_passed, test_count = validate_test_resources(bblock, - bblocks_register=bbr, - outputs_path=test_outputs_path, - base_url=base_url) + validation_passed, test_count, json_report = validate_test_resources( + bblock, + bblocks_register=bbr, + outputs_path=test_outputs_path, + base_url=base_url) + validation_reports.append(json_report) + bblock.metadata['validationPassed'] = validation_passed if not validation_passed: bblock.metadata['status'] = 'invalid' @@ -251,6 +259,9 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: else: print(f"{building_block.identifier} failed postprocessing, skipping...", file=sys.stderr) + print(f"Writing full validation report to {test_outputs_path / 'report.html'}", file=sys.stderr) + report_to_html(json_reports=validation_reports, report_fn=test_outputs_path / 'report.html') + if output_file: output_register_json = { 'imports': imported_registers or [], diff --git a/ogc/bblocks/validate.py b/ogc/bblocks/validate.py index 9d8cbe6..4eaa6d8 100644 --- a/ogc/bblocks/validate.py +++ b/ogc/bblocks/validate.py @@ -18,6 +18,7 @@ from datetime import datetime, timezone import jsonschema +from mako import exceptions as mako_exceptions, template as mako_template import requests from jsonschema.validators import validator_for from ogc.na.util import load_yaml, is_url @@ -143,74 +144,107 @@ def uplifted_files(self) -> dict[str, tuple[Path, str]]: def report_to_dict(bblock: BuildingBlock, - items: Sequence[ValidationReportItem], - report_fn: Path | None = None, + items: Sequence[ValidationReportItem] | None, base_url: str | None = None) -> dict: result = { 'title': f"Validation report for {bblock.identifier} - {bblock.name}", 'bblockName': bblock.name, 'bblockId': bblock.identifier, 'generated': datetime.now(timezone.utc).astimezone().isoformat(), + 'result': True, 'items': [], } global_errors = {} cwd = Path().resolve() - for item in items: - source = { - 'type': item.source.type.name, - 'requireFail': item.source.require_fail, - } - if item.source.filename: - source['filename'] = str(os.path.relpath(item.source.filename, cwd)) - if base_url: - source['filename'] = urljoin(base_url, source['filename']) - if item.source.example_index: - source['exampleIndex'] = item.source.example_index - if item.source.snippet_index: - source['snippetIndex'] = item.source.snippet_index - if item.source.language: - source['language'] = item.source.language - - sections = {} - for section, entries in item.sections.items(): - sections[section.name] = [] - for entry in entries: - entry_dict = {} - if entry.payload: - for k, v in entry.payload.items(): - if isinstance(v, Path): - v = str(os.path.relpath(v.resolve(), cwd)) - if base_url: - v = urljoin(base_url, v) - elif k == 'files' and isinstance(v, list): - fv = [] - for f in v: - if isinstance(f, Path): - f = str(os.path.relpath(f.resolve(), cwd)) + failed_count = 0 + if items: + for item in items: + source = { + 'type': item.source.type.name, + 'requireFail': item.source.require_fail, + } + if item.failed: + result['result'] = False + failed_count += 1 + if item.source.filename: + source['filename'] = str(os.path.relpath(item.source.filename, cwd)) + if base_url: + source['filename'] = urljoin(base_url, source['filename']) + if item.source.example_index: + source['exampleIndex'] = item.source.example_index + if item.source.snippet_index: + source['snippetIndex'] = item.source.snippet_index + if item.source.language: + source['language'] = item.source.language + + sections = {} + for section, entries in item.sections.items(): + sections[section.name] = [] + for entry in entries: + entry_dict = {} + if entry.payload: + for k, v in entry.payload.items(): + if isinstance(v, Path): + v = str(os.path.relpath(v.resolve(), cwd)) if base_url: - f = urljoin(base_url, f) - fv.append(f) - v = fv - entry_dict[k] = v - entry_dict['is_error'] = entry.is_error - entry_dict['message'] = entry.message - if not entry.is_global: - sections[section.name].append(entry_dict) - elif entry.is_error: - global_errors.setdefault(section.name, entry_dict) - - res_item = { - 'source': source, - 'sections': sections, + v = urljoin(base_url, v) + elif k == 'files' and isinstance(v, list): + fv = [] + for f in v: + if isinstance(f, Path): + f = str(os.path.relpath(f.resolve(), cwd)) + if base_url: + f = urljoin(base_url, f) + fv.append(f) + v = fv + entry_dict[k] = v + entry_dict['isError'] = entry.is_error + entry_dict['message'] = entry.message + if not entry.is_global: + sections[section.name].append(entry_dict) + elif entry.is_error: + global_errors.setdefault(section.name, entry_dict) + + res_item = { + 'source': source, + 'result': not item.failed, + 'sections': sections, + } + result['items'].append(res_item) + result['globalErrors'] = global_errors + result['counts'] = { + 'total': len(result['items']), + 'passed': len(result['items']) - failed_count, + 'failed': failed_count, } - result['items'].append(res_item) - result['globalErrors'] = global_errors return result +def report_to_html(json_reports: list[dict], + report_fn: Path | None = None) -> str | None: + + pass_count = sum(r['result'] for r in json_reports) + counts = { + 'total': len(json_reports), + 'passed': pass_count, + 'failed': len(json_reports) - pass_count, + } + template = mako_template.Template(filename=str(Path(__file__).parent / 'validation/report.html.mako')) + try: + result = template.render(reports=json_reports, counts=counts) + except: + raise ValueError(mako_exceptions.text_error_template().render()) + + if report_fn: + with open(report_fn, 'w') as f: + f.write(result) + else: + return result + + def _validate_resource(bblock: BuildingBlock, filename: Path, output_filename: Path, @@ -595,17 +629,16 @@ def validate_inner(): def validate_test_resources(bblock: BuildingBlock, bblocks_register: BuildingBlockRegister, - outputs_path: str | Path | None = None, - base_url: str | None = None) -> tuple[bool, int]: + outputs_path: Path | None = None, + base_url: str | None = None) -> tuple[bool, int, dict]: final_result = True test_count = 0 if not bblock.tests_dir.is_dir() and not bblock.examples: - return final_result, test_count + return final_result, test_count, report_to_dict(bblock, None, base_url) shacl_graph = Graph() shacl_error = None - cwd = Path().resolve() shacl_files = [] try: @@ -638,7 +671,7 @@ def validate_test_resources(bblock: BuildingBlock, json_error = f"{type(e).__name__}: {e}" if outputs_path: - output_dir = Path(outputs_path) / bblock.subdirs + output_dir = outputs_path / bblock.subdirs else: output_dir = bblock.tests_dir.resolve() / OUTPUT_SUBDIR shutil.rmtree(output_dir, ignore_errors=True) @@ -746,12 +779,11 @@ def validate_test_resources(bblock: BuildingBlock, 'path': fn, }) - if all_results: - json_report = report_to_dict(bblock=bblock, items=all_results, base_url=base_url) - with open(output_dir / '_report.json', 'w') as f: - json.dump(json_report, f, indent=2) + json_report = report_to_dict(bblock=bblock, items=all_results, base_url=base_url) + with open(output_dir / '_report.json', 'w') as f: + json.dump(json_report, f, indent=2) - return final_result, test_count + return final_result, test_count, json_report class RefResolver(jsonschema.validators.RefResolver): diff --git a/ogc/bblocks/validation/report.html.mako b/ogc/bblocks/validation/report.html.mako new file mode 100644 index 0000000..11fd1c5 --- /dev/null +++ b/ogc/bblocks/validation/report.html.mako @@ -0,0 +1,120 @@ +<% +from html import escape as e +from datetime import datetime, timezone +from urllib.parse import urlparse, urljoin +from os.path import basename +from pathlib import Path +from ogc.na.util import is_url +import re +import os.path + +get_filename = lambda s: basename(urlparse(s).path) +%> + + + + + + Building Blocks validation report + + + + +
+

Building blocks validation report

+ Generated at ${e(datetime.now(timezone.utc).astimezone().isoformat())} + % if counts['total'] > 0: +

+ Number of passing building blocks: ${counts['passed']} / ${counts['total']} (${f"{counts['passed'] * 100 / counts['total']:.2f}".rstrip('0').rstrip('.')}%) +

+ % endif +
+ Expand all + Collapse all +
+
+ % for i, report in enumerate(reports): +
+

+ +

+
+
+ % if report['counts']['total'] > 0: +

+ Test passed: ${report['counts']['passed']} / ${report['counts']['total']} +

+ % endif + % for item in report['items']: +
+
+
+ ${e(re.sub(r'.*/', '', item['source']['filename']))} + ${e(item['source']['type'].replace('_', ' ').capitalize())} + % if item['source']['requireFail']: + Requires fail + % endif +
+ % if item['result']: + Passed + % else: + Failed + % endif +
+
+

+ % for subsection_title, section in item['sections'].items(): + % for entry in section: +

${e(entry['message'])}
+ % endfor + % endfor +

+
+
+ % endfor +
+
+
+ % endfor +
+
+ + + + \ No newline at end of file