Skip to content

Commit

Permalink
Squashed commit of the following:
Browse files Browse the repository at this point in the history 
commit 3d16db89c9abb58109d10254c8d795ee53f6448e
Author: Alejandro Villar <[email protected]>
Date:   Thu Oct 19 15:17:04 2023 +0200

    Implement global HTML validation report

commit 80accb2081dc300b15a214d06ef0c12b50152cd3
Merge: b33c633 659eb08
Author: Alejandro Villar <[email protected]>
Date:   Thu Oct 19 12:30:05 2023 +0200

    Merge branch 'master' into report-html

commit b33c63357ef206450d332ce0aa88455f0c377955
Author: Alejandro Villar <[email protected]>
Date:   Fri Oct 13 11:52:06 2023 +0200

    WIP - HTML report
  • Loading branch information
@avillar
avillar committed Oct 19, 2023
1 parent 659eb08 commit 261e4da
Show file tree
Hide file tree
Showing 3 changed files with 228 additions and 65 deletions.
21 changes: 16 additions & 5 deletions ogc/bblocks/postprocess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from ogc.bblocks.generate_docs import DocGenerator
from ogc.bblocks.util import write_superbblocks_schemas, annotate_schema, BuildingBlock, \
write_jsonld_context, BuildingBlockRegister, ImportedBuildingBlocks
from ogc.bblocks.validate import validate_test_resources
from ogc.bblocks.validate import validate_test_resources, report_to_html
from ogc.bblocks.transform import apply_transforms, transformers

ANNOTATED_ITEM_CLASSES = ('schema', 'datatype')
Expand All @@ -38,6 +38,9 @@ def postprocess(registered_items_path: str | Path = 'registereditems',

cwd = Path().resolve()

if not isinstance(test_outputs_path, Path):
test_outputs_path = Path(test_outputs_path)

if base_url and base_url[-1] != '/':
base_url += '/'

Expand All @@ -64,6 +67,8 @@ def postprocess(registered_items_path: str | Path = 'registereditems',
annotated_path=annotated_path,
imported_bblocks=imported_bblocks)

validation_reports = []

def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:

try:
Expand Down Expand Up @@ -119,10 +124,13 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:

if not light:
print(f" > Running tests for {bblock.identifier}", file=sys.stderr)
validation_passed, test_count = validate_test_resources(bblock,
bblocks_register=bbr,
outputs_path=test_outputs_path,
base_url=base_url)
validation_passed, test_count, json_report = validate_test_resources(
bblock,
bblocks_register=bbr,
outputs_path=test_outputs_path,
base_url=base_url)
validation_reports.append(json_report)

bblock.metadata['validationPassed'] = validation_passed
if not validation_passed:
bblock.metadata['status'] = 'invalid'
Expand Down Expand Up @@ -251,6 +259,9 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool:
else:
print(f"{building_block.identifier} failed postprocessing, skipping...", file=sys.stderr)

print(f"Writing full validation report to {test_outputs_path / 'report.html'}", file=sys.stderr)
report_to_html(json_reports=validation_reports, report_fn=test_outputs_path / 'report.html')

if output_file:
output_register_json = {
'imports': imported_registers or [],
Expand Down
152 changes: 92 additions & 60 deletions ogc/bblocks/validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from datetime import datetime, timezone

import jsonschema
from mako import exceptions as mako_exceptions, template as mako_template
import requests
from jsonschema.validators import validator_for
from ogc.na.util import load_yaml, is_url
Expand Down Expand Up @@ -143,74 +144,107 @@ def uplifted_files(self) -> dict[str, tuple[Path, str]]:


def report_to_dict(bblock: BuildingBlock,
items: Sequence[ValidationReportItem],
report_fn: Path | None = None,
items: Sequence[ValidationReportItem] | None,
base_url: str | None = None) -> dict:
result = {
'title': f"Validation report for {bblock.identifier} - {bblock.name}",
'bblockName': bblock.name,
'bblockId': bblock.identifier,
'generated': datetime.now(timezone.utc).astimezone().isoformat(),
'result': True,
'items': [],
}

global_errors = {}
cwd = Path().resolve()

for item in items:
source = {
'type': item.source.type.name,
'requireFail': item.source.require_fail,
}
if item.source.filename:
source['filename'] = str(os.path.relpath(item.source.filename, cwd))
if base_url:
source['filename'] = urljoin(base_url, source['filename'])
if item.source.example_index:
source['exampleIndex'] = item.source.example_index
if item.source.snippet_index:
source['snippetIndex'] = item.source.snippet_index
if item.source.language:
source['language'] = item.source.language

sections = {}
for section, entries in item.sections.items():
sections[section.name] = []
for entry in entries:
entry_dict = {}
if entry.payload:
for k, v in entry.payload.items():
if isinstance(v, Path):
v = str(os.path.relpath(v.resolve(), cwd))
if base_url:
v = urljoin(base_url, v)
elif k == 'files' and isinstance(v, list):
fv = []
for f in v:
if isinstance(f, Path):
f = str(os.path.relpath(f.resolve(), cwd))
failed_count = 0
if items:
for item in items:
source = {
'type': item.source.type.name,
'requireFail': item.source.require_fail,
}
if item.failed:
result['result'] = False
failed_count += 1
if item.source.filename:
source['filename'] = str(os.path.relpath(item.source.filename, cwd))
if base_url:
source['filename'] = urljoin(base_url, source['filename'])
if item.source.example_index:
source['exampleIndex'] = item.source.example_index
if item.source.snippet_index:
source['snippetIndex'] = item.source.snippet_index
if item.source.language:
source['language'] = item.source.language

sections = {}
for section, entries in item.sections.items():
sections[section.name] = []
for entry in entries:
entry_dict = {}
if entry.payload:
for k, v in entry.payload.items():
if isinstance(v, Path):
v = str(os.path.relpath(v.resolve(), cwd))
if base_url:
f = urljoin(base_url, f)
fv.append(f)
v = fv
entry_dict[k] = v
entry_dict['is_error'] = entry.is_error
entry_dict['message'] = entry.message
if not entry.is_global:
sections[section.name].append(entry_dict)
elif entry.is_error:
global_errors.setdefault(section.name, entry_dict)

res_item = {
'source': source,
'sections': sections,
v = urljoin(base_url, v)
elif k == 'files' and isinstance(v, list):
fv = []
for f in v:
if isinstance(f, Path):
f = str(os.path.relpath(f.resolve(), cwd))
if base_url:
f = urljoin(base_url, f)
fv.append(f)
v = fv
entry_dict[k] = v
entry_dict['isError'] = entry.is_error
entry_dict['message'] = entry.message
if not entry.is_global:
sections[section.name].append(entry_dict)
elif entry.is_error:
global_errors.setdefault(section.name, entry_dict)

res_item = {
'source': source,
'result': not item.failed,
'sections': sections,
}
result['items'].append(res_item)
result['globalErrors'] = global_errors
result['counts'] = {
'total': len(result['items']),
'passed': len(result['items']) - failed_count,
'failed': failed_count,
}
result['items'].append(res_item)
result['globalErrors'] = global_errors

return result


def report_to_html(json_reports: list[dict],
report_fn: Path | None = None) -> str | None:

pass_count = sum(r['result'] for r in json_reports)
counts = {
'total': len(json_reports),
'passed': pass_count,
'failed': len(json_reports) - pass_count,
}
template = mako_template.Template(filename=str(Path(__file__).parent / 'validation/report.html.mako'))
try:
result = template.render(reports=json_reports, counts=counts)
except:
raise ValueError(mako_exceptions.text_error_template().render())

if report_fn:
with open(report_fn, 'w') as f:
f.write(result)
else:
return result


def _validate_resource(bblock: BuildingBlock,
filename: Path,
output_filename: Path,
Expand Down Expand Up @@ -595,17 +629,16 @@ def validate_inner():

def validate_test_resources(bblock: BuildingBlock,
bblocks_register: BuildingBlockRegister,
outputs_path: str | Path | None = None,
base_url: str | None = None) -> tuple[bool, int]:
outputs_path: Path | None = None,
base_url: str | None = None) -> tuple[bool, int, dict]:
final_result = True
test_count = 0

if not bblock.tests_dir.is_dir() and not bblock.examples:
return final_result, test_count
return final_result, test_count, report_to_dict(bblock, None, base_url)

shacl_graph = Graph()
shacl_error = None
cwd = Path().resolve()

shacl_files = []
try:
Expand Down Expand Up @@ -638,7 +671,7 @@ def validate_test_resources(bblock: BuildingBlock,
json_error = f"{type(e).__name__}: {e}"

if outputs_path:
output_dir = Path(outputs_path) / bblock.subdirs
output_dir = outputs_path / bblock.subdirs
else:
output_dir = bblock.tests_dir.resolve() / OUTPUT_SUBDIR
shutil.rmtree(output_dir, ignore_errors=True)
Expand Down Expand Up @@ -746,12 +779,11 @@ def validate_test_resources(bblock: BuildingBlock,
'path': fn,
})

if all_results:
json_report = report_to_dict(bblock=bblock, items=all_results, base_url=base_url)
with open(output_dir / '_report.json', 'w') as f:
json.dump(json_report, f, indent=2)
json_report = report_to_dict(bblock=bblock, items=all_results, base_url=base_url)
with open(output_dir / '_report.json', 'w') as f:
json.dump(json_report, f, indent=2)

return final_result, test_count
return final_result, test_count, json_report


class RefResolver(jsonschema.validators.RefResolver):
Expand Down
120 changes: 120 additions & 0 deletions ogc/bblocks/validation/report.html.mako
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
<%
from html import escape as e
from datetime import datetime, timezone
from urllib.parse import urlparse, urljoin
from os.path import basename
from pathlib import Path
from ogc.na.util import is_url
import re
import os.path
get_filename = lambda s: basename(urlparse(s).path)
%>
<!doctype html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Building Blocks validation report</title>
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
<style>
.entry-message {
white-space: pre-wrap;
font-size: 90%;
}
</style>
</head>
<body>
<div class="container">
<h1 class="title">Building blocks validation report</h1>
<small class="datetime">Generated at ${e(datetime.now(timezone.utc).astimezone().isoformat())}</small>
% if counts['total'] > 0:
<p class="summary fw-semibold ${'text-success' if counts['passed'] == counts['total'] else 'text-danger'}">
Number of passing building blocks: ${counts['passed']} / ${counts['total']} (${f"{counts['passed'] * 100 / counts['total']:.2f}".rstrip('0').rstrip('.')}%)
</p>
% endif
<div class="text-end small" id="expand-collapse">
<a href="#" class="expand-all">Expand all</a>
<a href="#" class="collapse-all">Collapse all</a>
</div>
<div class="accordion mt-2" id="bblock-reports">
% for i, report in enumerate(reports):
<div class="accordion-item bblock-report" data-bblock-id="${e(report['bblockId'])}" id="bblock-${i}">
<h2 class="accordion-header bblock-title">
<button class="accordion-button" type="button" data-bs-toggle="collapse" data-bs-target="#bblock-collapse-${i}">
<div class="flex-fill">
Validation report for ${e(report['bblockName'])}
<small class="ms-2 bblock-id">${e(report['bblockId'])}</small>
</div>

<span class="badge text-bg-${'success' if report['result'] else 'danger'} me-2">
% if report['result']:
Passed
% else:
Failed
% endif
% if report['counts']['total'] > 0:
(${f"{report['counts']['passed'] * 100 / report['counts']['total']:.2f}".rstrip('0').rstrip('.')}%)
% else:
(100%)
% endif
</span>
</button>
</h2>
<div class="accordion-collapse collapse ${'show' if not report['result'] else ''}" id="bblock-collapse-${i}">
<div class="accordion-body">
% if report['counts']['total'] > 0:
<p class="summary fw-semibold ${'text-success' if report['counts']['passed'] == report['counts']['total'] else 'text-danger'}">
Test passed: ${report['counts']['passed']} / ${report['counts']['total']}
</p>
% endif
% for item in report['items']:
<div class="card mb-2 validation-item ${'require-fail' if item['source']['requireFail'] else ''}">
<div class="card-body">
<div class="card-title">
<a href="${e(item['source']['filename'])}" target="_blank">${e(re.sub(r'.*/', '', item['source']['filename']))}</a>
<span class="badge bg-secondary ${e(item['source']['type'].lower())}">${e(item['source']['type'].replace('_', ' ').capitalize())}</span>
% if item['source']['requireFail']:
<span class="badge text-bg-info">Requires fail</span>
% endif
<div class="float-end">
% if item['result']:
<span class="badge text-bg-success me-2">Passed</span>
% else:
<span class="badge text-bg-danger me-2">Failed</span>
% endif
</div>
</div>
<p class="card-text">
% for subsection_title, section in item['sections'].items():
% for entry in section:
<div class="font-monospace entry-message section-${e(subsection_title.lower())} ${'text-danger' if entry['isError'] else ''}">${e(entry['message'])}</div>
% endfor
% endfor
</p>
</div>
</div>
% endfor
</div>
</div>
</div>
% endfor
</div>
</div>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" integrity="sha384-C6RzsynM9kWDrMNeT87bh95OGNyZPhcTNXj1NW7RuBCsyN/o0jlpcV8Qyq46cDfL" crossorigin="anonymous"></script>
<script type="text/javascript">
window.addEventListener('load', () => {
const accordionEntries = [...document.querySelectorAll('#bblock-reports .accordion-collapse')];
console.log(accordionEntries);
document.querySelector('#expand-collapse').addEventListener('click', ev => {
ev.preventDefault();
if (ev.target.matches('.expand-all')) {
accordionEntries.forEach(e => e.classList.add('show'));
} else if (ev.target.matches('.collapse-all')) {
accordionEntries.forEach(e => e.classList.remove('show'));
}
});
});
</script>
</body>
</html>

0 comments on commit 261e4da

Please sign in to comment.