From a73c540274170f3a49e1807e834cdcd8a97e8b5c Mon Sep 17 00:00:00 2001 From: Alejandro Villar Date: Tue, 7 Nov 2023 10:11:51 +0100 Subject: [PATCH] Keep track of source of inherited shacl rules --- ogc/bblocks/generate_docs.py | 11 +++++--- ogc/bblocks/postprocess.py | 14 +++++----- ogc/bblocks/templates/slate/index.html.md | 17 ++++++++---- ogc/bblocks/util.py | 29 ++++++++++++++------ ogc/bblocks/validate.py | 33 ++++++++++++----------- 5 files changed, 67 insertions(+), 37 deletions(-) diff --git a/ogc/bblocks/generate_docs.py b/ogc/bblocks/generate_docs.py index 5c5911a..c62b1e1 100644 --- a/ogc/bblocks/generate_docs.py +++ b/ogc/bblocks/generate_docs.py @@ -52,12 +52,14 @@ def __init__(self, base_url: str | None = None, output_dir: str | Path = 'generateddocs', templates_dir: str | Path = 'templates', - id_prefix: str = ''): + id_prefix: str = '', + bblocks_register: BuildingBlockRegister | None = None): self.base_url = base_url self.output_dir = output_dir if isinstance(output_dir, Path) else Path(output_dir) self.output_dir.joinpath('slate-build').mkdir(parents=True, exist_ok=True) self.templates_dir = templates_dir if isinstance(templates_dir, Path) else Path(templates_dir) self.id_prefix = id_prefix or '' + self.bblocks_register = bblocks_register self.templates = find_templates(self.templates_dir) @@ -112,7 +114,9 @@ def generate_doc(self, bblock: BuildingBlock): root_dir=Path(), base_url=self.base_url, git_repo=git_repo, - git_path=git_path)) + git_path=git_path, + bblocks_register=self.bblocks_register, + )) if template.id and template.mediatype: doc_url = f"{self.base_url}{self.output_dir}/" \ f"{template.dir_name}/{bblock.subdirs}/{template.template_file.name}" @@ -133,9 +137,10 @@ def generate_docs(regs: str | Path | Sequence[str | Path], filter_ids: str | list[str] | None = None, output_dir: str | Path = 'generateddocs', templates_dir: str | Path = 'templates'): + bblocks_register = BuildingBlockRegister(regs) doc_generator = DocGenerator(output_dir, templates_dir) - for bblock in BuildingBlockRegister(regs).bblocks.values(): + for bblock in bblocks_register.bblocks.values(): if not filter_ids or bblock.identifier in filter_ids: doc_generator.generate_doc(bblock) diff --git a/ogc/bblocks/postprocess.py b/ogc/bblocks/postprocess.py index 33b807e..c4b0e25 100644 --- a/ogc/bblocks/postprocess.py +++ b/ogc/bblocks/postprocess.py @@ -51,11 +51,6 @@ def postprocess(registered_items_path: str | Path = 'registereditems', github_base_url += '/' test_outputs_base_url = f"{github_base_url}{os.path.relpath(Path(test_outputs_path).resolve(), cwd)}/" - doc_generator = DocGenerator(base_url=base_url, - output_dir=generated_docs_path, - templates_dir=templates_dir, - id_prefix=id_prefix) - if not isinstance(registered_items_path, Path): registered_items_path = Path(registered_items_path) @@ -68,6 +63,12 @@ def postprocess(registered_items_path: str | Path = 'registereditems', annotated_path=annotated_path, imported_bblocks=imported_bblocks) + doc_generator = DocGenerator(base_url=base_url, + output_dir=generated_docs_path, + templates_dir=templates_dir, + id_prefix=id_prefix, + bblocks_register=bbr) + validation_reports = [] def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: @@ -151,7 +152,8 @@ def do_postprocess(bblock: BuildingBlock, light: bool = False) -> bool: if base_url: if bblock.shaclRules: - bblock.metadata['shaclRules'] = [urljoin(base_url, s) for s in bblock.shaclRules] + bblock.metadata['shaclRules'] = {k: [urljoin(base_url, s) for s in v] + for k, v in bblock.shaclRules.items()} if bblock.transforms: bblock.metadata['transforms'] = [] for transform in bblock.transforms: diff --git a/ogc/bblocks/templates/slate/index.html.md b/ogc/bblocks/templates/slate/index.html.md index d8d3d2f..34cd6fc 100644 --- a/ogc/bblocks/templates/slate/index.html.md +++ b/ogc/bblocks/templates/slate/index.html.md @@ -155,14 +155,21 @@ ${'#'} Validation ${'##'} SHACL Shapes -The following SHACL shapes are used for validating this building block: +The following sets of SHACL shapes are used for validating this building block: - % for rule in bblock.shaclRules: - % if rule.startswith('http://') or rule.startswith('https://'): -* [${rule}](${rule}) + % for shacl_bblock_id, shacl_rules in bblock.shaclRules.items(): + % if bblocks_register.get(shacl_bblock_id): +* ${bblocks_register.get(shacl_bblock_id).get('name')} ${shacl_bblock_id} % else: -* `${rule}` +* `${shacl_bblock_id}` % endif + % for rule in shacl_rules: + % if rule.startswith('http://') or rule.startswith('https://'): + * [${rule}](${rule}) + % else: + * `${rule}` + % endif + % endfor % endfor % endif diff --git a/ogc/bblocks/util.py b/ogc/bblocks/util.py index 9c83773..09835cd 100644 --- a/ogc/bblocks/util.py +++ b/ogc/bblocks/util.py @@ -120,7 +120,16 @@ def __init__(self, identifier: str, metadata_file: Path, default_shacl_rules = fp / 'rules.shacl' if default_shacl_rules.is_file(): shacl_rules.append('rules.shacl') - self.shacl_rules = [r if is_url(r) else fp / r for r in shacl_rules] + self.shacl_rules = set(r if is_url(r) else fp / r for r in shacl_rules) + + def __getattr__(self, item): + return self.metadata.get(item) + + def __getitem__(self, item): + return self.metadata.get(item) + + def get(self, item, default=None): + return self.metadata.get(item, default) def _load_examples(self): examples = None @@ -159,9 +168,6 @@ def description(self): self._lazy_properties['description'] = load_file(desc_file) if desc_file.is_file() else None return self._lazy_properties['description'] - def __getattr__(self, item): - return self.metadata.get(item) - @property def annotated_schema_contents(self): # We try to read it each time until we succeed, since it could @@ -345,15 +351,22 @@ def find_dependencies(self, identifier: str) -> list[dict | BuildingBlock]: return dependencies - def get_inherited_shacl_rules(self, identifier: str) -> set[str | Path]: - rules = set() + def get_inherited_shacl_rules(self, identifier: str) -> dict[str, set[str | Path]]: + rules: dict[str, set[str | Path]] = {} for dep in self.find_dependencies(identifier): if isinstance(dep, BuildingBlock): - rules.update(dep.shacl_rules or ()) + if dep.shacl_rules: + rules[dep.identifier] = dep.shacl_rules else: - rules.update(dep.get('shaclRules', ())) + dep_rules = dep.get('shaclRules') + if dep_rules: + for inh_id, inh_rules in dep_rules.items(): + rules.setdefault(inh_id, set()).update(inh_rules) return rules + def get(self, identifier: str): + return self.bblocks.get(identifier, self.imported_bblocks.get(identifier)) + @dataclasses.dataclass class TransformMetadata: diff --git a/ogc/bblocks/validate.py b/ogc/bblocks/validate.py index 29ce0b5..779acbf 100644 --- a/ogc/bblocks/validate.py +++ b/ogc/bblocks/validate.py @@ -638,24 +638,24 @@ def validate_test_resources(bblock: BuildingBlock, final_result = True test_count = 0 - if not bblock.tests_dir.is_dir() and not bblock.examples: - return final_result, test_count, report_to_dict(bblock, None, base_url) - shacl_graph = Graph() bblock_shacl_closure = Graph() shacl_error = None - shacl_files = [] + all_shacl_files = [] + inherited_shacl_rules = bblocks_register.get_inherited_shacl_rules(bblock.identifier) try: - for shacl_file in bblocks_register.get_inherited_shacl_rules(bblock.identifier): - if isinstance(shacl_file, Path) or (isinstance(shacl_file, str) and not is_url(shacl_file)): - # assume file - shacl_file = bblock.files_path / shacl_file - shacl_files.append(os.path.relpath(shacl_file)) - else: - shacl_files.append(shacl_file) - shacl_graph.parse(shacl_file, format='turtle') - bblock.metadata['shaclRules'] = shacl_files + for shacl_bblock in list(inherited_shacl_rules.keys()): + bblock_shacl_files = [] + for shacl_file in inherited_shacl_rules[shacl_bblock]: + if isinstance(shacl_file, Path) or (isinstance(shacl_file, str) and not is_url(shacl_file)): + # assume file + shacl_file = os.path.realpath(bblock.files_path / shacl_file) + bblock_shacl_files.append(shacl_file) + all_shacl_files.append(shacl_file) + shacl_graph.parse(shacl_file, format='turtle') + inherited_shacl_rules[shacl_bblock] = bblock_shacl_files + bblock.metadata['shaclRules'] = inherited_shacl_rules for sc in bblock.shaclClosures or (): bblock_shacl_closure.parse(bblock.resolve_file(sc), format='turtle') @@ -664,6 +664,9 @@ def validate_test_resources(bblock: BuildingBlock, except Exception as e: shacl_error = str(e) + if not bblock.tests_dir.is_dir() and not bblock.examples: + return final_result, test_count, report_to_dict(bblock, None, base_url) + json_error = None schema_validator = None jsonld_context = None @@ -703,7 +706,7 @@ def validate_test_resources(bblock: BuildingBlock, shacl_graph=shacl_graph, json_error=json_error, shacl_error=shacl_error, - shacl_files=shacl_files, + shacl_files=all_shacl_files, shacl_closure=bblock_shacl_closure) all_results.append(test_result) final_result = not test_result.failed and final_result @@ -770,7 +773,7 @@ def validate_test_resources(bblock: BuildingBlock, json_error=json_error, shacl_error=shacl_error, base_uri=snippet.get('base-uri', example_base_uri), - shacl_files=shacl_files, + shacl_files=all_shacl_files, schema_ref=snippet.get('schema-ref'), shacl_closure_files=snippet_shacl_closure, shacl_closure=bblock_shacl_closure)