Feature/metadata model (#10)

* Rename metadata schema file * Download RO base * Reorganize slots * Select some of metadata fields * Draft metadata from asct-b object type * Add metadata normalization pipe * Rename functions for clarity * Remove async * Write an empty function * Extract validation function from the normalization pipe * Add metadata folder to generate the schema file * Fix metadata-base schema * Exclude name and type fields from the normalized metadata * Revise main schemas * Replace DO identifier field from id to iri * Refactor normalization process for ref-organ * Remove id field from metadata spec * Add metadata normalization and validation pipes * Code refactor to handle error on async call * Remove unused import * Implement collection metadata * Write comments for clarity * Fix metadata schema * Fix serializing normalized metadata * Report all the missing digital objects before throwing the error * Implement metadata enrichment pipe * Replace id field with iri field * Fix the metadata IRI construction * Add distribution objects * Specify mandatory fields
hubmapconsortium · Aug 10, 2023 · 63115f0 · 63115f0
1 parent eb96241
commit 63115f0
Show file tree

Hide file tree

Showing 22 changed files with 628 additions and 300 deletions.
diff --git a/schemas/generated/docs/asct-b-metadata/about.md b/schemas/generated/docs/asct-b-metadata/about.md
@@ -0,0 +1,28 @@
+
+# Slot: about
+
+
+
+
+URI: [dcat:about](http://www.w3.org/ns/dcat#about)
+
+
+## Domain and Range
+
+None &#8594;  <sub>0..1</sub> [String](types/String.md)
+
+## Parents
+
+
+## Children
+
+
+## Used by
+
+
+## Other properties
+
+|  |  |  |
+| --- | --- | --- |
+| **Mappings:** | | schema:about |
+
diff --git a/schemas/src/digital-objects/asct-b.yaml b/schemas/src/digital-objects/asct-b.yaml
@@ -22,7 +22,7 @@ default_range: string
 
 imports:
   - linkml:types
-  - ../shared/metadata
+  - ../shared/metadata-base
 
 settings:
   uberon: "UBERON"
@@ -135,61 +135,33 @@ classes:
   AsctbMetadata:
     class_uri: dcat:Dataset
     slots:
-      - type
-      - name
-      - version
       - title
       - description
       - creators
-      - project_leads
-      - reviewers
-      - externalReviewers
+      - version
       - creation_date
       - license
       - publisher
-      - funders
-      - hubmapId
-      - doi
-      - citation
-      - citationOverall
-      - datatable
   Container:
     tree_root: true
     attributes:
-      id:
+      iri:
         range: uriorcurie
       metadata:
         range: AsctbMetadata
       data:
         range: AsctbDataset
     annotations:
       owl.template: |-
-        AnnotationAssertion( schema:about {{id}} "{{metadata.type}}" )
-        AnnotationAssertion( schema:name {{id}} "{{metadata.name}}" )
-        AnnotationAssertion( schema:version {{id}} "{{metadata.version}}" )
-        AnnotationAssertion( dct:title {{id}} "{{metadata.title}}" )
-        AnnotationAssertion( dct:description {{id}} "{{metadata.description|e}}" )
+        AnnotationAssertion( dct:title {{iri}} "{{metadata.title}}" )
+        AnnotationAssertion( dct:description {{iri}} "{{metadata.description}}" )
         {% for c in metadata.creators %}
-        AnnotationAssertion( dct:creator {{id}} "{{c.fullName}} ({{c.orcid}})" )
-        {% endfor %}
-        {% for l in metadata.project_leads %}
-        AnnotationAssertion( ccf:project_lead {{id}} "{{l.fullName}} ({{l.orcid}})" )
-        {% endfor %}
-        {% for r in metadata.reviewers %}
-        AnnotationAssertion( schema:reviewedBy {{id}} "{{r.fullName}} ({{r.orcid}})" )
-        {% endfor %}
-        {% for r in metadata.externalReviewers %}
-        AnnotationAssertion( ccf:externallyReviewedBy {{id}} "{{r.fullName}} ({{r.orcid}})" )
-        {% endfor %}
-        AnnotationAssertion( schema:dateCreated {{id}} "{{metadata.creation_date}}" )
-        AnnotationAssertion( dct:license {{id}} "{{metadata.license}}" )
-        AnnotationAssertion( dct:publisher {{id}} "{{metadata.publisher}}" )
-        {% for f in metadata.funders %}
-        AnnotationAssertion( schema:funding {{id}} "Award number {{f.awardNumber}} from {{f.funder}}" )
+        AnnotationAssertion( dct:creator {{iri}} "{{c.fullName}} ({{c.orcid}})" )
         {% endfor %}
-        AnnotationAssertion( ccf:hubmap_id {{id}} "{{metadata.hubmapId}}" )
-        AnnotationAssertion( schema:citation {{id}} "{{metadata.citation}}" )
-        AnnotationAssertion( ccf:citation_overall {{id}} "{{metadata.citationOverall}}" )
+        AnnotationAssertion( schema:version {{iri}} "{{metadata.version}}" )
+        AnnotationAssertion( schema:dateCreated {{iri}} "{{metadata.creation_date}}" )
+        AnnotationAssertion( dct:license {{iri}} "{{metadata.license}}" )
+        AnnotationAssertion( dct:publisher {{iri}} "{{metadata.publisher}}" )
 
 slots:
   id:

diff --git a/schemas/src/digital-objects/collection.yaml b/schemas/src/digital-objects/collection.yaml
@@ -14,34 +14,23 @@ default_range: string
 
 imports:
   - linkml:types
-  - ../shared/metadata
+  - ../shared/metadata-base
 
 classes:
   CollectionMetadata:
     class_uri: dcat:Dataset
     slots:
-      - type
-      - name
-      - version
       - title
       - description
       - creators
-      - project_leads
-      - reviewers
-      - externalReviewers
+      - version
       - creation_date
       - license
       - publisher
-      - funders
-      - hubmapId
-      - doi
-      - citation
-      - citationOverall
-      - datatable
   Container:
     tree_root: true
     attributes:
-      id:
+      iri:
         range: uriorcurie
       metadata:
         range: CollectionMetadata
@@ -51,31 +40,12 @@ classes:
         inlined_as_list: true
     annotations:
       owl.template: |-
-        AnnotationAssertion( schema:about {{id}} "{{metadata.type}}" )
-        AnnotationAssertion( schema:version {{id}} "{{metadata.version}}" )
-        AnnotationAssertion( dct:title {{id}} "{{metadata.title}}" )
-        AnnotationAssertion( dct:description {{id}} "{{metadata.description|e}}" )
+        AnnotationAssertion( dct:title {{iri}} "{{metadata.title}}" )
+        AnnotationAssertion( dct:description {{iri}} "{{metadata.description}}" )
         {% for c in metadata.creators %}
-        AnnotationAssertion( dct:creator {{id}} "{{c.fullName}} ({{c.orcid}})" )
-        {% endfor %}
-        {% for l in metadata.project_leads %}
-        AnnotationAssertion( ccf:project_lead {{id}} "{{l.fullName}} ({{l.orcid}})" )
-        {% endfor %}
-        {% for r in metadata.reviewers %}
-        AnnotationAssertion( schema:reviewedBy {{id}} "{{r.fullName}} ({{r.orcid}})" )
-        {% endfor %}
-        {% for r in metadata.externalReviewers %}
-        AnnotationAssertion( ccf:externallyReviewedBy {{id}} "{{r.fullName}} ({{r.orcid}})" )
-        {% endfor %}
-        AnnotationAssertion( schema:dateCreated {{id}} "{{metadata.creation_date}}" )
-        AnnotationAssertion( dct:license {{id}} "{{metadata.license}}" )
-        AnnotationAssertion( dct:publisher {{id}} "{{metadata.publisher}}" )
-        {% for f in metadata.funders %}
-        AnnotationAssertion( schema:funding {{id}} "Award number {{f.awardNumber}} from {{f.funder}}" )
+        AnnotationAssertion( dct:creator {{iri}} "{{c.fullName}} ({{c.orcid}})" )
         {% endfor %}
-        AnnotationAssertion( ccf:hubmap_id {{id}} "{{metadata.hubmapId}}" )
-        AnnotationAssertion( schema:citation {{id}} "{{metadata.citation}}" )
-        AnnotationAssertion( ccf:citation_overall {{id}} "{{metadata.citationOverall}}" )
-        {% for do in data %}
-        AnnotationAssertion( schema:isBasedOn {{id}} https://purl.humanatlas.io/{{do}} )
-        {% endfor %}
+        AnnotationAssertion( schema:version {{iri}} "{{metadata.version}}" )
+        AnnotationAssertion( schema:dateCreated {{iri}} "{{metadata.creation_date}}" )
+        AnnotationAssertion( dct:license {{iri}} "{{metadata.license}}" )
+        AnnotationAssertion( dct:publisher {{iri}} "{{metadata.publisher}}" )
diff --git a/schemas/src/digital-objects/ref-organ.yaml b/schemas/src/digital-objects/ref-organ.yaml
@@ -14,7 +14,7 @@ default_range: string
 
 imports:
   - linkml:types
-  - ../shared/metadata
+  - ../shared/metadata-base
 
 classes:
   SpatialEntity:
@@ -39,7 +39,6 @@ classes:
       - reference_organ
       - extraction_set
       - rui_rank
-
   SpatialObjectReference:
     class_uri: ccf:SpatialObjectReference
     slots:
@@ -82,76 +81,35 @@ classes:
   RefOrganMetadata:
     class_uri: dcat:Dataset
     slots:
-      - type
-      - name
-      - version
       - title
       - description
       - creators
-      - project_leads
-      - reviewers
-      - externalReviewers
+      - version
       - creation_date
       - license
       - publisher
-      - funders
-      - hubmapId
-      - doi
-      - citation
-      - citationOverall
-      - datatable
   Container:
     tree_root: true
     attributes:
-      id:
+      iri:
         range: uriorcurie
       metadata:
         range: RefOrganMetadata
-
       data:
         multivalued: true
         inlined_as_list: true
         range: SpatialEntity
     annotations:
       owl.template: |-
-        AnnotationAssertion( schema:about {{id}} "{{metadata.type}}" )
-        AnnotationAssertion( schema:version {{id}} "{{metadata.version}}" )
-        AnnotationAssertion( dct:title {{id}} "{{metadata.title}}" )
-        AnnotationAssertion( dct:description {{id}} "{{metadata.description|e}}" )
+        AnnotationAssertion( dct:title {{iri}} "{{metadata.title}}" )
+        AnnotationAssertion( dct:description {{iri}} "{{metadata.description}}" )
         {% for c in metadata.creators %}
-        AnnotationAssertion( dct:creator {{id}} "{{c.fullName}} ({{c.orcid}})" )
-        {% endfor %}
-        {% for l in metadata.project_leads %}
-        AnnotationAssertion( ccf:project_lead {{id}} "{{l.fullName}} ({{l.orcid}})" )
-        {% endfor %}
-        {% for r in metadata.reviewers %}
-        AnnotationAssertion( schema:reviewedBy {{id}} "{{r.fullName}} ({{r.orcid}})" )
+        AnnotationAssertion( dct:creator {{iri}} "{{c.fullName}} ({{c.orcid}})" )
         {% endfor %}
-        {% for r in metadata.externalReviewers %}
-        AnnotationAssertion( ccf:externallyReviewedBy {{id}} "{{r.fullName}} ({{r.orcid}})" )
-        {% endfor %}
-        AnnotationAssertion( schema:dateCreated {{id}} "{{metadata.creation_date}}" )
-        AnnotationAssertion( dct:license {{id}} "{{metadata.license}}" )
-        AnnotationAssertion( dct:publisher {{id}} "{{metadata.publisher}}" )
-        {% for f in metadata.funders %}
-        AnnotationAssertion( schema:funding {{id}} "Award number {{f.awardNumber}} from {{f.funder}}" )
-        {% endfor %}
-        AnnotationAssertion( ccf:hubmap_id {{id}} "{{metadata.hubmapId}}" )
-        AnnotationAssertion( schema:citation {{id}} "{{metadata.citation}}" )
-        AnnotationAssertion( ccf:citation_overall {{id}} "{{metadata.citationOverall}}" )
-  
-      # extraction_sets:
-      #   multivalued: true
-      #   inlined_as_list: true
-      #   range: ExtractionSet
-      # spatial_entities:
-      #   multivalued: true
-      #   inlined_as_list: true
-      #   range: SpatialEntity
-      # rui_placements:
-      #   multivalued: true
-      #   inlined_as_list: true
-      #   range: SpatialPlacement
+        AnnotationAssertion( schema:version {{iri}} "{{metadata.version}}" )
+        AnnotationAssertion( schema:dateCreated {{iri}} "{{metadata.creation_date}}" )
+        AnnotationAssertion( dct:license {{iri}} "{{metadata.license}}" )
+        AnnotationAssertion( dct:publisher {{iri}} "{{metadata.publisher}}" )  
 
 slots:
   id:
@@ -161,7 +119,7 @@ slots:
     slot_uri: rdfs:label
   creator:
     required: true
-    slot_uri: dcterms:creator
+    slot_uri: ccf:creator
   creator_first_name:
     required: true
     slot_uri: ccf:creator_first_name
@@ -173,7 +131,7 @@ slots:
     slot_uri: ccf:creator_orcid
   creation_date:
     required: true
-    slot_uri: dcterms:created
+    slot_uri: ccf:creation_date
     range: date
   x_dimension:
     required: true
@@ -272,7 +230,7 @@ slots:
     slot_uri: ccf:file_format
   placement_date:
     required: true
-    slot_uri: dcterms:created
+    slot_uri: ccf:placement_date
     range: date
   target:
     required: true
@@ -291,12 +249,12 @@ slots:
     inlined: true
   placement:
     required: true
-    slot_uri: ccf:placement_for
+    slot_uri: ccf:placement
     range: SpatialPlacement
     inlined: true
   placements:
     required: false
-    slot_uri: ccf:placement_for
+    slot_uri: ccf:placements
     range: SpatialPlacement
     inlined: true
     multivalued: true

diff --git a/schemas/src/metadata/asct-b-metadata.yaml b/schemas/src/metadata/asct-b-metadata.yaml
@@ -0,0 +1,60 @@
+id: https://purl.humanatlas.io/specs/asct-b-metadata
+name: asct-b-metadata
+prefixes:
+  ccf: http://purl.org/ccf/
+  pav: http://purl.org/pav/
+  dcat: http://www.w3.org/ns/dcat#
+  dct: http://purl.org/dc/terms/
+  foaf: http://xmlns.com/foaf/0.1/
+  schema: http://schema.org/
+  rdfs: http://www.w3.org/2000/01/rdf-schema#
+  xsd: http://www.w3.org/2001/XMLSchema#
+  linkml: https://w3id.org/linkml/
+
+default_prefix: dcat
+default_range: string
+
+imports:
+  - linkml:types
+  - ../shared/metadata-base
+
+classes:
+  Container:
+    tree_root: true
+    class_uri: dcat:Dataset
+    slots:
+      - iri
+      - title
+      - description
+      - creators
+      - project_leads
+      - reviewers
+      - externalReviewers
+      - version
+      - creation_date
+      - license
+      - publisher
+      - funders
+      - hubmapId
+      - doi
+      - citation
+      - citationOverall
+      - datatable
+      - distributions
+    slot_usage:
+      iri:
+        required: true
+      title:
+        required: true
+      description:
+        required: true
+      creators:
+        required: true
+      version:
+        required: true
+      creation_date:
+        required: true
+      license:
+        required: true
+      distributions:
+        required: true