Migration and doi updates for v2.0

hubmapconsortium · Dec 8, 2023 · 432e517 · 432e517
1 parent fa8df5b
commit 432e517
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 24 deletions.
diff --git a/src/finalizing/templates/doi-xml.njk b/src/finalizing/templates/doi-xml.njk
@@ -10,54 +10,54 @@
     <creatorName>{{ creator.fullName | safe }}</creatorName>
     <givenName>{{ creator.firstName | safe }}</givenName>
     <familyName>{{ creator.lastName | safe }}</familyName>
-    <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}<nameIdentifier>
+    <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}</nameIdentifier>
   </creator>
 {%- endfor %}
 </creators>
 <publisher>{{ publisher }}</publisher>
 <publicationYear>{{ creation_date | year }}</publicationYear>
-<resourceType resourceTypeGeneral="{{ resource_mappings[type] }}">{{ resource_title_mappings[type] }}</resourceType>
+<resourceType resourceTypeGeneral="{{ resource_mappings[resource_type] }}">{{ resource_title_mappings[resource_type] }}</resourceType>
 <subjects>
   <subject>{{ title }}, {{ version }}</subject>
-<subjects>
+</subjects>
 <contributors>
 {%- for creator in project_leads %}
   <contributor contributorType="ProjectLeader">
       <contributorName>{{ creator.fullName | safe }}</contributorName>
       <givenName>{{ creator.firstName | safe }}</givenName>
       <familyName>{{ creator.lastName | safe }}</familyName>
-      <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}<nameIdentifier>
+      <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}</nameIdentifier>
   </contributor>
 {%- endfor -%}
 {%- for creator in reviewers %}
   <contributor contributorType="Other">
     <contributorName>{{ creator.fullName | safe }}</contributorName>
     <givenName>{{ creator.firstName | safe }}</givenName>
     <familyName>{{ creator.lastName | safe }}</familyName>
-    <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}<nameIdentifier>
+    <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}</nameIdentifier>
   </contributor>
 {%- endfor %}
 {%- for creator in externalReviewers %}
   <contributor contributorType="Other">
     <contributorName>{{ creator.fullName | safe }}</contributorName>
     <givenName>{{ creator.firstName | safe }}</givenName>
     <familyName>{{ creator.lastName | safe }}</familyName>
-    <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}<nameIdentifier>
+    <nameIdentifier nameIdentifierScheme="ORCID" schemeURI="https://orcid.org/{{ creator.orcid }}">{{ creator.orcid }}</nameIdentifier>
   </contributor>
 {%- endfor %}
 </contributors>
 <dates>
   <date dateType="Available">{{ creation_date }}</date>
 </dates>
 <descriptions>
-  <description>{{ description }}</description>
-</decsriptions>
+  <description descriptionType="Abstract">{{ description }}</description>
+</descriptions>
 <language>EN</language>
-<alternateIdentfiers>
-  <alternateIdentfier alternateIdentiferType="HuBMAP ID">{{ hubmapId }}</alternateIdentfier>
-  <alternateIdentfier alternateIdentfierType="How to Cite This {{ cite_model_mappings[type] }}">{{ citation }}</alternateIdentfier>
-  <alternateIdentfier alternateIdentfierType="How to Cite {{ cite_overall_model_mappings[type] }} Overall">{{ citationOverall }}</alternateIdentfier>
-</alternateIdentfiers>
+<alternateIdentifiers>
+  <alternateIdentifier alternateIdentifierType="HuBMAP ID">{{ hubmapId }}</alternateIdentifier>
+  <alternateIdentifier alternateIdentifierType="How to Cite This {{ cite_model_mappings[type] }}">{{ citation }}</alternateIdentifier>
+  <alternateIdentifier alternateIdentifierType="How to Cite {{ cite_overall_model_mappings[type] }} Overall">{{ citationOverall }}</alternateIdentifier>
+</alternateIdentifiers>
 <formats>
 	<format>.{{ datatable | fileExtension }}</format>
 </formats>

diff --git a/src/finalizing/utils/generate-doi-xml.js b/src/finalizing/utils/generate-doi-xml.js
@@ -10,12 +10,23 @@ const TYPE_MAPPINGS = {
     omap: 'Organ Mapping Antibody Panels(OMAPs)',
     'ref-organ': '3D Reference Organs',
   },
-  resource_mappings: { 'asct-b': 'Dataset', '2d-ftu': 'Image', omap: 'Dataset', 'ref-organ': 'Model' },
+  resource_mappings: {
+    'asct-b': 'Dataset',
+    '2d-ftu': 'Image',
+    omap: 'Dataset',
+    'ref-organ': 'Model',
+    'asct-b-crosswalk': 'Dataset',
+    'ref-organ-crosswalk': 'Dataset',
+    '2d-ftu-crosswalk': 'Dataset',
+  },
   resource_title_mappings: {
     'asct-b': 'ASCT+B Table',
     '2d-ftu': '2D reference human organ FTU object',
     omap: 'Organ Mapping Antibody Panel',
     'ref-organ': '3D reference human organ model',
+    'asct-b-crosswalk': 'ASCT+B Crosswalk Table',
+    '2d-ftu-crosswalk': 'ASCT+B table to 2D FTU crosswalk Table',
+    'ref-organ-crosswalk': 'ASCT+B table to 3D model crosswalk Table',
   },
   cite_model_mappings: { 'asct-b': 'Data Table', '2d-ftu': '2D Data', omap: 'OMAP Tables', 'ref-organ': '3D Data' },
   cite_overall_model_mappings: {
@@ -24,14 +35,15 @@ const TYPE_MAPPINGS = {
     omap: 'OMAP Tables',
     'ref-organ': '3D Data',
   },
-  extension_fixes: { ai: 'svg', xlsx: 'csv' }
+  extension_fixes: { ai: 'svg', xlsx: 'csv' },
 };
 
 export function renderDoiXml(templateFile, context, metadata) {
   const env = new Environment(undefined, { autoescape: false });
   env.addFilter('fileExtension', (str) => {
     str = Array.isArray(str) ? str[0] : str;
-    const ext = str !== undefined ? str.slice(str.replace('.zip', '').lastIndexOf('.') + 1).replace(')', '') : '';
+    str = str.replace('.zip', '').replace('.7z', '');
+    const ext = str !== undefined ? str.slice(str.lastIndexOf('.') + 1).replace(')', '') : '';
     return TYPE_MAPPINGS.extension_fixes[ext] || ext;
   });
   env.addFilter('mdLinkAsUrlOnly', (str) => str.slice(str.lastIndexOf('(') + 1, str.lastIndexOf(')') - 1));
@@ -40,9 +52,11 @@ export function renderDoiXml(templateFile, context, metadata) {
       ? 'Creative Commons Attribution 4.0 International (CC BY 4.0)'
       : str.slice(str.lastIndexOf('[') + 1, str.lastIndexOf(']')).replace(/\*/g, '')
   );
-  env.addFilter('year', (str) => String(str || new Date().getFullYear()).match(/\d\d\d\d/)[0])
+  env.addFilter('year', (str) => String(str || new Date().getFullYear()).match(/\d\d\d\d/)[0]);
   const template = readFileSync(templateFile).toString();
-  return env.renderString(template, { ...TYPE_MAPPINGS, ...metadata, ...context.selectedDigitalObject });
+  const data = { ...TYPE_MAPPINGS, ...metadata, ...context.selectedDigitalObject };
+  data.resource_type = data.name.includes('crosswalk') ? `${data.type}-crosswalk` : data.type;
+  return env.renderString(template, data);
 }
 
 export function writeDoiXml(context, metadata) {

diff --git a/src/migration/ccf-releases/2d-ftu-lookup.csv b/src/migration/ccf-releases/2d-ftu-lookup.csv
@@ -8,8 +8,8 @@ kidney-outer-medullary-collecting-duct,UBERON:0004204,outer medullary collecting
 kidney-renal-corpuscle,UBERON:0001229,renal corpuscle,#FTURenalCorpuscle,Kidney,UBERON:0002113
 kidney-thick-ascending-loop-of-henle,UBERON:0001291,thick ascending limb of loop of Henle,#FTUThickAscendingLimb
 large-intestine-crypt-lieberkuhn,UBERON:0001984,crypt of Lieberkuhn of large intestine,#FTUCryptOfLieberkuhn,Large Intestine,UBERON:0000059
-liver-liver-lobule,UBERON:0004647,liver lobule,#FTULiverLobule_inset1,Liver,UBERON:0002107
-liver-liver-lobule,UBERON:0004647,liver lobule,#FTULiverLobule_inset2,Liver,UBERON:0002107
+liver-liver-lobule,UBERON:0004647,liver lobule,#FTULiverLobule_Inset1,Liver,UBERON:0002107
+liver-liver-lobule,UBERON:0004647,liver lobule,#FTULiverLobule_Inset2,Liver,UBERON:0002107
 lung-bronchial-submucosal-gland,UBERON_8410043,bronchus submucosal gland,#FTUBronchialSubmucosalGland
 lung-pulmonary-alveolus,UBERON:0002299,alveolus of lung,#FTUAlveoli,Lung,UBERON:0002048
 pancreas-intercalated-duct,UBERON:0014726,intercalated duct of pancreas,#FTUIntercalatedDuct
@@ -18,6 +18,7 @@ pancreas-pancreatic-acinus,UBERON:0001263,pancreatic acinus,#FTUAcinus
 prostate-prostate-glandular-acinus,UBERON:0004179,prostate glandular acinus,#FTUProstateGlandularAcinus,Prostate Gland,UBERON:0002367
 skin-dermal-papilla,UBERON:0001992,papillary layer of dermis,#FTUDermalPapilla
 skin-epidermal-ridge,UBERON:0013487,epidermal ridge of digit,#FTUEpidermalRidge
+small-intestine-villus,UBERON:0001213,intestinal villus,#FTUVillus
 spleen-red-pulp,UBERON:0001250,red pulp of spleen,#FTURedPulp_Inset1
 spleen-red-pulp,UBERON:0001250,red pulp of spleen,#FTURedPulp_Inset2
 spleen-white-pulp,UBERON:0001959,white pulp of spleen,#FTUWhitePulp_Inset1

diff --git a/src/migration/ccf-releases/md-parser.js b/src/migration/ccf-releases/md-parser.js
@@ -72,9 +72,9 @@ export class HraMarkdownParser {
     const funders = this.getMultiValue(funderKey);
     const awards = this.getMultiValue(awardKey);
 
-    return funders.map((funder, index) => ({
-      funder,
-      awardNumber: awards[index],
+    return awards.map((awardNumber, index) => ({
+      funder: funders[index] || (funders?.slice(-1)[0] ?? undefined),
+      awardNumber,
     }));
   }
 

diff --git a/src/migration/ccf-releases/migrate.js b/src/migration/ccf-releases/migrate.js
@@ -66,7 +66,7 @@ export function migrateCcfReleases(context) {
   const srcDir = resolve(context.processorHome, 'src/migration/ccf-releases');
 
   const allMd = sh
-    .ls(resolve(inputDir, 'v1.*/markdown/*/*.md'))
+    .ls(resolve(inputDir, 'v2.*/markdown/*/*.md'))
     .map((s) => s.split('/').slice(-5))
     .map((s) => [s[1], s[3], s[4].replace('.md', '')]);
 

diff --git a/src/migration/ccf-releases/ref-organ-lookup.csv b/src/migration/ccf-releases/ref-organ-lookup.csv
@@ -1,4 +1,5 @@
 glbFile,oldId
+3d-vh-m-allen-brain,#VHMAllenBrain
 Allen_M_Brain,#VHMAllenBrain
 SBU_M_Intestine_Large,#VHMColon
 SBU_Intestine_Large,#VHMColon
@@ -20,6 +21,7 @@ VH_M_Lung,#VHMLung
 NIH_M_Lymph_Node_Left,#VHMLeftLymphNode
 NIH_M_Lymph_Node_Right,#VHMRightLymphNode
 NIH_M_Lymph_Node,#VHMLymphNode
+3d-vh-m-pancreas,#VHMPancreas
 VH_M_Pancreas,#VHMPancreas
 VH_M_Pelvis,#VHMPelvis
 VH_M_Prostate,#VHMProstate
@@ -37,6 +39,7 @@ VH_M_Spinal_Cord,#VHMSpinalCord
 3d-vh-m-palatine-tonsil-r,#VHMRightPalatineTonsil
 3d-vh-m-trachea,#VHMTrachea
 Allen_F_Brain,#VHFAllenBrain
+3d-vh-f-allen-brain,#VHFAllenBrain
 NIH_F_Lymph_Node_Left,#VHFLeftLymphNode
 NIH_F_Lymph_Node_Right,#VHFRightLymphNode
 NIH_F_Lymph_Node,#VHFLymphNode
@@ -59,6 +62,7 @@ VH_F_Liver,#VHFLiver
 VH_F_Lung,#VHFLung
 VH_F_Ovary_L,#VHFLeftOvary
 VH_F_Ovary_R,#VHFRightOvary
+3d-vh-f-pancreas,#VHFPancreas
 VH_F_Pancreas,#VHFPancreas
 VH_F_Pelvis,#VHFPelvis
 3d-vh-f-skin,#VHFSkin