From 862febfe87916c9bc3462568150455fafeea452c Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 3 Mar 2021 11:26:42 -0600 Subject: [PATCH 01/25] Simplify MDF Organization model --- connect_aux_data/organizations.json | 665 +++++++++++----------------- schemas/organization.json | 28 +- 2 files changed, 268 insertions(+), 425 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 623721e..1199210 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -1,399 +1,268 @@ [{ - "canonical_name": "Sample Organization", - "aliases": [ - "Test Organization", - "Do Not Use" - ], - "description": "A sample organization for testing, and not for Production use.", - "homepage": "https://materialsdatafacility.org", - "permission_groups": [ - "5fc63928-3752-11e8-9c6f-0e00fd09bf20" - ], - "acl": [ - "5fc63928-3752-11e8-9c6f-0e00fd09bf20" - ], - "rightsList": [{ - "rightsURI": "https://example.com/license", - "rights": "Example License: No legal effect" - }], - "fundingReferences": [{ - "funderName": "Materials Data Facility", - "awardNumber": { - "awardNumber": "42" - }, - "awardTitle": "MDF Exemplary Organization Grant" - }], - "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" - ], - "curation": true, - "project_blocks": [], - "required_fields": [ - "dc.descriptions.description" - ], - "parent_organizations": [], - "services": {} -}, { - "canonical_name": "Advanced Photon Source", - "aliases": [ - "APS" - ], - "description": "The Advanced Photon Source is an Office of Science User Facility operated for the U.S. Department of Energy Office of Science by Argonne National Laboratory", - "homepage": "https://www.aps.anl.gov/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - "Argonne National Laboratory" - ] -}, { - "canonical_name": "AFRL Additive Manufacturing Challenge", - "aliases": [ - "MIDAS" - ], - "description": "Organization for AFRL Additive Manufacturing Challenge datasets", - "permission_groups": [ - "fe6bbb86-74fe-11e8-b558-0a7d99bc78fe" - ], - "acl": [ - "abcb2d16-02da-11e9-87e3-0e8017bdda58" - ], - "dataset_acl":[ - "abcb2d16-02da-11e9-87e3-0e8017bdda58" - ], - "curation": true, - "data_destinations":[ - "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/afrl-midas/submissions" - ] -}, { - "canonical_name": "APS Sector 1", - "aliases": [ - ], - "description": "Sector 1 of the Advanced Photon Source.", - "homepage": "https://www.aps.anl.gov/Users-Information/Help-Reference/Contacts/Sector-Beamline-Locations-Phones", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - "Advanced Photon Source" - ] -}, { - "canonical_name": "APS Sector 2", - "aliases": [ - ], - "description": "Sector 2 of the Advanced Photon Source.", - "homepage": "https://www.aps.anl.gov/Users-Information/Help-Reference/Contacts/Sector-Beamline-Locations-Phones", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - "Advanced Photon Source" - ] -}, { - "canonical_name": "Argonne National Laboratory", - "aliases": [ - "ANL" - ], - "description": "Argonne serves America as a science and energy laboratory distinguished by the breadth of our R&D capabilities in concert with our powerful suite of experimental and computational facilities.", - "homepage": "https://www.anl.gov/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - ] -}, { - "canonical_name": "Center for Hierarchical Materials Design", - "aliases": [ - "CHiMaD" - ], - "description": "Center for Hierarchical Materials Design (CHiMaD) is a NIST-sponsored center of excellence for advanced materials research focusing on developing the next generation of computational tools, databases and experimental techniques in order to enable the accelerated design of novel materials and their integration to industry, one of the primary goals of the U.S. Government's Materials Genome Initiative (MGI).", - "homepage": "http://chimad.northwestern.edu/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - "National Institute of Standards and Technology" - ] -}, { - "canonical_name": "Center for Predictive Integrated Structural Materials Science", - "aliases": [ - "PRISMS", - "Department of Energy Software Innovation Center for Integrated Multi-Scale Modeling of Structural Metals" - ], - "description": "Combining the efforts of experimental and computational researchers, the overarching goal of the PRISMS Center is to establish a unique scientific platform that will enable accelerated predictive materials science for structural metals.", - "homepage": "http://prisms-center.org/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - ] -}, { - "canonical_name": "Center for Predictive Simulation of Functional Materials", - "aliases": [ - "CPSFM" - ], - "description": "The Center for Predictive Simulation of Functional Materials develops, applies, validates, and disseminates parameter-free methods, open source codes, and scientific data to predict and explain the properties of functional materials for energy applications.", - "homepage": "https://cpsfm.ornl.gov", - "permission_groups": [ - "cc192dca-3751-11e8-90c1-0a7c735d220a" - ], - "acl": [ - "public" - ] -}, { - "canonical_name": "High Throughput Experimental Materials Database", - "aliases": [ - "HTEM" - ], - "description": "The HTEM DB contains information about materials obtained from high-throughput experiments at NREL.", - "homepage": "https://htem.nrel.gov/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - "National Renewable Energy Laboratory" - ] -}, { - "canonical_name": "Materials Commons", - "aliases": [ - "MCPub" - ], - "description": "The Materials Commons is a platform for organizing, collaborating, publishing and sharing research data.", - "homepage": "https://materialscommons.org/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - "Center for Predictive Integrated Structural Materials Science" - ] -}, { - "canonical_name": "MDF Open", - "aliases": [ - "Open" - ], - "description": "A template for open and published data.", - "permission_groups": [ - "cc192dca-3751-11e8-90c1-0a7c735d220a" - ], - "acl": [ - "public" - ], - "curation": true, - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" - } - } -}, { - "canonical_name": "NanoMFG", - "description": "The aim of the nanomanufacturing (nanoMFG) node is to develop computational software tools aimed at creating smart, model-driven and experimentally informed nanomanufactured structures and devices.", - "homepage": "https://nanohub.org/groups/nanomfg", - "permission_groups": [ - "ad88f7cb-cf53-11e9-8526-0e161d24c936" - ], - "project_blocks": [ - "nanomfg" - ], - "parent_organizations": [ - "National Science Foundation" - ] -}, { - "canonical_name": "National Institute of Standards and Technology", - "aliases": [ - "NIST" - ], - "description": "The National Institute of Standards and Technology (NIST) was founded in 1901 and is now part of the U.S. Department of Commerce. NIST is one of the nation's oldest physical science laboratories.", - "homepage": "https://www.nist.gov/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - ] -}, { - "canonical_name": "National Renewable Energy Laboratory", - "aliases": [ - "NREL" - ], - "description": "The National Renewable Energy Laboratory is a national laboratory of the U.S. Department of Energy, Office of Energy Efficiency and Renewable Energy, operated by the Alliance for Sustainable Energy, LLC.", - "homepage": "https://www.nrel.gov/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - ] -}, { - "canonical_name": "National Science Foundation", - "aliases": [ - "NSF" - ], - "description": "The National Science Foundation (NSF) is an independent federal agency created by Congress in 1950 \"to promote the progress of science; to advance the national health, prosperity, and welfare; to secure the national defense...\"", - "homepage": "https://nsf.gov/", - "permission_groups": [ - "public" - ] -}, { - "canonical_name": "NIST Materials Data Repository", - "aliases": [ - "NIST MDR", - "MDR" - ], - "description": "The National Institute of Standards and Technology has created a materials science data repository as part of an effort in coordination with the Materials Genome Initiative (MGI) to establish data exchange protocols and mechanisms that will foster data sharing and reuse across a wide community of researchers, with the goal of enhancing the quality of materials data and models.", - "homepage": "https://materialsdata.nist.gov/", - "permission_groups": [ - "public" - ], - "parent_organizations": [ - "National Institute of Standards and Technology" - ] -}, { - "canonical_name": "Polymer Property Predictor and Database", - "aliases": [ - "PPPDB" - ], - "description": "An organization to support the Polymer Property Predictor and Database", - "permission_groups": [ - "5773d4af-d316-11e9-9c88-0ad4acb67ed4" - ], - "acl": [ - "public" - ], - "curation": true, - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/pppdb/" - } - } -}, { - "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", - "aliases": [ - "VERDE", - "VERDE DB", - "VERDE Materials DB" - ], - "description": "An organization to support the VERDE Materials DB", - "permission_groups": [ - "cc35fe9d-d312-11e9-9c88-0ad4acb67ed4" - ], - "acl": [ - "public" - ], - "curation": true, - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/verde/" - } - } -}, -{ - "canonical_name": "XPCS 8-ID", - "description": "XPCS Beamline 8-ID at Argonne National Laboratory's Advanced Photon Source (APS)", - "permission_groups": [ - "13d05c3b-0a3e-11ea-94e3-0a9045086069" - ], - "acl": [ - "13d05c3b-0a3e-11ea-94e3-0a9045086069" - ], - "curation": true, - "parent_organizations": [ - "Advanced Photon Source" - ], - "services": { - "mdf_publish": { - "publication_location": "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/XPCSDATA/MDF/" - } - } -},{ - "canonical_name": "Hersam Group", - "description": "Hersam Group organization", - "permission_groups": [ - "a422c034-13a3-11e6-8367-22000ab80e73" - ], - "acl": [ - "ea14d488-13a3-11e6-81e9-22000aef184d" - ], - "dataset_acl":[ - "public" - ], - "curation": true, - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/hersam_protected/" - } - } -},{ - "canonical_name": "Lauhon Group", - "description": "Lauhon Group organization", - "permission_groups": [ - "8eca1609-0994-11eb-ab07-0a15f6c86f93" - ], - "acl": [ - "b62955ff-0994-11eb-a171-0aba59fa28e5" - ], - "dataset_acl":[ - "public" - ], - "curation": true, - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/lauhon_protected/" - } - } -},{ - "canonical_name": "Foundry", - "description": "Foundry data package organization", - "permission_groups": [ - "42a7a77c-4789-11ea-95b7-0ef992ed7ca1" - ], - "acl": [ - "public" - ], - "dataset_acl":[ - "public" - ], - "curation": true, - "project_blocks":["foundry"], - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" - } - } -}, -{ - "canonical_name": "Electrochemical Energy Lab", - "description": "Electrochemical Energy Lab at MIT", - "permission_groups": [ - "cd106a54-6635-11eb-bfe2-0aa21a0136a3" - ], - "acl": [ - "cd106a54-6635-11eb-bfe2-0aa21a0136a3" - ], - "dataset_acl":[ - "cd106a54-6635-11eb-bfe2-0aa21a0136a3" - ], - "curation": true, - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/EEL_protected/" - } - } -}, -{ - "canonical_name": "Differentiate Catalysis", - "description": "Differentiate Catalysis Team", - "permission_groups": [ - "4421683d-665e-11eb-b92d-0a4debe59093" - ], - "acl": [ - "4421683d-665e-11eb-b92d-0a4debe59093" - ], - "dataset_acl":[ - "4421683d-665e-11eb-b92d-0a4debe59093" - ], - "curation": true, - "services": { - "mdf_publish": { - "publication_location": "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/differentiate_protected/" - } - } -} -] + "canonical_name": "AFRL Additive Manufacturing Challenge", + "aliases": [ + "MIDAS" + ], + "description": "Organization for AFRL Additive Manufacturing Challenge datasets", + "permission_groups": [ + "fe6bbb86-74fe-11e8-b558-0a7d99bc78fe" + ], + "acl": [ + "abcb2d16-02da-11e9-87e3-0e8017bdda58" + ], + "dataset_acl": [ + "abcb2d16-02da-11e9-87e3-0e8017bdda58" + ], + "curation": true, + "data_destinations": [ + "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/afrl-midas/submissions" + ] + }, + { + "canonical_name": "APS Sector 1", + "aliases": [], + "description": "Sector 1 of the Advanced Photon Source.", + "homepage": "https://www.aps.anl.gov/Users-Information/Help-Reference/Contacts/Sector-Beamline-Locations-Phones", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "Center for Hierarchical Materials Design", + "aliases": [ + "CHiMaD" + ], + "description": "Center for Hierarchical Materials Design (CHiMaD) is a NIST-sponsored center of excellence for advanced materials research focusing on developing the next generation of computational tools, databases and experimental techniques in order to enable the accelerated design of novel materials and their integration to industry, one of the primary goals of the U.S. Government's Materials Genome Initiative (MGI).", + "homepage": "http://chimad.northwestern.edu/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "Center for Predictive Integrated Structural Materials Science", + "aliases": [ + "PRISMS", + "Department of Energy Software Innovation Center for Integrated Multi-Scale Modeling of Structural Metals" + ], + "description": "Combining the efforts of experimental and computational researchers, the overarching goal of the PRISMS Center is to establish a unique scientific platform that will enable accelerated predictive materials science for structural metals.", + "homepage": "http://prisms-center.org/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "Center for Predictive Simulation of Functional Materials", + "aliases": [ + "CPSFM" + ], + "description": "The Center for Predictive Simulation of Functional Materials develops, applies, validates, and disseminates parameter-free methods, open source codes, and scientific data to predict and explain the properties of functional materials for energy applications.", + "homepage": "https://cpsfm.ornl.gov", + "permission_groups": [ + "cc192dca-3751-11e8-90c1-0a7c735d220a" + ], + "acl": [ + "public" + ] + }, + { + "canonical_name": "High Throughput Experimental Materials Database", + "aliases": [ + "HTEM" + ], + "description": "The HTEM DB contains information about materials obtained from high-throughput experiments at NREL.", + "homepage": "https://htem.nrel.gov/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "Materials Commons", + "aliases": [ + "MCPub" + ], + "description": "The Materials Commons is a platform for organizing, collaborating, publishing and sharing research data.", + "homepage": "https://materialscommons.org/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "MDF Open", + "aliases": [ + "Open" + ], + "description": "A template for open and published data.", + "permission_groups": [ + "cc192dca-3751-11e8-90c1-0a7c735d220a" + ], + "acl": [ + "public" + ], + "curation": true + }, + { + "canonical_name": "NanoMFG", + "description": "The aim of the nanomanufacturing (nanoMFG) node is to develop computational software tools aimed at creating smart, model-driven and experimentally informed nanomanufactured structures and devices.", + "homepage": "https://nanohub.org/groups/nanomfg", + "permission_groups": [ + "ad88f7cb-cf53-11e9-8526-0e161d24c936" + ], + "project_blocks": [ + "nanomfg" + ] + }, + { + "canonical_name": "National Institute of Standards and Technology", + "aliases": [ + "NIST" + ], + "description": "The National Institute of Standards and Technology (NIST) was founded in 1901 and is now part of the U.S. Department of Commerce. NIST is one of the nation's oldest physical science laboratories.", + "homepage": "https://www.nist.gov/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "National Renewable Energy Laboratory", + "aliases": [ + "NREL" + ], + "description": "The National Renewable Energy Laboratory is a national laboratory of the U.S. Department of Energy, Office of Energy Efficiency and Renewable Energy, operated by the Alliance for Sustainable Energy, LLC.", + "homepage": "https://www.nrel.gov/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "National Science Foundation", + "aliases": [ + "NSF" + ], + "description": "The National Science Foundation (NSF) is an independent federal agency created by Congress in 1950 \"to promote the progress of science; to advance the national health, prosperity, and welfare; to secure the national defense...\"", + "homepage": "https://nsf.gov/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "NIST Materials Data Repository", + "aliases": [ + "NIST MDR", + "MDR" + ], + "description": "The National Institute of Standards and Technology has created a materials science data repository as part of an effort in coordination with the Materials Genome Initiative (MGI) to establish data exchange protocols and mechanisms that will foster data sharing and reuse across a wide community of researchers, with the goal of enhancing the quality of materials data and models.", + "homepage": "https://materialsdata.nist.gov/", + "permission_groups": [ + "public" + ] + }, + { + "canonical_name": "Polymer Property Predictor and Database", + "aliases": [ + "PPPDB" + ], + "description": "An organization to support the Polymer Property Predictor and Database", + "permission_groups": [ + "5773d4af-d316-11e9-9c88-0ad4acb67ed4" + ], + "acl": [ + "public" + ], + "curation": true + }, + { + "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", + "aliases": [ + "VERDE", + "VERDE DB", + "VERDE Materials DB" + ], + "description": "An organization to support the VERDE Materials DB", + "permission_groups": [ + "cc35fe9d-d312-11e9-9c88-0ad4acb67ed4" + ], + "acl": [ + "public" + ], + "curation": true + }, + { + "canonical_name": "XPCS 8-ID", + "description": "XPCS Beamline 8-ID at Argonne National Laboratory's Advanced Photon Source (APS)", + "permission_groups": [ + "13d05c3b-0a3e-11ea-94e3-0a9045086069" + ], + "acl": [ + "13d05c3b-0a3e-11ea-94e3-0a9045086069" + ], + "curation": true + }, + { + "canonical_name": "Hersam Group", + "description": "Hersam Group organization", + "permission_groups": [ + "a422c034-13a3-11e6-8367-22000ab80e73" + ], + "acl": [ + "ea14d488-13a3-11e6-81e9-22000aef184d" + ], + "dataset_acl": [ + "public" + ], + "curation": true + }, { + "canonical_name": "Lauhon Group", + "description": "Lauhon Group organization", + "permission_groups": [ + "8eca1609-0994-11eb-ab07-0a15f6c86f93" + ], + "acl": [ + "b62955ff-0994-11eb-a171-0aba59fa28e5" + ], + "dataset_acl": [ + "public" + ], + "curation": true + }, { + "canonical_name": "Foundry", + "description": "Foundry data package organization", + "permission_groups": [ + "42a7a77c-4789-11ea-95b7-0ef992ed7ca1" + ], + "acl": [ + "public" + ], + "dataset_acl": [ + "public" + ], + "curation": true + }, + { + "canonical_name": "Electrochemical Energy Lab", + "description": "Electrochemical Energy Lab at MIT", + "permission_groups": [ + "cd106a54-6635-11eb-bfe2-0aa21a0136a3" + ], + "acl": [ + "cd106a54-6635-11eb-bfe2-0aa21a0136a3" + ], + "dataset_acl": [ + "cd106a54-6635-11eb-bfe2-0aa21a0136a3" + ], + "curation": true + }, + { + "canonical_name": "Differentiate Catalysis", + "description": "Differentiate Catalysis Team", + "permission_groups": [ + "4421683d-665e-11eb-b92d-0a4debe59093" + ], + "acl": [ + "4421683d-665e-11eb-b92d-0a4debe59093" + ], + "dataset_acl": [ + "4421683d-665e-11eb-b92d-0a4debe59093" + ], + "curation": true + } +] \ No newline at end of file diff --git a/schemas/organization.json b/schemas/organization.json index 6f668a6..8b99e18 100644 --- a/schemas/organization.json +++ b/schemas/organization.json @@ -50,12 +50,6 @@ "description": "One Globus Auth UUID (for one Globus Auth identity or Globus Group) or the special value 'public' to require all dataset entries be public." } }, - "rightsList": { - "$ref": "dc.json#/properties/dc/properties/rightsList" - }, - "fundingReferences": { - "$ref": "dc.json#/properties/dc/properties/fundingReferences" - }, "data_destinations": { "type": "array", "description": "Data locations (which must be Globus endpoints) to send data for this organization.", @@ -75,30 +69,10 @@ "type": "string", "description": "One project block name." } - }, - "required_fields": { - "type": "array", - "description": "Fields or blocks in the dataset metadata that are required for an organizational dataset to be valid, using dot syntax.", - "items": { - "type": "string", - "description": "One required field or block in the dataset metadata." - } - }, - "parent_organizations": { - "type": "array", - "descriptions": "The organization(s) that are direct parents to this one. Grandparent organizations are implied and should not be specified.", - "items": { - "type": "string", - "description": "The canonical_name of one parent organization." - } - }, - "services": { - "$ref": "services.json#/properties/services" } }, - "additionalProperties": false, "required": [ "canonical_name", "permission_groups" ] -} +} \ No newline at end of file From 934d9d94fe210599b9715cccdd26019abc8aa1ff Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 3 Mar 2021 15:42:41 -0600 Subject: [PATCH 02/25] Add data_destination for MDF Open --- connect_aux_data/organizations.json | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 1199210..3b5b1e2 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -88,18 +88,21 @@ }, { "canonical_name": "MDF Open", - "aliases": [ - "Open" - ], - "description": "A template for open and published data.", - "permission_groups": [ - "cc192dca-3751-11e8-90c1-0a7c735d220a" - ], - "acl": [ - "public" - ], - "curation": true - }, + "aliases": [ + "Open" + ], + "description": "A template for open and published data.", + "permission_groups": [ + "cc192dca-3751-11e8-90c1-0a7c735d220a" + ], + "acl": [ + "public" + ], + "curation": true, + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ] + }, { "canonical_name": "NanoMFG", "description": "The aim of the nanomanufacturing (nanoMFG) node is to develop computational software tools aimed at creating smart, model-driven and experimentally informed nanomanufactured structures and devices.", From bea940969fcc79d808b705fffd36dc5bf37b05f3 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 8 Apr 2021 11:40:09 -0500 Subject: [PATCH 03/25] Add action_id to internal status --- schemas/internal_status.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schemas/internal_status.json b/schemas/internal_status.json index f376b2d..4e9cd5f 100644 --- a/schemas/internal_status.json +++ b/schemas/internal_status.json @@ -73,6 +73,11 @@ "type": "string", "description": "The original Connect submission, as a JSON string." }, + "action_id": { + "type": "string", + "description": "The Globus Automate ID of the action processing this record." + }, + "updates": { "type": "array", "description": "Any submitted updates ot the original submission." From 3c5303112bc31468bd02e3a272cc91e2f997fd49 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 1 Jul 2021 16:18:25 -0500 Subject: [PATCH 04/25] Add version to status record --- schemas/internal_status.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/schemas/internal_status.json b/schemas/internal_status.json index 4e9cd5f..6442112 100644 --- a/schemas/internal_status.json +++ b/schemas/internal_status.json @@ -77,7 +77,10 @@ "type": "string", "description": "The Globus Automate ID of the action processing this record." }, - + "version": { + "type": "string", + "description": "SemVer version string for this submission" + }, "updates": { "type": "array", "description": "Any submitted updates ot the original submission." @@ -108,6 +111,7 @@ "submitter", "test", "title", - "user_id" + "user_id", + "version" ] } From 89081ff3c00c2285cd6a25aac070b301d8e1e7ec Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Mon, 23 Aug 2021 12:13:49 -0500 Subject: [PATCH 05/25] Update tests to reflect new internal status schema --- tests/test_files/internal_status/success_maximal.json | 1 + tests/test_files/internal_status/success_minimal.json | 1 + tests/test_schemas.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/tests/test_files/internal_status/success_maximal.json b/tests/test_files/internal_status/success_maximal.json index b953c3b..8b68e0f 100644 --- a/tests/test_files/internal_status/success_maximal.json +++ b/tests/test_files/internal_status/success_maximal.json @@ -18,6 +18,7 @@ "user_email": "test@example.com", "user_id": "abc123", "original_submission": "{...}", + "version": "1.0", "extensions": [ "json" ], diff --git a/tests/test_files/internal_status/success_minimal.json b/tests/test_files/internal_status/success_minimal.json index c7dd914..e37c4df 100644 --- a/tests/test_files/internal_status/success_minimal.json +++ b/tests/test_files/internal_status/success_minimal.json @@ -14,5 +14,6 @@ "submitter": "Tester McPerson", "user_id": "abc123", "test": true, + "version": "1.0", "title": "Thing" } diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 17fb400..bce2284 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -31,6 +31,7 @@ def test_sample_validation(): "internal_status" ] for schema_name in jsonschemas_to_test: + print("Schema ", schema_name) # Get schema with open(os.path.join(SCHEMA_PATH, schema_name+".json")) as f: schema = json.load(f) @@ -40,6 +41,7 @@ def test_sample_validation(): # Run all test cases test_files_dir = os.path.join(TEST_FILES_PATH, schema_name) for test_file in os.listdir(test_files_dir): + print("....Test ", test_file) with open(os.path.join(test_files_dir, test_file)) as t: test_case = json.load(t) # Success cases start with "success", failure cases start with "failure" From 870ee3cd0076b750017edeba2d23d92e46fb6bd2 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Mon, 23 Aug 2021 12:17:14 -0500 Subject: [PATCH 06/25] Update data destinations for organizations --- connect_aux_data/organizations.json | 59 +++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 3b5b1e2..1efd17e 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -171,7 +171,10 @@ "acl": [ "public" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/pppdb/" + ] }, { "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", @@ -187,7 +190,10 @@ "acl": [ "public" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/verde/" + ] }, { "canonical_name": "XPCS 8-ID", @@ -198,7 +204,10 @@ "acl": [ "13d05c3b-0a3e-11ea-94e3-0a9045086069" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/XPCSDATA/MDF/" + ] }, { "canonical_name": "Hersam Group", @@ -212,7 +221,11 @@ "dataset_acl": [ "public" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/hersam_protected/" + ] + }, { "canonical_name": "Lauhon Group", "description": "Lauhon Group organization", @@ -225,7 +238,10 @@ "dataset_acl": [ "public" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/lauhon_protected/" + ] }, { "canonical_name": "Foundry", "description": "Foundry data package organization", @@ -238,7 +254,28 @@ "dataset_acl": [ "public" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" + ] + }, + { + "canonical_name": "Foundry-dev", + "description": "Foundry dataset organization", + "permission_groups": [ + "cc192dca-3751-11e8-90c1-0a7c735d220a" + ], + "acl": [ + "public" + ], + "dataset_acl":[ + "public" + ], + "curation": true, + "project_blocks":["foundry-dev"], + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" + ] }, { "canonical_name": "Electrochemical Energy Lab", @@ -252,7 +289,10 @@ "dataset_acl": [ "cd106a54-6635-11eb-bfe2-0aa21a0136a3" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/EEL_protected/" + ] }, { "canonical_name": "Differentiate Catalysis", @@ -266,6 +306,9 @@ "dataset_acl": [ "4421683d-665e-11eb-b92d-0a4debe59093" ], - "curation": true + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/differentiate_protected/" + ] } ] \ No newline at end of file From 3d5615b67d7493933931e3d6ed4754899e312cee Mon Sep 17 00:00:00 2001 From: eschondorf <37815736+eschondorf@users.noreply.github.com> Date: Wed, 27 Oct 2021 11:31:28 -0500 Subject: [PATCH 07/25] Meta only (#53) * added additional paremter to connect_submission file * adding update_meta_only property to interanal_status --- schemas/connect_submission.json | 6 +++--- schemas/internal_status.json | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/schemas/connect_submission.json b/schemas/connect_submission.json index ad337ef..5f4e24b 100644 --- a/schemas/connect_submission.json +++ b/schemas/connect_submission.json @@ -248,9 +248,9 @@ "type": "boolean", "description": "Must be true for dataset updates, and false for new datasets. Serves as a check against accidental dataset overwriting." }, - "incremental_update": { - "type": "string", - "description": "Set to a source_id previously submitted to perform an incremental update of that source_id, changing only the other metadata provided in this submission." + "update_meta_only": { + "type": "boolean", + "description": "When true performs an update on the metadata only with no file transfer." }, "curation": { "type": "boolean", diff --git a/schemas/internal_status.json b/schemas/internal_status.json index 6442112..b161c35 100644 --- a/schemas/internal_status.json +++ b/schemas/internal_status.json @@ -96,6 +96,10 @@ "hibernating": { "type": "boolean", "description": "True for submission processes that appear dead, but are still active. This flag will prevent a submission from being cancelled when its process dies." + }, + "update_meta_only": { + "type": "boolean", + "description": "True when update without file transfer" } }, "additionalProperties": false, From 2d1dc1bfe85e8eb00b179d07a3bcaaa2e36e03de Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 17 Nov 2021 10:36:51 -0600 Subject: [PATCH 08/25] Remove obsolete organizations --- connect_aux_data/organizations.json | 64 ----------------------------- 1 file changed, 64 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 1efd17e..076fd2f 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -27,17 +27,6 @@ "public" ] }, - { - "canonical_name": "Center for Hierarchical Materials Design", - "aliases": [ - "CHiMaD" - ], - "description": "Center for Hierarchical Materials Design (CHiMaD) is a NIST-sponsored center of excellence for advanced materials research focusing on developing the next generation of computational tools, databases and experimental techniques in order to enable the accelerated design of novel materials and their integration to industry, one of the primary goals of the U.S. Government's Materials Genome Initiative (MGI).", - "homepage": "http://chimad.northwestern.edu/", - "permission_groups": [ - "public" - ] - }, { "canonical_name": "Center for Predictive Integrated Structural Materials Science", "aliases": [ @@ -114,28 +103,6 @@ "nanomfg" ] }, - { - "canonical_name": "National Institute of Standards and Technology", - "aliases": [ - "NIST" - ], - "description": "The National Institute of Standards and Technology (NIST) was founded in 1901 and is now part of the U.S. Department of Commerce. NIST is one of the nation's oldest physical science laboratories.", - "homepage": "https://www.nist.gov/", - "permission_groups": [ - "public" - ] - }, - { - "canonical_name": "National Renewable Energy Laboratory", - "aliases": [ - "NREL" - ], - "description": "The National Renewable Energy Laboratory is a national laboratory of the U.S. Department of Energy, Office of Energy Efficiency and Renewable Energy, operated by the Alliance for Sustainable Energy, LLC.", - "homepage": "https://www.nrel.gov/", - "permission_groups": [ - "public" - ] - }, { "canonical_name": "National Science Foundation", "aliases": [ @@ -159,23 +126,6 @@ "public" ] }, - { - "canonical_name": "Polymer Property Predictor and Database", - "aliases": [ - "PPPDB" - ], - "description": "An organization to support the Polymer Property Predictor and Database", - "permission_groups": [ - "5773d4af-d316-11e9-9c88-0ad4acb67ed4" - ], - "acl": [ - "public" - ], - "curation": true, - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/pppdb/" - ] - }, { "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", "aliases": [ @@ -195,20 +145,6 @@ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/verde/" ] }, - { - "canonical_name": "XPCS 8-ID", - "description": "XPCS Beamline 8-ID at Argonne National Laboratory's Advanced Photon Source (APS)", - "permission_groups": [ - "13d05c3b-0a3e-11ea-94e3-0a9045086069" - ], - "acl": [ - "13d05c3b-0a3e-11ea-94e3-0a9045086069" - ], - "curation": true, - "data_destinations": [ - "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/XPCSDATA/MDF/" - ] - }, { "canonical_name": "Hersam Group", "description": "Hersam Group organization", From bc2d7685f2c1e3b1ddbef6945ec508df6a7022d7 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 17 Nov 2021 10:45:28 -0600 Subject: [PATCH 09/25] Add CI Job to validate schemas --- .github/workflows/validate_schemas.yaml | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/validate_schemas.yaml diff --git a/.github/workflows/validate_schemas.yaml b/.github/workflows/validate_schemas.yaml new file mode 100644 index 0000000..d4c59ad --- /dev/null +++ b/.github/workflows/validate_schemas.yaml @@ -0,0 +1,29 @@ +name: CI + +on: + push: + branches: + - "*" + tags: + - "*" + pull_request: + +jobs: + validate_schemas: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r tests/test_requirements.txt + pip list + + - name: Test with pytest + run: | + python -m pytest tests From 4cab3692891ba724fa809b4a2d0fbd9d31184b3c Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 17 Nov 2021 12:17:31 -0600 Subject: [PATCH 10/25] Update to new spec for organizations --- connect_aux_data/organizations.json | 509 ++++++++++++++-------------- schemas/organization.json | 22 +- 2 files changed, 270 insertions(+), 261 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 076fd2f..bbd5162 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -1,250 +1,261 @@ -[{ - "canonical_name": "AFRL Additive Manufacturing Challenge", - "aliases": [ - "MIDAS" - ], - "description": "Organization for AFRL Additive Manufacturing Challenge datasets", - "permission_groups": [ - "fe6bbb86-74fe-11e8-b558-0a7d99bc78fe" - ], - "acl": [ - "abcb2d16-02da-11e9-87e3-0e8017bdda58" - ], - "dataset_acl": [ - "abcb2d16-02da-11e9-87e3-0e8017bdda58" - ], - "curation": true, - "data_destinations": [ - "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/afrl-midas/submissions" - ] - }, - { - "canonical_name": "APS Sector 1", - "aliases": [], - "description": "Sector 1 of the Advanced Photon Source.", - "homepage": "https://www.aps.anl.gov/Users-Information/Help-Reference/Contacts/Sector-Beamline-Locations-Phones", - "permission_groups": [ - "public" - ] - }, - { - "canonical_name": "Center for Predictive Integrated Structural Materials Science", - "aliases": [ - "PRISMS", - "Department of Energy Software Innovation Center for Integrated Multi-Scale Modeling of Structural Metals" - ], - "description": "Combining the efforts of experimental and computational researchers, the overarching goal of the PRISMS Center is to establish a unique scientific platform that will enable accelerated predictive materials science for structural metals.", - "homepage": "http://prisms-center.org/", - "permission_groups": [ - "public" - ] - }, - { - "canonical_name": "Center for Predictive Simulation of Functional Materials", - "aliases": [ - "CPSFM" - ], - "description": "The Center for Predictive Simulation of Functional Materials develops, applies, validates, and disseminates parameter-free methods, open source codes, and scientific data to predict and explain the properties of functional materials for energy applications.", - "homepage": "https://cpsfm.ornl.gov", - "permission_groups": [ - "cc192dca-3751-11e8-90c1-0a7c735d220a" - ], - "acl": [ - "public" - ] - }, - { - "canonical_name": "High Throughput Experimental Materials Database", - "aliases": [ - "HTEM" - ], - "description": "The HTEM DB contains information about materials obtained from high-throughput experiments at NREL.", - "homepage": "https://htem.nrel.gov/", - "permission_groups": [ - "public" - ] - }, - { - "canonical_name": "Materials Commons", - "aliases": [ - "MCPub" - ], - "description": "The Materials Commons is a platform for organizing, collaborating, publishing and sharing research data.", - "homepage": "https://materialscommons.org/", - "permission_groups": [ - "public" - ] - }, - { - "canonical_name": "MDF Open", - "aliases": [ - "Open" - ], - "description": "A template for open and published data.", - "permission_groups": [ - "cc192dca-3751-11e8-90c1-0a7c735d220a" - ], - "acl": [ - "public" - ], - "curation": true, - "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" - ] - }, - { - "canonical_name": "NanoMFG", - "description": "The aim of the nanomanufacturing (nanoMFG) node is to develop computational software tools aimed at creating smart, model-driven and experimentally informed nanomanufactured structures and devices.", - "homepage": "https://nanohub.org/groups/nanomfg", - "permission_groups": [ - "ad88f7cb-cf53-11e9-8526-0e161d24c936" - ], - "project_blocks": [ - "nanomfg" - ] - }, - { - "canonical_name": "National Science Foundation", - "aliases": [ - "NSF" - ], - "description": "The National Science Foundation (NSF) is an independent federal agency created by Congress in 1950 \"to promote the progress of science; to advance the national health, prosperity, and welfare; to secure the national defense...\"", - "homepage": "https://nsf.gov/", - "permission_groups": [ - "public" - ] - }, - { - "canonical_name": "NIST Materials Data Repository", - "aliases": [ - "NIST MDR", - "MDR" - ], - "description": "The National Institute of Standards and Technology has created a materials science data repository as part of an effort in coordination with the Materials Genome Initiative (MGI) to establish data exchange protocols and mechanisms that will foster data sharing and reuse across a wide community of researchers, with the goal of enhancing the quality of materials data and models.", - "homepage": "https://materialsdata.nist.gov/", - "permission_groups": [ - "public" - ] - }, - { - "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", - "aliases": [ - "VERDE", - "VERDE DB", - "VERDE Materials DB" - ], - "description": "An organization to support the VERDE Materials DB", - "permission_groups": [ - "cc35fe9d-d312-11e9-9c88-0ad4acb67ed4" - ], - "acl": [ - "public" - ], - "curation": true, - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/verde/" - ] - }, - { - "canonical_name": "Hersam Group", - "description": "Hersam Group organization", - "permission_groups": [ - "a422c034-13a3-11e6-8367-22000ab80e73" - ], - "acl": [ - "ea14d488-13a3-11e6-81e9-22000aef184d" - ], - "dataset_acl": [ - "public" - ], - "curation": true, - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/hersam_protected/" - ] - - }, { - "canonical_name": "Lauhon Group", - "description": "Lauhon Group organization", - "permission_groups": [ - "8eca1609-0994-11eb-ab07-0a15f6c86f93" - ], - "acl": [ - "b62955ff-0994-11eb-a171-0aba59fa28e5" - ], - "dataset_acl": [ - "public" - ], - "curation": true, - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/lauhon_protected/" - ] - }, { - "canonical_name": "Foundry", - "description": "Foundry data package organization", - "permission_groups": [ - "42a7a77c-4789-11ea-95b7-0ef992ed7ca1" - ], - "acl": [ - "public" - ], - "dataset_acl": [ - "public" - ], - "curation": true, - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" - ] - }, - { - "canonical_name": "Foundry-dev", - "description": "Foundry dataset organization", - "permission_groups": [ - "cc192dca-3751-11e8-90c1-0a7c735d220a" - ], - "acl": [ - "public" - ], - "dataset_acl":[ - "public" - ], - "curation": true, - "project_blocks":["foundry-dev"], - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" - ] - }, - { - "canonical_name": "Electrochemical Energy Lab", - "description": "Electrochemical Energy Lab at MIT", - "permission_groups": [ - "cd106a54-6635-11eb-bfe2-0aa21a0136a3" - ], - "acl": [ - "cd106a54-6635-11eb-bfe2-0aa21a0136a3" - ], - "dataset_acl": [ - "cd106a54-6635-11eb-bfe2-0aa21a0136a3" - ], - "curation": true, - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/EEL_protected/" - ] - }, - { - "canonical_name": "Differentiate Catalysis", - "description": "Differentiate Catalysis Team", - "permission_groups": [ - "4421683d-665e-11eb-b92d-0a4debe59093" - ], - "acl": [ - "4421683d-665e-11eb-b92d-0a4debe59093" - ], - "dataset_acl": [ - "4421683d-665e-11eb-b92d-0a4debe59093" - ], - "curation": true, - "data_destinations": [ - "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/differentiate_protected/" - ] - } +[ + { + "canonical_name": "AFRL Additive Manufacturing Challenge", + "aliases": [ + "MIDAS" + ], + "description": "Organization for AFRL Additive Manufacturing Challenge datasets", + "permission_groups": [ + "fe6bbb86-74fe-11e8-b558-0a7d99bc78fe" + ], + "visible_to": [ + "public" + ], + "curation": true, + "data_destinations": [ + "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/afrl-midas/submissions" + ] + }, + { + "canonical_name": "APS Sector 1", + "aliases": [], + "description": "Sector 1 of the Advanced Photon Source.", + "homepage": "https://www.aps.anl.gov/Users-Information/Help-Reference/Contacts/Sector-Beamline-Locations-Phones", + "permission_groups": [ + "public" + ], + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ], + "visible_to": [ + "public" + ] + }, + { + "canonical_name": "Center for Predictive Integrated Structural Materials Science", + "aliases": [ + "PRISMS", + "Department of Energy Software Innovation Center for Integrated Multi-Scale Modeling of Structural Metals" + ], + "description": "Combining the efforts of experimental and computational researchers, the overarching goal of the PRISMS Center is to establish a unique scientific platform that will enable accelerated predictive materials science for structural metals.", + "homepage": "http://prisms-center.org/", + "permission_groups": [ + "public" + ], + "visible_to": [ + "public" + ], + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ] + }, + { + "canonical_name": "Center for Predictive Simulation of Functional Materials", + "aliases": [ + "CPSFM" + ], + "description": "The Center for Predictive Simulation of Functional Materials develops, applies, validates, and disseminates parameter-free methods, open source codes, and scientific data to predict and explain the properties of functional materials for energy applications.", + "homepage": "https://cpsfm.ornl.gov", + "permission_groups": [ + "cc192dca-3751-11e8-90c1-0a7c735d220a" + ], + "visible_to": [ + "public" + ], + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ] + }, + { + "canonical_name": "High Throughput Experimental Materials Database", + "aliases": [ + "HTEM" + ], + "description": "The HTEM DB contains information about materials obtained from high-throughput experiments at NREL.", + "homepage": "https://htem.nrel.gov/", + "permission_groups": [ + "public" + ], + "visible_to": [ + "public" + ], + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ] + }, + { + "canonical_name": "Materials Commons", + "aliases": [ + "MCPub" + ], + "description": "The Materials Commons is a platform for organizing, collaborating, publishing and sharing research data.", + "homepage": "https://materialscommons.org/", + "permission_groups": [ + "public" + ], + "visible_to": [ + "public" + ], + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ] + }, + { + "canonical_name": "MDF Open", + "aliases": [ + "Open" + ], + "description": "A template for open and published data.", + "permission_groups": [ + "cc192dca-3751-11e8-90c1-0a7c735d220a" + ], + "visible_to": [ + "public" + ], + "curation": true, + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ] + }, + { + "canonical_name": "NanoMFG", + "description": "The aim of the nanomanufacturing (nanoMFG) node is to develop computational software tools aimed at creating smart, model-driven and experimentally informed nanomanufactured structures and devices.", + "homepage": "https://nanohub.org/groups/nanomfg", + "permission_groups": [ + "ad88f7cb-cf53-11e9-8526-0e161d24c936" + ], + "visible_to": [ + "public" + ], + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ], + "project_blocks": [ + "nanomfg" + ] + }, + { + "canonical_name": "NIST Materials Data Repository", + "aliases": [ + "NIST MDR", + "MDR" + ], + "description": "The National Institute of Standards and Technology has created a materials science data repository as part of an effort in coordination with the Materials Genome Initiative (MGI) to establish data exchange protocols and mechanisms that will foster data sharing and reuse across a wide community of researchers, with the goal of enhancing the quality of materials data and models.", + "homepage": "https://materialsdata.nist.gov/", + "permission_groups": [ + "public" + ], + "visible_to": [ + "public" + ], + "data_destinations": [ + "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + ] + }, + { + "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", + "aliases": [ + "VERDE", + "VERDE DB", + "VERDE Materials DB" + ], + "description": "An organization to support the VERDE Materials DB", + "permission_groups": [ + "cc35fe9d-d312-11e9-9c88-0ad4acb67ed4" + ], + "visible_to": [ + "public" + ], + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/verde/" + ] + }, + { + "canonical_name": "Hersam Group", + "description": "Hersam Group organization", + "permission_groups": [ + "a422c034-13a3-11e6-8367-22000ab80e73" + ], + "visible_to": [ + "public" + ], + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/hersam_protected/" + ] + }, + { + "canonical_name": "Lauhon Group", + "description": "Lauhon Group organization", + "permission_groups": [ + "8eca1609-0994-11eb-ab07-0a15f6c86f93" + ], + "visible_to": [ + "public" + ], + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/lauhon_protected/" + ] + }, + { + "canonical_name": "Foundry", + "description": "Foundry data package organization", + "permission_groups": [ + "42a7a77c-4789-11ea-95b7-0ef992ed7ca1" + ], + "visible_to": [ + "public" + ], + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" + ] + }, + { + "canonical_name": "Foundry-dev", + "description": "Foundry dataset organization", + "permission_groups": [ + "cc192dca-3751-11e8-90c1-0a7c735d220a" + ], + "visible_to": [ + "public" + ], + "curation": true, + "project_blocks": [ + "foundry-dev" + ], + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" + ] + }, + { + "canonical_name": "Electrochemical Energy Lab", + "description": "Electrochemical Energy Lab at MIT", + "permission_groups": [ + "cd106a54-6635-11eb-bfe2-0aa21a0136a3" + ], + "visible_to": [ + "urn:globus:groups:id:cd106a54-6635-11eb-bfe2-0aa21a0136a3" + ], + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/EEL_protected/" + ] + }, + { + "canonical_name": "Differentiate Catalysis", + "description": "Differentiate Catalysis Team", + "permission_groups": [ + "4421683d-665e-11eb-b92d-0a4debe59093" + ], + "visible_to": [ + "urn:globus:groups:id:4421683d-665e-11eb-b92d-0a4debe59093" + ], + "curation": true, + "data_destinations": [ + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/differentiate_protected/" + ] + } ] \ No newline at end of file diff --git a/schemas/organization.json b/schemas/organization.json index 8b99e18..7d97d53 100644 --- a/schemas/organization.json +++ b/schemas/organization.json @@ -34,20 +34,12 @@ }, "minItems": 1 }, - "acl": { + "visible_to": { "type": "array", - "description": "The minimum Access Control List for full access to datasets in this organization.", + "description": "This is a list of security principals allowed to read the metadata.", "items": { "type": "string", - "description": "One Globus Auth UUID or the special value 'public' to require all datasets be fully public." - } - }, - "dataset_acl": { - "type": "array", - "description": "The minimum Access Control List for access to dataset entries in this organization. This does not grant permission to access records or files from datasets, only permission to see the dataset entry itself.", - "items": { - "type": "string", - "description": "One Globus Auth UUID (for one Globus Auth identity or Globus Group) or the special value 'public' to require all dataset entries be public." + "description": "Each string will be in the form of a Principal URN, or the special string \"public\"" } }, "data_destinations": { @@ -62,6 +54,10 @@ "type": "boolean", "description": "Whether or not curation is required for all datasets in this organization." }, + "mint_doi": { + "type": "boolean", + "description": "Whether datasets submitted to this organization mint a Digital Object Identifier" + }, "project_blocks": { "type": "array", "description": "The special project blocks associated with this organization's metadata.", @@ -73,6 +69,8 @@ }, "required": [ "canonical_name", - "permission_groups" + "permission_groups", + "visible_to", + "data_destinations" ] } \ No newline at end of file From 89ddcbbfca7181a90cd30f4762cb5450dd1574d2 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 17 Nov 2021 16:41:27 -0600 Subject: [PATCH 11/25] Remove more unused and default data destination --- connect_aux_data/organizations.json | 77 ++--------------------------- 1 file changed, 4 insertions(+), 73 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index bbd5162..86786c0 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -25,81 +25,12 @@ "public" ], "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" ], "visible_to": [ "public" ] }, - { - "canonical_name": "Center for Predictive Integrated Structural Materials Science", - "aliases": [ - "PRISMS", - "Department of Energy Software Innovation Center for Integrated Multi-Scale Modeling of Structural Metals" - ], - "description": "Combining the efforts of experimental and computational researchers, the overarching goal of the PRISMS Center is to establish a unique scientific platform that will enable accelerated predictive materials science for structural metals.", - "homepage": "http://prisms-center.org/", - "permission_groups": [ - "public" - ], - "visible_to": [ - "public" - ], - "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" - ] - }, - { - "canonical_name": "Center for Predictive Simulation of Functional Materials", - "aliases": [ - "CPSFM" - ], - "description": "The Center for Predictive Simulation of Functional Materials develops, applies, validates, and disseminates parameter-free methods, open source codes, and scientific data to predict and explain the properties of functional materials for energy applications.", - "homepage": "https://cpsfm.ornl.gov", - "permission_groups": [ - "cc192dca-3751-11e8-90c1-0a7c735d220a" - ], - "visible_to": [ - "public" - ], - "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" - ] - }, - { - "canonical_name": "High Throughput Experimental Materials Database", - "aliases": [ - "HTEM" - ], - "description": "The HTEM DB contains information about materials obtained from high-throughput experiments at NREL.", - "homepage": "https://htem.nrel.gov/", - "permission_groups": [ - "public" - ], - "visible_to": [ - "public" - ], - "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" - ] - }, - { - "canonical_name": "Materials Commons", - "aliases": [ - "MCPub" - ], - "description": "The Materials Commons is a platform for organizing, collaborating, publishing and sharing research data.", - "homepage": "https://materialscommons.org/", - "permission_groups": [ - "public" - ], - "visible_to": [ - "public" - ], - "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" - ] - }, { "canonical_name": "MDF Open", "aliases": [ @@ -114,7 +45,7 @@ ], "curation": true, "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" ] }, { @@ -128,7 +59,7 @@ "public" ], "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" ], "project_blocks": [ "nanomfg" @@ -149,7 +80,7 @@ "public" ], "data_destinations": [ - "globus://e38ee745-6d04-11e5-ba46-22000b92c6ec/MDF/mdf_connect/test_files/deleteme_contents/" + "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" ] }, { From 7a2e244837efd517114953c9aa7ae289897b615f Mon Sep 17 00:00:00 2001 From: eschondorf Date: Mon, 29 Nov 2021 16:27:36 -0500 Subject: [PATCH 12/25] making change to metadata --- schemas/connect_submission.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/connect_submission.json b/schemas/connect_submission.json index 5f4e24b..371ac76 100644 --- a/schemas/connect_submission.json +++ b/schemas/connect_submission.json @@ -248,7 +248,7 @@ "type": "boolean", "description": "Must be true for dataset updates, and false for new datasets. Serves as a check against accidental dataset overwriting." }, - "update_meta_only": { + "update_metadata_only": { "type": "boolean", "description": "When true performs an update on the metadata only with no file transfer." }, From 14fe1f17f1f16a5422293ee61bba205eade2d0ae Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 2 Dec 2021 14:51:29 -0600 Subject: [PATCH 13/25] Update update_metadata_only in internal_status --- schemas/internal_status.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/internal_status.json b/schemas/internal_status.json index b161c35..e4ac7b8 100644 --- a/schemas/internal_status.json +++ b/schemas/internal_status.json @@ -97,7 +97,7 @@ "type": "boolean", "description": "True for submission processes that appear dead, but are still active. This flag will prevent a submission from being cancelled when its process dies." }, - "update_meta_only": { + "update_metadata_only": { "type": "boolean", "description": "True when update without file transfer" } From 879a05450d013004cdee627db3ee6813d853bc9f Mon Sep 17 00:00:00 2001 From: eschondorf Date: Mon, 13 Dec 2021 17:50:21 -0600 Subject: [PATCH 14/25] adding mint_doi flag --- schemas/connect_submission.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/connect_submission.json b/schemas/connect_submission.json index 371ac76..9e36b92 100644 --- a/schemas/connect_submission.json +++ b/schemas/connect_submission.json @@ -252,6 +252,10 @@ "type": "boolean", "description": "When true performs an update on the metadata only with no file transfer." }, + "mint_doi": { + "type": "boolean", + "description": "When true mints doi for file." + }, "curation": { "type": "boolean", "description": "When true, trigger curation for this submission. This is normally set by an organization." From ca1445729b1a597e516aac42d3205144a251829f Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 16 Dec 2021 16:10:50 -0600 Subject: [PATCH 15/25] Add mint_doi property to selected organizations --- connect_aux_data/organizations.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 86786c0..03c8794 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -44,6 +44,7 @@ "public" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" ] @@ -112,6 +113,7 @@ "public" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/hersam_protected/" ] From b4cad16599499f43a153dae64ed671164360f449 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 26 Jan 2022 11:49:58 -0600 Subject: [PATCH 16/25] Merge in master changes to projects --- schemas/projects.json | 127 +++++++++++++++++++++++++++++------------- 1 file changed, 88 insertions(+), 39 deletions(-) diff --git a/schemas/projects.json b/schemas/projects.json index 621bce6..1ba897c 100644 --- a/schemas/projects.json +++ b/schemas/projects.json @@ -110,61 +110,110 @@ }, "foundry": { "type": "object", - "description": "Project block for Foundry data packages.", + "description": "Project block for Foundry datasets.", + "additionalProperties": false, "properties": { - "inputs":{ - "type":"array", - "items": { - "type": "string" - } + "short_name": { + "type": "string" }, - "input_descriptions":{ - "type":"array", - "items": { - "type": "string" - } + "data_type": { + "type": "string", + "description": "The kind of data in the dataset, e.g. tabular, json, hdf5" }, - "input_units":{ - "type":"array", - "items": { + "task_type": { + "type": "array", + "description": "The type of task. e.g., supervised, unsupervised, generative.", + "items": { "type": "string" } }, - "outputs":{ - "type":"array", - "items": { + "domain": { + "type": "array", + "description": "The domain of applicability. e.g., materials science, chemistry, machine vision", + "items": { "type": "string" } }, - "output_descriptions":{ - "type":"array", - "items": { - "type": "string" - } + "n_items": { + "description": "The number of total items in the dataset including all splits.", + "type": "number" }, - "output_units":{ - "type":"array", - "items": { - "type": "string" + "splits": { + "type": "array", + "description": "Define all partitions of the dataset (train, test, validation, etc.)", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "description": "The kind of partition of the dataset (train, test, validation, etc)" + }, + "path": { + "type": "string", + "description": "The full filepath to the dataset file or directory" + }, + "label": { + "type": "string", + "description": "A label to assign to this split" + } + } } }, - "output_labels":{ - "type":"array", - "items": { - "type": "string" + "keys": { + "type": "array", + "description": "Keys describing how to load the data", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "type": "array", + "description": "Column or header name for tabular data, key/path for HDF5 data", + "items": { + "type": "string" + } + }, + "type": { + "type": "string", + "description": "Whether input or target" + }, + "filter": { + "type": "string", + "description": "How apply the defined key" + }, + "description": { + "type": "string", + "description": "Free text description of the key" + }, + "units": { + "type": "string", + "description": "The units associated with the key" + }, + "classes": { + "type": "array", + "items": { + "type": "object" + }, + "additionalProperties": false, + "properties": { + "label": { + "type": "string", + "description": "The label that exists in the data" + }, + "name": { + "type": "string", + "description": "The name the label maps onto." + } + } + } + } } - }, - "short_name":{ - "type":"string" - }, - "package_type":{ - "type":"string" } - }, - "additionalProperties": false + } } }, "additionalProperties": false } } -} +} \ No newline at end of file From 1678c6eb7fd89949d47db747613fc85cabbcb738 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Mon, 31 Jan 2022 11:45:54 -0600 Subject: [PATCH 17/25] Make mint_doi property manditory --- connect_aux_data/organizations.json | 10 ++++++++++ schemas/organization.json | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 03c8794..fd6820d 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -12,6 +12,7 @@ "public" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/afrl-midas/submissions" ] @@ -21,6 +22,7 @@ "aliases": [], "description": "Sector 1 of the Advanced Photon Source.", "homepage": "https://www.aps.anl.gov/Users-Information/Help-Reference/Contacts/Sector-Beamline-Locations-Phones", + "mint_doi": true, "permission_groups": [ "public" ], @@ -53,6 +55,7 @@ "canonical_name": "NanoMFG", "description": "The aim of the nanomanufacturing (nanoMFG) node is to develop computational software tools aimed at creating smart, model-driven and experimentally informed nanomanufactured structures and devices.", "homepage": "https://nanohub.org/groups/nanomfg", + "mint_doi": true, "permission_groups": [ "ad88f7cb-cf53-11e9-8526-0e161d24c936" ], @@ -74,6 +77,7 @@ ], "description": "The National Institute of Standards and Technology has created a materials science data repository as part of an effort in coordination with the Materials Genome Initiative (MGI) to establish data exchange protocols and mechanisms that will foster data sharing and reuse across a wide community of researchers, with the goal of enhancing the quality of materials data and models.", "homepage": "https://materialsdata.nist.gov/", + "mint_doi": false, "permission_groups": [ "public" ], @@ -99,6 +103,7 @@ "public" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/verde/" ] @@ -128,6 +133,7 @@ "public" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/lauhon_protected/" ] @@ -142,6 +148,7 @@ "public" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" ] @@ -156,6 +163,7 @@ "public" ], "curation": true, + "mint_doi": true, "project_blocks": [ "foundry-dev" ], @@ -173,6 +181,7 @@ "urn:globus:groups:id:cd106a54-6635-11eb-bfe2-0aa21a0136a3" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/EEL_protected/" ] @@ -187,6 +196,7 @@ "urn:globus:groups:id:4421683d-665e-11eb-b92d-0a4debe59093" ], "curation": true, + "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/differentiate_protected/" ] diff --git a/schemas/organization.json b/schemas/organization.json index 7d97d53..ee68ffd 100644 --- a/schemas/organization.json +++ b/schemas/organization.json @@ -71,6 +71,7 @@ "canonical_name", "permission_groups", "visible_to", - "data_destinations" + "data_destinations", + "mint_doi" ] } \ No newline at end of file From 1a3b516f3ef8e57fe151b942729d3c88321aa9ba Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Mon, 31 Jan 2022 14:32:50 -0600 Subject: [PATCH 18/25] Only allow a single organization --- schemas/connect_submission.json | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/schemas/connect_submission.json b/schemas/connect_submission.json index 9e36b92..63f30fc 100644 --- a/schemas/connect_submission.json +++ b/schemas/connect_submission.json @@ -23,13 +23,9 @@ "type": "string", "description": "A unique program-friendly name for this dataset (will be generated from the title if not specified)." }, - "organizations": { - "type": "array", - "description": "The organizations responsible for dataset creation. Adding organizations here will apply any organizational rules and requirements to the dataset.", - "items": { - "type": "string", - "description": "Name of an organization (e.g., NIST)." - } + "organization": { + "type": "string", + "description": "The organization responsible for dataset creation. Will apply any organizational rules and requirements to the dataset.", } }, "additionalProperties": false From 3e87c01752c0ef392672fdb63aa04751ef782540 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Mon, 31 Jan 2022 14:40:24 -0600 Subject: [PATCH 19/25] Fix typo in schema. Update tests to use single organization --- schemas/connect_submission.json | 2 +- tests/test_files/connect_submission/success_max_alt.json | 2 +- tests/test_files/connect_submission/success_maximal.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/connect_submission.json b/schemas/connect_submission.json index 63f30fc..4615ca4 100644 --- a/schemas/connect_submission.json +++ b/schemas/connect_submission.json @@ -25,7 +25,7 @@ }, "organization": { "type": "string", - "description": "The organization responsible for dataset creation. Will apply any organizational rules and requirements to the dataset.", + "description": "The organization responsible for dataset creation. Will apply any organizational rules and requirements to the dataset." } }, "additionalProperties": false diff --git a/tests/test_files/connect_submission/success_max_alt.json b/tests/test_files/connect_submission/success_max_alt.json index bf90def..101f574 100644 --- a/tests/test_files/connect_submission/success_max_alt.json +++ b/tests/test_files/connect_submission/success_max_alt.json @@ -113,7 +113,7 @@ }, "mdf": { "acl": ["public"], - "organizations": ["Org1", "Org2"], + "organization": "Org1", "source_name": "src_nm" }, "mrr": { diff --git a/tests/test_files/connect_submission/success_maximal.json b/tests/test_files/connect_submission/success_maximal.json index 16b6094..857daf1 100644 --- a/tests/test_files/connect_submission/success_maximal.json +++ b/tests/test_files/connect_submission/success_maximal.json @@ -112,7 +112,7 @@ }, "mdf": { "acl": ["public"], - "organizations": ["Org1", "Org2"], + "organization": "Org1", "source_name": "src_nm" }, "mrr": { From 7e12a4429bdcc051c8c6580ea59306373bc848b6 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Tue, 8 Mar 2022 09:36:37 -0600 Subject: [PATCH 20/25] Remove curation as a submission option --- schemas/connect_submission.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/schemas/connect_submission.json b/schemas/connect_submission.json index 4615ca4..fcc47b4 100644 --- a/schemas/connect_submission.json +++ b/schemas/connect_submission.json @@ -252,10 +252,6 @@ "type": "boolean", "description": "When true mints doi for file." }, - "curation": { - "type": "boolean", - "description": "When true, trigger curation for this submission. This is normally set by an organization." - }, "dataset_acl": { "type": "array", "description": "The UUIDs of Users or Groups allowed to view only the dataset entry, or the special (and default) value 'public'. This does not grant permission to view records or files in the dataset, only the dataset entry itself.", From 740d6a2802e4b3ea121b0d6f295dec3ff6e21be2 Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Tue, 22 Mar 2022 09:36:20 -0500 Subject: [PATCH 21/25] Add domains to mdf metadata --- schemas/mdf.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/schemas/mdf.json b/schemas/mdf.json index 66484c4..71809f4 100644 --- a/schemas/mdf.json +++ b/schemas/mdf.json @@ -28,6 +28,14 @@ }, "minItems": 1 }, + "domains": { + "type": "array", + "description": "Facility domains of the dataset. e.g., materials, chemistry.", + "items": { + "type": "string", + "description": "One facility domain. e.g., materials" + } + }, "ingest_date": { "type": "string", "format": "date-time", From 3f57bde48b502c16e4132aebfcac8c5d266d8225 Mon Sep 17 00:00:00 2001 From: Ben Blaiszik Date: Tue, 22 Mar 2022 09:37:50 -0500 Subject: [PATCH 22/25] Add domains to submission metadata schema --- schemas/organization.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/schemas/organization.json b/schemas/organization.json index ee68ffd..4cfcbbd 100644 --- a/schemas/organization.json +++ b/schemas/organization.json @@ -58,6 +58,14 @@ "type": "boolean", "description": "Whether datasets submitted to this organization mint a Digital Object Identifier" }, + "domains": { + "type": "array", + "description": "Facility domains of the submission. e.g., materials, chemistry.", + "items": { + "type": "string", + "description": "One facility domain. e.g., materials" + } + }, "project_blocks": { "type": "array", "description": "The special project blocks associated with this organization's metadata.", @@ -74,4 +82,4 @@ "data_destinations", "mint_doi" ] -} \ No newline at end of file +} From f232be76b61dfd9c69e990b6d55b59f7645fdbe7 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 31 Mar 2022 12:04:12 -0500 Subject: [PATCH 23/25] Set initial default domains for all organizations --- connect_aux_data/organizations.json | 33 +++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index fd6820d..0b7f721 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -15,7 +15,8 @@ "mint_doi": true, "data_destinations": [ "globus://e55b4eab-6d04-11e5-ba46-22000b92c6ec/afrl-midas/submissions" - ] + ], + "domains": ["materials"] }, { "canonical_name": "APS Sector 1", @@ -31,7 +32,8 @@ ], "visible_to": [ "public" - ] + ], + "domains": ["materials"] }, { "canonical_name": "MDF Open", @@ -49,7 +51,8 @@ "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" - ] + ], + "domains": ["materials"] }, { "canonical_name": "NanoMFG", @@ -67,7 +70,8 @@ ], "project_blocks": [ "nanomfg" - ] + ], + "domains": ["materials"] }, { "canonical_name": "NIST Materials Data Repository", @@ -86,7 +90,8 @@ ], "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" - ] + ], + "domains": ["materials"] }, { "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", @@ -106,7 +111,8 @@ "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/verde/" - ] + ], + "domains": ["materials"] }, { "canonical_name": "Hersam Group", @@ -136,7 +142,8 @@ "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/lauhon_protected/" - ] + ], + "domains": ["materials"] }, { "canonical_name": "Foundry", @@ -151,7 +158,8 @@ "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" - ] + ], + "domains": ["materials"] }, { "canonical_name": "Foundry-dev", @@ -169,7 +177,8 @@ ], "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/foundry/" - ] + ], + "domains": ["materials"] }, { "canonical_name": "Electrochemical Energy Lab", @@ -184,7 +193,8 @@ "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/EEL_protected/" - ] + ], + "domains": ["materials"] }, { "canonical_name": "Differentiate Catalysis", @@ -199,6 +209,7 @@ "mint_doi": true, "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/differentiate_protected/" - ] + ], + "domains": ["materials"] } ] \ No newline at end of file From 1d2572c685dd5100b8c485a479ea98099cf5c206 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Fri, 8 Apr 2022 14:19:31 -0500 Subject: [PATCH 24/25] Make curation a required property for organizations --- connect_aux_data/organizations.json | 9 ++++++--- schemas/organization.json | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/connect_aux_data/organizations.json b/connect_aux_data/organizations.json index 0b7f721..2e82743 100644 --- a/connect_aux_data/organizations.json +++ b/connect_aux_data/organizations.json @@ -33,7 +33,8 @@ "visible_to": [ "public" ], - "domains": ["materials"] + "domains": ["materials"], + "curation": false }, { "canonical_name": "MDF Open", @@ -71,7 +72,8 @@ "project_blocks": [ "nanomfg" ], - "domains": ["materials"] + "domains": ["materials"], + "curation": true }, { "canonical_name": "NIST Materials Data Repository", @@ -91,7 +93,8 @@ "data_destinations": [ "globus://82f1b5c6-6e9b-11e5-ba47-22000b92c6ec/mdf_open/" ], - "domains": ["materials"] + "domains": ["materials"], + "curation": false }, { "canonical_name": "Virtual Excited State Reference for the Discovery of Electronic Materials Database", diff --git a/schemas/organization.json b/schemas/organization.json index 4cfcbbd..0504f3b 100644 --- a/schemas/organization.json +++ b/schemas/organization.json @@ -80,6 +80,7 @@ "permission_groups", "visible_to", "data_destinations", - "mint_doi" + "mint_doi", + "curation" ] } From f16dd817b2cfc63bec329ae0ad075e4c87d18239 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Fri, 8 Apr 2022 15:07:58 -0500 Subject: [PATCH 25/25] Add previous versions property --- schemas/internal_status.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/schemas/internal_status.json b/schemas/internal_status.json index e4ac7b8..17a5cba 100644 --- a/schemas/internal_status.json +++ b/schemas/internal_status.json @@ -81,6 +81,14 @@ "type": "string", "description": "SemVer version string for this submission" }, + "previous_versions": { + "type": "array", + "description": "Ordered list of previous versions", + "items": { + "type": "string", + "description": "Versioned SourceID" + } + }, "updates": { "type": "array", "description": "Any submitted updates ot the original submission."