wip: Single file manifest generation

JacobiClark · JacobiClark · commit 238bedeff0d2 · 2024-10-22T23:29:45.000-07:00
diff --git a/src/pyflask/curate/curate.py b/src/pyflask/curate/curate.py
@@ -3705,184 +3705,83 @@ def copytree(src, dst, symlinks=False, ignore=None):
 
 
 def generate_manifest_file_data(dataset_structure_obj):
-    # modify this function here to handle paths from pennsieve
-    # create path using bfpath key from json object
-
     local_timezone = TZLOCAL()
-
+    
     double_extensions = [
-        ".ome.tiff",
-        ".ome.tif",
-        ".ome.tf2,",
-        ".ome.tf8",
-        ".ome.btf",
-        ".ome.xml",
-        ".brukertiff.gz",
-        ".mefd.gz",
-        ".moberg.gz",
-        ".nii.gz",
-        ".mgh.gz",
-        ".tar.gz",
-        ".bcl.gz",
+        ".ome.tiff", ".ome.tif", ".ome.tf2,", ".ome.tf8", ".ome.btf", ".ome.xml", 
+        ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", ".bcl.gz"
     ]
 
     def get_name_extension(file_name):
-        double_ext = False
         for ext in double_extensions:
-            if file_name.find(ext) != -1:
-                double_ext = True
-                break
-
-        ext = ""
-
-        if double_ext == False:
-            ext = os.path.splitext(file_name)[1]
+            if file_name.endswith(ext):
+                # Extract the base extension before the double extension
+                base_ext = os.path.splitext(os.path.splitext(file_name)[0])[1]
+                return base_ext + ext
+        return os.path.splitext(file_name)[1]
+
+    def build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name):
+        file_manifest_template_data = []
+        filename_entry = "/".join(ds_struct_path) + "/" + file_name if ds_struct_path else file_name
+        file_type_entry = get_name_extension(file_name)
+
+        if filename_entry[:1] == "/":
+            file_manifest_template_data.append(filename_entry[1:])
         else:
-            ext = (
-                os.path.splitext(os.path.splitext(file_name)[0])[1]
-                + os.path.splitext(file_name)[1]
-            )
-        return ext
-
-    def guided_recursive_folder_traversal(folder, hlf_data_array, ds_struct_path):
-        if "files" in folder.keys():
-            standard_manifest_columns = ["filename", "timestamp", "description", "file type", "Additional Metadata"]
-            if(len(hlf_data_array) < 1):
-                hlf_data_array.append(standard_manifest_columns)
-            for item in list(folder["files"]):
-                # do not generate a manifest file entry for the manifest file itself
-                if item in ["manifest.xlsx", "manifest.csv"]:
-                    continue
-                file_manifest_template_data = []
-                local_path_to_file = folder["files"][item]["path"].replace("\\", "/")
-                item_description = folder["files"][item]["description"]
-                item_additional_info = folder["files"][item]["additional-metadata"]
-
-                # The name of the file eg "file.txt"
-                file_name = os.path.basename(local_path_to_file)
-                if file_name != item:
-                    file_name = item
-                if len(ds_struct_path) > 0:
-                    filename_entry = "/".join(ds_struct_path) + "/" + file_name
-                else:
-                    filename_entry = file_name
+            file_manifest_template_data.append(filename_entry)
 
-                # The extension of the file eg ".txt"
-                file_type_entry = get_name_extension(file_name)
+        file_manifest_template_data.append(timestamp_entry)
+        file_manifest_template_data.append(folder["files"][item]["description"])
+        file_manifest_template_data.append(file_type_entry)
+        file_manifest_template_data.append(folder["files"][item]["additional-metadata"])
 
-                # The timestamp of the file on the user's local machine
-                file_path = pathlib.Path(local_path_to_file)
-                mtime = file_path.stat().st_mtime
-                last_mod_time = datetime.fromtimestamp(mtime, tz=local_timezone).fromtimestamp(mtime).astimezone(
-                    local_timezone
-                )
-                timestamp_entry = last_mod_time.isoformat().replace(".", ",").replace("+00:00", "Z")
+        if "extra_columns" in folder["files"][item]:
+            for key, value in folder["files"][item]["extra_columns"].items():
+                file_manifest_template_data.append(value)
+                if key not in hlf_data_array[0]:
+                    hlf_data_array[0].append(key)
 
-                if filename_entry[:1] == "/":
-                    file_manifest_template_data.append(filename_entry[:1])
-                else:
-                    file_manifest_template_data.append(filename_entry)
-
-                file_manifest_template_data.append(timestamp_entry)
-                file_manifest_template_data.append(item_description)
-                file_manifest_template_data.append(file_type_entry)
-                file_manifest_template_data.append(item_additional_info)
-
-                # extra column key is an object of all extra columns of a manifest
-                # key will be the column header and value will be the value of the column+row 
-                # (from the excel) (now in the form of a dict)
-                if "extra_columns" in folder["files"][item]:
-                    for key in folder["files"][item]["extra_columns"]:
-                        file_manifest_template_data.append(folder["files"][item]["extra_columns"][key])
-                        if key not in hlf_data_array[0]:
-                            # add column name to manifest column names array
-                            hlf_data_array[0].append(key)
-
-                hlf_data_array.append(file_manifest_template_data)
-
-        if "folders" in folder.keys():
-            for item in list(folder["folders"]):
-                relative_structure_path.append(item)
-                guided_recursive_folder_traversal(
-                    folder["folders"][item], hlf_data_array, relative_structure_path
-                )
-                relative_structure_path.pop()
-        return
+        return file_manifest_template_data
 
-    def pennsieve_recursive_folder_traversal(folder, hlf_data_array, ds_struct_path):
-        if "files" in folder.keys():
-            standard_manifest_columns = ["filename", "timestamp", "description", "file type", "Additional Metadata"]
-            if(len(hlf_data_array) < 1):
+    def recursive_folder_traversal(folder, hlf_data_array, ds_struct_path, is_pennsieve):
+        if "files" in folder:
+            standard_manifest_columns = ["filename", "timestamp", "description", "file type", "entity", "data modality", "also in dataset", "data dictionary path", "entity is transitive", "Additional Metadata"]
+            if not hlf_data_array:
                 hlf_data_array.append(standard_manifest_columns)
-            for item in list(folder["files"]):
-                file_manifest_template_data = []
+
+            for item in folder["files"]:
                 if item in ["manifest.xlsx", "manifest.csv"]:
                     continue
-                item_description = folder["files"][item]["description"]
-                item_additional_info = folder["files"][item]["additional-metadata"]
-                file_name = ""
-                if folder["files"][item]["type"] == "bf": 
+                
+                if is_pennsieve and folder["files"][item]["type"] == "bf":
                     file_name = os.path.basename(item)
                     timestamp_entry = folder["files"][item]["timestamp"]
                 else:
                     local_path_to_file = folder["files"][item]["path"].replace("\\", "/")
                     file_name = os.path.basename(local_path_to_file)
-                    file_path = pathlib.Path(local_path_to_file)
-                    mtime = file_path.stat().st_mtime
-                    last_mod_time = datetime.fromtimestamp(mtime, tz=local_timezone).fromtimestamp(mtime).astimezone(local_timezone)
-                    timestamp_entry = last_mod_time.isoformat().replace(".", ",").replace("+00:00", "Z")
-
+                    mtime = pathlib.Path(local_path_to_file).stat().st_mtime
+                    timestamp_entry = datetime.fromtimestamp(mtime, tz=local_timezone).isoformat().replace(".", ",").replace("+00:00", "Z")
 
-                filename_entry = "/".join(ds_struct_path) + "/" + file_name
-                file_type_entry = get_name_extension(file_name)
+                hlf_data_array.append(build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name))
 
-                if filename_entry[:1] == "/":
-                    file_manifest_template_data.append(filename_entry[1:])
-                else:
-                    file_manifest_template_data.append(filename_entry)
-                file_manifest_template_data.append(timestamp_entry)
-                file_manifest_template_data.append(item_description)
-                file_manifest_template_data.append(file_type_entry)
-                file_manifest_template_data.append(item_additional_info)
-                # extra column key is an object of all extra columns of a manifest
-                # key will be the column header and value will be the value of the column+row 
-                # (from the excel) (now in the form of a dict)
-                if "extra_columns" in folder["files"][item]:
-                    for key in folder["files"][item]["extra_columns"]:
-                        file_manifest_template_data.append(folder["files"][item]["extra_columns"][key])
-                        if key not in hlf_data_array[0]:
-                            # add column name to manifest column names array
-                            hlf_data_array[0].append(key)
-
-                hlf_data_array.append(file_manifest_template_data)
+        if "folders" in folder:
+            for item in folder["folders"]:
+                ds_struct_path.append(item)
+                recursive_folder_traversal(folder["folders"][item], hlf_data_array, ds_struct_path, is_pennsieve)
+                ds_struct_path.pop()
 
-        if "folders" in folder.keys():
-            for item in list(folder["folders"]):
-                relative_structure_path.append(item)
-                pennsieve_recursive_folder_traversal(
-                    folder["folders"][item], hlf_data_array, relative_structure_path
-                )
-                relative_structure_path.pop()
-        return
-
-    # Initialize the array that the manifest data will be added to.
     hlf_manifest_data = {}
-    # any additional columns created by the user will be appended with the high level folder when found
 
+    namespace_logger.info("Generating manifest file data")
+    namespace_logger.info(dataset_structure_obj)
 
-    # Loop through each high level folder and create a manifest data array for each.
-    for high_level_folder in list(dataset_structure_obj["folders"]):
+    for high_level_folder in dataset_structure_obj["folders"]:
         hlf_data_array = []
-
-        # create an array to keep track of the path to the obj being recursed over
         relative_structure_path = []
-        # hlf_data_array.append(standard_manifest_columns)
 
-        if "bfpath" in dataset_structure_obj["folders"][high_level_folder]:
-            # means the json is from a pennsieve dataset
-            pennsieve_recursive_folder_traversal(dataset_structure_obj["folders"][high_level_folder], hlf_data_array, relative_structure_path)
-        else:
-            guided_recursive_folder_traversal(dataset_structure_obj["folders"][high_level_folder], hlf_data_array, relative_structure_path)
+        is_pennsieve = "bfpath" in dataset_structure_obj["folders"][high_level_folder]
+        recursive_folder_traversal(dataset_structure_obj["folders"][high_level_folder], hlf_data_array, relative_structure_path, is_pennsieve)
+
         hlf_manifest_data[high_level_folder] = hlf_data_array
 
     return hlf_manifest_data
diff --git a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx
@@ -106,4 +106,5 @@ const DatasetTreeViewRenderer = ({ datasetStructure }) => {
     </Stack>
   );
 };
+
 export default DatasetTreeViewRenderer;
diff --git a/src/renderer/src/components/shared/manifest/ManifestEntitySelector.jsx b/src/renderer/src/components/shared/manifest/ManifestEntitySelector.jsx
@@ -9,15 +9,16 @@ const ManifestEntitySelector = () => {
 
   const handleButtonClick = () => {
     console.log("Button clicked!");
-    setDatasetStructureJSONObj(window.datasetStructureJSONObj);
+    const datasetStructureJSONObjCopy = JSON.parse(JSON.stringify(window.datasetStructureJSONObj));
+    setDatasetStructureJSONObj(datasetStructureJSONObjCopy);
   };
 
   return (
     <FullWidthContainer>
       <Button onClick={handleButtonClick}>Set Name</Button>
       <Grid>
-        <Grid.Col span={3}>Entity</Grid.Col>
-        <Grid.Col span={9}>
+        <Grid.Col span={4}>Entity</Grid.Col>
+        <Grid.Col span={8}>
           <DatasetTreeViewRenderer datasetStructure={datasetStructureJSONObj} />
         </Grid.Col>
       </Grid>
diff --git a/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js b/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js
@@ -1278,7 +1278,7 @@ const savePageChanges = async (pageBeingLeftID) => {
     }
 
     if (pageBeingLeftID === "guided-subjects-addition-tab") {
-      if (getExistingSubjectNames().length === 0) {
+      if (window.getExistingSubjectNames().length === 0) {
         errorArray.push({
           type: "notyf",
           message: "Please add at least one subject",
@@ -1461,7 +1461,7 @@ const savePageChanges = async (pageBeingLeftID) => {
           throw errorArray;
         }
 
-        const subjects = getExistingSubjectNames();
+        const subjects = window.getExistingSubjectNames();
         if (subjects.length === 0) {
           errorArray.push({
             type: "swal",
@@ -5284,7 +5284,7 @@ window.openPage = async (targetPageID) => {
       // The page is unskipped only if the user has not added any subjects,
       // indicated that they will be adding subjects, and the user is not starting from Pennsieve
       if (
-        getExistingSubjectNames().length === 0 &&
+        window.getExistingSubjectNames().length === 0 &&
         window.sodaJSONObj["starting-point"]["type"] != "bf" &&
         window.sodaJSONObj["button-config"]["dataset-contains-subjects"] === "yes"
       ) {
@@ -5373,19 +5373,16 @@ window.openPage = async (targetPageID) => {
     }
 
     if (targetPageID === "guided-dataset-structure-review-tab") {
-      // Delete the guided high level folders if they are empty
-      // They could possibly be empty if the user did not add any subject data
-      // These will be added back safely when the user traverses back to the high level folder's page
-      for (const folder of guidedHighLevelFolders) {
-        const rootFolderPath = window.datasetStructureJSONObj["folders"][folder];
-
+      // Remove empty guided high-level folders (primary, source, derivative)
+      guidedHighLevelFolders.forEach((folder) => {
+        const rootFolderPath = window.datasetStructureJSONObj?.folders?.[folder];
         if (rootFolderPath && folderIsEmpty(rootFolderPath)) {
-          delete window.datasetStructureJSONObj["folders"][folder];
+          delete window.datasetStructureJSONObj?.folders?.[folder];
         }
-      }
+      });
 
       guidedShowTreePreview(
-        window.sodaJSONObj["digital-metadata"]["name"],
+        window.sodaJSONObj?.["digital-metadata"]?.name,
         "guided-folder-structure-review"
       );
     }
@@ -5432,6 +5429,8 @@ window.openPage = async (targetPageID) => {
         { timeout: 0 }
       );
       const manifestRes = res.data;
+      console.log("manifestRes");
+      console.log(manifestRes);
       //loop through each of the high level folders and store their manifest headers and data
       //into the window.sodaJSONObj
 
@@ -5463,6 +5462,8 @@ window.openPage = async (targetPageID) => {
       //Rerender the manifest cards
       renderManifestCards();
     }
+    if (targetPageID === "guided-manifest-subject-entity-selector-tab") {
+    }
 
     if (targetPageID === "guided-create-submission-metadata-tab") {
       if (pageNeedsUpdateFromPennsieve(targetPageID)) {
@@ -6843,7 +6844,7 @@ const guidedOpenEntityEditSwal = async (entityName) => {
   let entityPrefix;
 
   if (entityName.startsWith("sub-")) {
-    preExistingEntities = getExistingSubjectNames();
+    preExistingEntities = window.getExistingSubjectNames();
     entityNameSingular = "subject";
     entityPrefix = "sub-";
   }
@@ -11192,7 +11193,7 @@ const addSubjectSpecificationTableRow = () => {
   }
 };
 
-const getExistingSubjectNames = () => {
+window.getExistingSubjectNames = () => {
   // Get all subjects in pools and outside of pools
   const [subjectsInPools, subjectsOutsidePools] = window.sodaJSONObj.getAllSubjects();
   // Combine the two arrays
@@ -11489,7 +11490,7 @@ document
       const sampleName = lowercaseFirstLetter(row["sample id"]);
 
       // Check to see if the subject already exists
-      const subjectAlreadyExists = getExistingSubjectNames().includes(subjectName);
+      const subjectAlreadyExists = window.getExistingSubjectNames().includes(subjectName);
       if (!subjectAlreadyExists) {
         window.sodaJSONObj.addSubject(subjectName);
         if (subjectsPool) {
@@ -11569,7 +11570,7 @@ const guidedAddListOfSubjects = async (subjectNameArray, showWarningForExistingS
   formattedSubjectNameArray.filter((subjectName) => subjectName.length > 0);
 
   // Get an array of existing subjects to check for duplicates
-  const existingSubjects = getExistingSubjectNames();
+  const existingSubjects = window.getExistingSubjectNames();
 
   // Array of the subjects that already exist in the dataset
   const duplicateSubjects = formattedSubjectNameArray.filter((subjectName) =>
@@ -11647,14 +11648,14 @@ const convertArrayToCommaSeparatedString = (array) => {
 
 const guidedOpenEntityAdditionSwal = async (entityName) => {
   // Get a list of the existing entities so we can check for duplicates
-  // const subjects = getExistingSubjectNames();
+  // const subjects = window.getExistingSubjectNames();
   let preExistingEntities;
   let entityNameSingular;
   let entityPrefix;
 
   // case when adding subjects
   if (entityName === "subjects") {
-    preExistingEntities = getExistingSubjectNames();
+    preExistingEntities = window.getExistingSubjectNames();
     entityNameSingular = "subject";
     entityPrefix = "sub-";
   }
@@ -13376,7 +13377,7 @@ const handleMultipleSubSectionDisplay = async (controlledSectionID) => {
     // (case for updating a dataset from Pennsieve or old progress files),
     // Select the option for them
     if (!sodaJSONObj["button-config"]["subject-addition-method"]) {
-      if (getExistingSubjectNames().length > 0) {
+      if (window.getExistingSubjectNames().length > 0) {
         document.getElementById("guided-button-add-subject-structure-manually").click();
       }
     }
diff --git a/src/renderer/src/scripts/others/renderer.js b/src/renderer/src/scripts/others/renderer.js
diff --git a/src/renderer/src/scripts/others/tab-effects.js b/src/renderer/src/scripts/others/tab-effects.js
