wip: Add DFS search to dataset tree view

fairdataihub · Oct 30, 2024 · c6b38b5 · c6b38b5
1 parent 31a9a3b
commit c6b38b5
Show file tree

Hide file tree

Showing 6 changed files with 178 additions and 111 deletions.
diff --git a/src/pyflask/curate/curate.py b/src/pyflask/curate/curate.py
@@ -3706,48 +3706,49 @@ def copytree(src, dst, symlinks=False, ignore=None):
 
 def generate_manifest_file_data(dataset_structure_obj):
     local_timezone = TZLOCAL()
-    namespace_logger.info("Generating manifest file data for dataset structure:")
-    namespace_logger.info(dataset_structure_obj)
-
-    manifest_headers = [
-        'filename', 'timestamp', 'description', 'file type', 'entity', 'data modality', 
-        'also in dataset', 'also in dataset path', 'data dictionary path', 
-        'entity is transitive', 'Additional Metadata'
+
+    double_extensions = [
+        ".ome.tiff", ".ome.tif", ".ome.tf2,", ".ome.tf8", ".ome.btf", ".ome.xml", 
+        ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", ".bcl.gz"
     ]
-    manifest_data = []
 
     def get_name_extension(file_name):
-        double_extensions = [".ome.tiff", ".ome.tif", ".ome.tf2,", ".ome.tf8", ".ome.btf", ".ome.xml", 
-                             ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", 
-                             ".bcl.gz"]
         for ext in double_extensions:
             if file_name.endswith(ext):
+                # Extract the base extension before the double extension
                 base_ext = os.path.splitext(os.path.splitext(file_name)[0])[1]
                 return base_ext + ext
         return os.path.splitext(file_name)[1]
 
     def build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name):
-        # Basic columns for a manifest entry
-        file_manifest_template_data = [
-            "/".join(ds_struct_path) + "/" + file_name if ds_struct_path else file_name,
-            timestamp_entry,
-            folder["files"][item].get("description", ""),
-            get_name_extension(file_name),
-            folder["files"][item].get("additional-metadata", "")
-        ]
+        file_manifest_template_data = []
+        filename_entry = "/".join(ds_struct_path) + "/" + file_name if ds_struct_path else file_name
+        file_type_entry = get_name_extension(file_name)
+
+        if filename_entry[:1] == "/":
+            file_manifest_template_data.append(filename_entry[1:])
+        else:
+            file_manifest_template_data.append(filename_entry)
+
+        file_manifest_template_data.append(timestamp_entry)
+        file_manifest_template_data.append(folder["files"][item]["description"])
+        file_manifest_template_data.append(file_type_entry)
+        file_manifest_template_data.append(folder["files"][item]["additional-metadata"])
 
-        # Add extra columns dynamically if they exist
         if "extra_columns" in folder["files"][item]:
             for key, value in folder["files"][item]["extra_columns"].items():
-                if key not in manifest_headers:
-                    manifest_headers.append(key)
                 file_manifest_template_data.append(value)
+                if key not in hlf_data_array[0]:
+                    hlf_data_array[0].append(key)
 
         return file_manifest_template_data
 
-    def recursive_folder_traversal(folder, ds_struct_path, is_pennsieve):
-        # Traverse files in the folder
+    def recursive_folder_traversal(folder, hlf_data_array, ds_struct_path, is_pennsieve):
         if "files" in folder:
+            standard_manifest_columns = ["filename", "timestamp", "description", "file type", "entity", "data modality", "also in dataset", "data dictionary path", "entity is transitive", "Additional Metadata"]
+            if not hlf_data_array:
+                hlf_data_array.append(standard_manifest_columns)
+
             for item in folder["files"]:
                 if item in ["manifest.xlsx", "manifest.csv"]:
                     continue
@@ -3761,28 +3762,33 @@ def recursive_folder_traversal(folder, ds_struct_path, is_pennsieve):
                     mtime = pathlib.Path(local_path_to_file).stat().st_mtime
                     timestamp_entry = datetime.fromtimestamp(mtime, tz=local_timezone).isoformat().replace(".", ",").replace("+00:00", "Z")
 
-                manifest_data.append(build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name))
+                hlf_data_array.append(build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name))
 
-        # Recursively traverse subfolders
         if "folders" in folder:
-            for subfolder_name, subfolder_content in folder["folders"].items():
-                ds_struct_path.append(subfolder_name)
-                recursive_folder_traversal(subfolder_content, ds_struct_path, is_pennsieve)
+            for item in folder["folders"]:
+                ds_struct_path.append(item)
+                recursive_folder_traversal(folder["folders"][item], hlf_data_array, ds_struct_path, is_pennsieve)
                 ds_struct_path.pop()
 
-    # Begin recursive traversal from the top-level folders
-    for high_level_folder, folder_content in dataset_structure_obj["folders"].items():
-        is_pennsieve = "bfpath" in folder_content
-        recursive_folder_traversal(folder_content, [high_level_folder], is_pennsieve)
+    hlf_manifest_data = {}
+
+    namespace_logger.info("Generating manifest file data")
+    namespace_logger.info(dataset_structure_obj)
+
+    for high_level_folder in dataset_structure_obj["folders"]:
+        hlf_data_array = []
+        relative_structure_path = []
+
+        is_pennsieve = "bfpath" in dataset_structure_obj["folders"][high_level_folder]
+        recursive_folder_traversal(dataset_structure_obj["folders"][high_level_folder], hlf_data_array, relative_structure_path, is_pennsieve)
 
-    namespace_logger.info("Generated manifest data:")
-    namespace_logger.info(manifest_data)
+        hlf_manifest_data[high_level_folder] = hlf_data_array
 
-    return manifest_headers, manifest_data
+    return hlf_manifest_data
 
 
 def handle_duplicate_package_name_error(e, soda_json_structure):
     if "if-existing-files" in soda_json_structure["generate-dataset"] and (soda_json_structure["generate-dataset"]["if-existing-files"] == "create-duplicate") and (e.response.text== '{"type":"BadRequest","message":"package name must be unique","code":400}'):
         return
 
-    raise e
+    raise e
diff --git a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx
@@ -1,5 +1,5 @@
 import { useState } from "react";
-import { Collapse, Text, Stack, UnstyledButton } from "@mantine/core";
+import { Collapse, Text, Stack, UnstyledButton, TextInput } from "@mantine/core";
 import { useHover } from "@mantine/hooks";
 import {
   IconFolder,
@@ -16,8 +16,10 @@ import {
   IconFileTypeXls,
   IconFileTypeXml,
   IconFileTypeZip,
+  IconSearch,
 } from "@tabler/icons-react";
-import { getEntityForRelativePath } from "../../../stores/slices/manifestEntitySelectorSlice";
+import useGlobalStore from "../../../stores/globalStore";
+import { setDatasetstructureSearchFilter } from "../../../stores/slices/datasetTreeViewSlice";
 
 // Constants
 const FOLDER_ICON_COLOR = "#ADD8E6";
@@ -42,39 +44,42 @@ const fileIconMap = {
   jp2: <IconPhoto size={FILE_ICON_SIZE} />,
 };
 
+// Retrieve icon based on file extension
 const getFileTypeIcon = (fileName) => {
   const extension = fileName.split(".").pop().toLowerCase();
   return fileIconMap[extension] || <IconFile size={FILE_ICON_SIZE} />;
 };
 
 // FileItem component
-const FileItem = ({ name, content, onFileClick, getFileBackgroundColor }) => {
-  const fileBackgroundColor = getFileBackgroundColor(content.relativePath);
-  return (
-    <div
-      style={{
-        paddingLeft: 10,
-        display: "flex",
-        alignItems: "center",
-        gap: 5,
-        backgroundColor: fileBackgroundColor,
-      }}
-      onClick={() => onFileClick(name, content)}
-    >
-      {getFileTypeIcon(name)}
-      <Text>{name}</Text>
-    </div>
-  );
-};
+const FileItem = ({ name, content, onFileClick, getFileBackgroundColor }) => (
+  <div
+    style={{
+      paddingLeft: 10,
+      display: "flex",
+      alignItems: "center",
+      gap: 5,
+      backgroundColor: getFileBackgroundColor(content.relativePath),
+    }}
+    onClick={() => onFileClick(name, content)}
+  >
+    {getFileTypeIcon(name)}
+    <Text>{name}</Text>
+  </div>
+);
 
 // FolderItem component
-const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgroundColor }) => {
+const FolderItem = ({
+  name,
+  content,
+  onFolderClick,
+  onFileClick,
+  getFileBackgroundColor,
+  searchFilter,
+}) => {
   const [isOpen, setIsOpen] = useState(false);
   const { hovered, ref } = useHover();
 
-  const toggleFolder = () => {
-    setIsOpen((prev) => !prev);
-  };
+  const toggleFolder = () => setIsOpen((prev) => !prev);
 
   return (
     <Stack gap={1}>
@@ -100,22 +105,23 @@ const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgrou
           <Text size="lg">{name}</Text>
         </UnstyledButton>
       </div>
-      <Collapse in={isOpen} ml="xs">
-        {Object.keys(content.folders || {}).map((folderName) => (
+      <Collapse in={isOpen}>
+        {content.filteredFolders?.map(([folderName, folderContent]) => (
           <FolderItem
             key={folderName}
             name={folderName}
-            content={content.folders[folderName]}
+            content={folderContent}
             onFolderClick={onFolderClick}
             onFileClick={onFileClick}
             getFileBackgroundColor={getFileBackgroundColor}
+            searchFilter={searchFilter}
           />
         ))}
-        {Object.keys(content.files || {}).map((fileName) => (
+        {content.filteredFiles?.map(([fileName, fileContent]) => (
           <FileItem
             key={fileName}
             name={fileName}
-            content={content.files[fileName]}
+            content={fileContent}
             onFileClick={onFileClick}
             getFileBackgroundColor={getFileBackgroundColor}
           />
@@ -125,35 +131,74 @@ const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgrou
   );
 };
 
+// Recursive function to filter folders and files, marking parent folders if a child matches
+const filterStructure = (structure, searchFilter) => {
+  const lowerCaseFilter = searchFilter.toLowerCase();
+
+  const folders = Object.entries(structure.folders || {}).reduce((acc, [name, content]) => {
+    const filteredContent = filterStructure(content, searchFilter);
+    if (filteredContent || name.toLowerCase().includes(lowerCaseFilter)) {
+      acc[name] = { ...content, ...filteredContent };
+    }
+    return acc;
+  }, {});
+
+  const files = Object.entries(structure.files || {}).filter(([name]) =>
+    name.toLowerCase().includes(lowerCaseFilter)
+  );
+
+  if (Object.keys(folders).length || files.length) {
+    return { filteredFolders: Object.entries(folders), filteredFiles: files };
+  }
+  return null;
+};
+
 // Main component
 const DatasetTreeView = ({
   datasetStructure,
   onFolderClick,
   onFileClick,
   getFileBackgroundColor,
-}) =>
-  !datasetStructure?.folders && !datasetStructure?.files ? null : (
+}) => {
+  const datasetStructureSearchFilter = useGlobalStore(
+    (state) => state.datasetStructureSearchFilter
+  );
+
+  const handleSearchChange = (event) => setDatasetstructureSearchFilter(event.target.value);
+
+  const filteredStructure = filterStructure(datasetStructure, datasetStructureSearchFilter);
+
+  return (
     <Stack gap={1}>
-      {Object.keys(datasetStructure.files || {}).map((fileName) => (
-        <FileItem
-          key={fileName}
-          name={fileName}
-          content={datasetStructure.files[fileName]}
-          onFileClick={onFileClick}
-          getFileBackgroundColor={getFileBackgroundColor}
-        />
-      ))}
-      {Object.keys(datasetStructure.folders || {}).map((folderName) => (
+      <TextInput
+        label="Search files and folders:"
+        placeholder="Search files and folders..."
+        value={datasetStructureSearchFilter}
+        onChange={handleSearchChange}
+        leftSection={<IconSearch stroke={1.5} />}
+      />
+      {filteredStructure?.filteredFolders?.map(([folderName, folderContent]) => (
         <FolderItem
           key={folderName}
           name={folderName}
-          content={datasetStructure.folders[folderName]}
+          content={folderContent}
           onFolderClick={onFolderClick}
           onFileClick={onFileClick}
           getFileBackgroundColor={getFileBackgroundColor}
+          searchFilter={datasetStructureSearchFilter}
+        />
+      ))}
+      {filteredStructure?.filteredFiles?.map(([fileName, fileContent]) => (
+        <FileItem
+          key={fileName}
+          name={fileName}
+          content={fileContent}
+          onFileClick={onFileClick}
+          getFileBackgroundColor={getFileBackgroundColor}
         />
       ))}
     </Stack>
   );
+};
 
 export default DatasetTreeView;
diff --git a/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js b/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js
@@ -44,11 +44,8 @@ import {
   setGuidedDatasetName,
   setGuidedDatasetSubtitle,
 } from "../../stores/slices/guidedModeSlice";
-import {
-  setDatasetStructureJSONObj,
-  setEntityList,
-  setEntityType,
-} from "../../stores/slices/manifestEntitySelectorSlice";
+import { setEntityList, setEntityType } from "../../stores/slices/manifestEntitySelectorSlice";
+import { setTreeViewDatasetStructure } from "../../stores/slices/datasetTreeViewSlice";
 
 import "bootstrap-select";
 import Cropper from "cropperjs";
@@ -5467,7 +5464,7 @@ window.openPage = async (targetPageID) => {
     if (targetPageID === "guided-manifest-subject-entity-selector-tab") {
       //
       setEntityList(window.getExistingSubjectNames());
-      setDatasetStructureJSONObj(window.datasetStructureJSONObj);
+      setTreeViewDatasetStructure(window.datasetStructureJSONObj);
       setEntityType("subjects");
     }
 

diff --git a/src/renderer/src/stores/globalStore.js b/src/renderer/src/stores/globalStore.js
@@ -6,6 +6,7 @@ import { dropDownSlice } from "./slices/dropDownSlice";
 import { singleColumnTableSlice } from "./slices/tableRowSlice";
 import { backgroundServicesSlice } from "./slices/backgroundServicesSlice";
 import { manifestEntitySelectorSlice } from "./slices/manifestEntitySelectorSlice";
+import { datasetTreeViewSlice } from "./slices/datasetTreeViewSlice";
 
 const useGlobalStore = create(
   immer((...a) => ({
@@ -15,6 +16,7 @@ const useGlobalStore = create(
     ...singleColumnTableSlice(...a),
     ...backgroundServicesSlice(...a),
     ...manifestEntitySelectorSlice(...a),
+    ...datasetTreeViewSlice(...a),
   }))
 );