diff --git a/src/pyflask/curate/curate.py b/src/pyflask/curate/curate.py index 4cf69ba46..5477c8909 100644 --- a/src/pyflask/curate/curate.py +++ b/src/pyflask/curate/curate.py @@ -3706,48 +3706,49 @@ def copytree(src, dst, symlinks=False, ignore=None): def generate_manifest_file_data(dataset_structure_obj): local_timezone = TZLOCAL() - namespace_logger.info("Generating manifest file data for dataset structure:") - namespace_logger.info(dataset_structure_obj) - - manifest_headers = [ - 'filename', 'timestamp', 'description', 'file type', 'entity', 'data modality', - 'also in dataset', 'also in dataset path', 'data dictionary path', - 'entity is transitive', 'Additional Metadata' + + double_extensions = [ + ".ome.tiff", ".ome.tif", ".ome.tf2,", ".ome.tf8", ".ome.btf", ".ome.xml", + ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", ".bcl.gz" ] - manifest_data = [] def get_name_extension(file_name): - double_extensions = [".ome.tiff", ".ome.tif", ".ome.tf2,", ".ome.tf8", ".ome.btf", ".ome.xml", - ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", - ".bcl.gz"] for ext in double_extensions: if file_name.endswith(ext): + # Extract the base extension before the double extension base_ext = os.path.splitext(os.path.splitext(file_name)[0])[1] return base_ext + ext return os.path.splitext(file_name)[1] def build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name): - # Basic columns for a manifest entry - file_manifest_template_data = [ - "/".join(ds_struct_path) + "/" + file_name if ds_struct_path else file_name, - timestamp_entry, - folder["files"][item].get("description", ""), - get_name_extension(file_name), - folder["files"][item].get("additional-metadata", "") - ] + file_manifest_template_data = [] + filename_entry = "/".join(ds_struct_path) + "/" + file_name if ds_struct_path else file_name + file_type_entry = get_name_extension(file_name) + + if filename_entry[:1] == "/": + file_manifest_template_data.append(filename_entry[1:]) + else: + file_manifest_template_data.append(filename_entry) + + file_manifest_template_data.append(timestamp_entry) + file_manifest_template_data.append(folder["files"][item]["description"]) + file_manifest_template_data.append(file_type_entry) + file_manifest_template_data.append(folder["files"][item]["additional-metadata"]) - # Add extra columns dynamically if they exist if "extra_columns" in folder["files"][item]: for key, value in folder["files"][item]["extra_columns"].items(): - if key not in manifest_headers: - manifest_headers.append(key) file_manifest_template_data.append(value) + if key not in hlf_data_array[0]: + hlf_data_array[0].append(key) return file_manifest_template_data - def recursive_folder_traversal(folder, ds_struct_path, is_pennsieve): - # Traverse files in the folder + def recursive_folder_traversal(folder, hlf_data_array, ds_struct_path, is_pennsieve): if "files" in folder: + standard_manifest_columns = ["filename", "timestamp", "description", "file type", "entity", "data modality", "also in dataset", "data dictionary path", "entity is transitive", "Additional Metadata"] + if not hlf_data_array: + hlf_data_array.append(standard_manifest_columns) + for item in folder["files"]: if item in ["manifest.xlsx", "manifest.csv"]: continue @@ -3761,28 +3762,33 @@ def recursive_folder_traversal(folder, ds_struct_path, is_pennsieve): mtime = pathlib.Path(local_path_to_file).stat().st_mtime timestamp_entry = datetime.fromtimestamp(mtime, tz=local_timezone).isoformat().replace(".", ",").replace("+00:00", "Z") - manifest_data.append(build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name)) + hlf_data_array.append(build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name)) - # Recursively traverse subfolders if "folders" in folder: - for subfolder_name, subfolder_content in folder["folders"].items(): - ds_struct_path.append(subfolder_name) - recursive_folder_traversal(subfolder_content, ds_struct_path, is_pennsieve) + for item in folder["folders"]: + ds_struct_path.append(item) + recursive_folder_traversal(folder["folders"][item], hlf_data_array, ds_struct_path, is_pennsieve) ds_struct_path.pop() - # Begin recursive traversal from the top-level folders - for high_level_folder, folder_content in dataset_structure_obj["folders"].items(): - is_pennsieve = "bfpath" in folder_content - recursive_folder_traversal(folder_content, [high_level_folder], is_pennsieve) + hlf_manifest_data = {} + + namespace_logger.info("Generating manifest file data") + namespace_logger.info(dataset_structure_obj) + + for high_level_folder in dataset_structure_obj["folders"]: + hlf_data_array = [] + relative_structure_path = [] + + is_pennsieve = "bfpath" in dataset_structure_obj["folders"][high_level_folder] + recursive_folder_traversal(dataset_structure_obj["folders"][high_level_folder], hlf_data_array, relative_structure_path, is_pennsieve) - namespace_logger.info("Generated manifest data:") - namespace_logger.info(manifest_data) + hlf_manifest_data[high_level_folder] = hlf_data_array - return manifest_headers, manifest_data + return hlf_manifest_data def handle_duplicate_package_name_error(e, soda_json_structure): if "if-existing-files" in soda_json_structure["generate-dataset"] and (soda_json_structure["generate-dataset"]["if-existing-files"] == "create-duplicate") and (e.response.text== '{"type":"BadRequest","message":"package name must be unique","code":400}'): return - raise e \ No newline at end of file + raise e diff --git a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx index 57e3da380..dd2040582 100644 --- a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx +++ b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx @@ -1,5 +1,5 @@ import { useState } from "react"; -import { Collapse, Text, Stack, UnstyledButton } from "@mantine/core"; +import { Collapse, Text, Stack, UnstyledButton, TextInput } from "@mantine/core"; import { useHover } from "@mantine/hooks"; import { IconFolder, @@ -16,8 +16,10 @@ import { IconFileTypeXls, IconFileTypeXml, IconFileTypeZip, + IconSearch, } from "@tabler/icons-react"; -import { getEntityForRelativePath } from "../../../stores/slices/manifestEntitySelectorSlice"; +import useGlobalStore from "../../../stores/globalStore"; +import { setDatasetstructureSearchFilter } from "../../../stores/slices/datasetTreeViewSlice"; // Constants const FOLDER_ICON_COLOR = "#ADD8E6"; @@ -42,39 +44,42 @@ const fileIconMap = { jp2: , }; +// Retrieve icon based on file extension const getFileTypeIcon = (fileName) => { const extension = fileName.split(".").pop().toLowerCase(); return fileIconMap[extension] || ; }; // FileItem component -const FileItem = ({ name, content, onFileClick, getFileBackgroundColor }) => { - const fileBackgroundColor = getFileBackgroundColor(content.relativePath); - return ( -
onFileClick(name, content)} - > - {getFileTypeIcon(name)} - {name} -
- ); -}; +const FileItem = ({ name, content, onFileClick, getFileBackgroundColor }) => ( +
onFileClick(name, content)} + > + {getFileTypeIcon(name)} + {name} +
+); // FolderItem component -const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgroundColor }) => { +const FolderItem = ({ + name, + content, + onFolderClick, + onFileClick, + getFileBackgroundColor, + searchFilter, +}) => { const [isOpen, setIsOpen] = useState(false); const { hovered, ref } = useHover(); - const toggleFolder = () => { - setIsOpen((prev) => !prev); - }; + const toggleFolder = () => setIsOpen((prev) => !prev); return ( @@ -100,22 +105,23 @@ const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgrou {name} - - {Object.keys(content.folders || {}).map((folderName) => ( + + {content.filteredFolders?.map(([folderName, folderContent]) => ( ))} - {Object.keys(content.files || {}).map((fileName) => ( + {content.filteredFiles?.map(([fileName, fileContent]) => ( @@ -125,35 +131,74 @@ const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgrou ); }; +// Recursive function to filter folders and files, marking parent folders if a child matches +const filterStructure = (structure, searchFilter) => { + const lowerCaseFilter = searchFilter.toLowerCase(); + + const folders = Object.entries(structure.folders || {}).reduce((acc, [name, content]) => { + const filteredContent = filterStructure(content, searchFilter); + if (filteredContent || name.toLowerCase().includes(lowerCaseFilter)) { + acc[name] = { ...content, ...filteredContent }; + } + return acc; + }, {}); + + const files = Object.entries(structure.files || {}).filter(([name]) => + name.toLowerCase().includes(lowerCaseFilter) + ); + + if (Object.keys(folders).length || files.length) { + return { filteredFolders: Object.entries(folders), filteredFiles: files }; + } + return null; +}; + // Main component const DatasetTreeView = ({ datasetStructure, onFolderClick, onFileClick, getFileBackgroundColor, -}) => - !datasetStructure?.folders && !datasetStructure?.files ? null : ( +}) => { + const datasetStructureSearchFilter = useGlobalStore( + (state) => state.datasetStructureSearchFilter + ); + + const handleSearchChange = (event) => setDatasetstructureSearchFilter(event.target.value); + + const filteredStructure = filterStructure(datasetStructure, datasetStructureSearchFilter); + + return ( - {Object.keys(datasetStructure.files || {}).map((fileName) => ( - - ))} - {Object.keys(datasetStructure.folders || {}).map((folderName) => ( + } + /> + {filteredStructure?.filteredFolders?.map(([folderName, folderContent]) => ( + ))} + {filteredStructure?.filteredFiles?.map(([fileName, fileContent]) => ( + ))} ); +}; export default DatasetTreeView; diff --git a/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js b/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js index 7b346aef4..07eaad262 100644 --- a/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js +++ b/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js @@ -44,11 +44,8 @@ import { setGuidedDatasetName, setGuidedDatasetSubtitle, } from "../../stores/slices/guidedModeSlice"; -import { - setDatasetStructureJSONObj, - setEntityList, - setEntityType, -} from "../../stores/slices/manifestEntitySelectorSlice"; +import { setEntityList, setEntityType } from "../../stores/slices/manifestEntitySelectorSlice"; +import { setTreeViewDatasetStructure } from "../../stores/slices/datasetTreeViewSlice"; import "bootstrap-select"; import Cropper from "cropperjs"; @@ -5467,7 +5464,7 @@ window.openPage = async (targetPageID) => { if (targetPageID === "guided-manifest-subject-entity-selector-tab") { // setEntityList(window.getExistingSubjectNames()); - setDatasetStructureJSONObj(window.datasetStructureJSONObj); + setTreeViewDatasetStructure(window.datasetStructureJSONObj); setEntityType("subjects"); } diff --git a/src/renderer/src/stores/globalStore.js b/src/renderer/src/stores/globalStore.js index c88ed057c..ec514ad5e 100644 --- a/src/renderer/src/stores/globalStore.js +++ b/src/renderer/src/stores/globalStore.js @@ -6,6 +6,7 @@ import { dropDownSlice } from "./slices/dropDownSlice"; import { singleColumnTableSlice } from "./slices/tableRowSlice"; import { backgroundServicesSlice } from "./slices/backgroundServicesSlice"; import { manifestEntitySelectorSlice } from "./slices/manifestEntitySelectorSlice"; +import { datasetTreeViewSlice } from "./slices/datasetTreeViewSlice"; const useGlobalStore = create( immer((...a) => ({ @@ -15,6 +16,7 @@ const useGlobalStore = create( ...singleColumnTableSlice(...a), ...backgroundServicesSlice(...a), ...manifestEntitySelectorSlice(...a), + ...datasetTreeViewSlice(...a), })) ); diff --git a/src/renderer/src/stores/slices/datasetTreeViewSlice.js b/src/renderer/src/stores/slices/datasetTreeViewSlice.js new file mode 100644 index 000000000..308c68499 --- /dev/null +++ b/src/renderer/src/stores/slices/datasetTreeViewSlice.js @@ -0,0 +1,43 @@ +import useGlobalStore from "../globalStore"; + +const initialState = { + datasetStructureJSONObj: null, + datasetStructureSearchFilter: "", +}; + +export const datasetTreeViewSlice = (set) => ({ + ...initialState, +}); + +export const setDatasetstructureSearchFilter = (datasetStructureSearchFilter) => { + useGlobalStore.setState((state) => ({ + ...state, + datasetStructureSearchFilter, + })); +}; + +export const setTreeViewDatasetStructure = (datasetStructureJSONObj) => { + // Create a deep copy of the object so that the original object is not mutated + const datasetStructureJSONObjCopy = JSON.parse(JSON.stringify(datasetStructureJSONObj)); + // Recursively add the relative paths to the datasetStructureJSONObjCopy object + // for use in the manifest entity selector component + const addRelativePaths = (obj, path = "") => { + const objFolders = Object.keys(obj.folders || {}); + const objFiles = Object.keys(obj.files || {}); + objFolders.forEach((folder) => { + const folderPath = path ? `${path}/${folder}` : folder; + obj.folders[folder].relativePath = folderPath; + addRelativePaths(obj.folders[folder], folderPath); + }); + objFiles.forEach((file) => { + const filePath = path ? `${path}/${file}` : file; + obj.files[file].relativePath = filePath; + }); + }; + addRelativePaths(datasetStructureJSONObjCopy); + console.log("Dataset structure JSON object:", datasetStructureJSONObjCopy); + useGlobalStore.setState((state) => ({ + ...state, + datasetStructureJSONObj: datasetStructureJSONObjCopy, + })); +}; diff --git a/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js b/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js index aec684590..98ac82a3f 100644 --- a/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js +++ b/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js @@ -21,32 +21,6 @@ export const resetManifestEntitySelectorState = () => { ); }; -export const setDatasetStructureJSONObj = (datasetStructureJSONObj) => { - // Create a deep copy of the object so that the original object is not mutated - const datasetStructureJSONObjCopy = JSON.parse(JSON.stringify(datasetStructureJSONObj)); - // Recursively add the relative paths to the datasetStructureJSONObjCopy object - // for use in the manifest entity selector component - const addRelativePaths = (obj, path = "") => { - const objFolders = Object.keys(obj.folders || {}); - const objFiles = Object.keys(obj.files || {}); - objFolders.forEach((folder) => { - const folderPath = path ? `${path}/${folder}` : folder; - obj.folders[folder].relativePath = folderPath; - addRelativePaths(obj.folders[folder], folderPath); - }); - objFiles.forEach((file) => { - const filePath = path ? `${path}/${file}` : file; - obj.files[file].relativePath = filePath; - }); - }; - addRelativePaths(datasetStructureJSONObjCopy); - console.log("Dataset structure JSON object:", datasetStructureJSONObjCopy); - useGlobalStore.setState((state) => ({ - ...state, - datasetStructureJSONObj: datasetStructureJSONObjCopy, - })); -}; - export const setEntityList = (entityList) => { useGlobalStore.setState((state) => ({ ...state,