diff --git a/src/pyflask/curate/curate.py b/src/pyflask/curate/curate.py
index 4cf69ba46..5477c8909 100644
--- a/src/pyflask/curate/curate.py
+++ b/src/pyflask/curate/curate.py
@@ -3706,48 +3706,49 @@ def copytree(src, dst, symlinks=False, ignore=None):
def generate_manifest_file_data(dataset_structure_obj):
local_timezone = TZLOCAL()
- namespace_logger.info("Generating manifest file data for dataset structure:")
- namespace_logger.info(dataset_structure_obj)
-
- manifest_headers = [
- 'filename', 'timestamp', 'description', 'file type', 'entity', 'data modality',
- 'also in dataset', 'also in dataset path', 'data dictionary path',
- 'entity is transitive', 'Additional Metadata'
+
+ double_extensions = [
+ ".ome.tiff", ".ome.tif", ".ome.tf2,", ".ome.tf8", ".ome.btf", ".ome.xml",
+ ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz", ".bcl.gz"
]
- manifest_data = []
def get_name_extension(file_name):
- double_extensions = [".ome.tiff", ".ome.tif", ".ome.tf2,", ".ome.tf8", ".ome.btf", ".ome.xml",
- ".brukertiff.gz", ".mefd.gz", ".moberg.gz", ".nii.gz", ".mgh.gz", ".tar.gz",
- ".bcl.gz"]
for ext in double_extensions:
if file_name.endswith(ext):
+ # Extract the base extension before the double extension
base_ext = os.path.splitext(os.path.splitext(file_name)[0])[1]
return base_ext + ext
return os.path.splitext(file_name)[1]
def build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name):
- # Basic columns for a manifest entry
- file_manifest_template_data = [
- "/".join(ds_struct_path) + "/" + file_name if ds_struct_path else file_name,
- timestamp_entry,
- folder["files"][item].get("description", ""),
- get_name_extension(file_name),
- folder["files"][item].get("additional-metadata", "")
- ]
+ file_manifest_template_data = []
+ filename_entry = "/".join(ds_struct_path) + "/" + file_name if ds_struct_path else file_name
+ file_type_entry = get_name_extension(file_name)
+
+ if filename_entry[:1] == "/":
+ file_manifest_template_data.append(filename_entry[1:])
+ else:
+ file_manifest_template_data.append(filename_entry)
+
+ file_manifest_template_data.append(timestamp_entry)
+ file_manifest_template_data.append(folder["files"][item]["description"])
+ file_manifest_template_data.append(file_type_entry)
+ file_manifest_template_data.append(folder["files"][item]["additional-metadata"])
- # Add extra columns dynamically if they exist
if "extra_columns" in folder["files"][item]:
for key, value in folder["files"][item]["extra_columns"].items():
- if key not in manifest_headers:
- manifest_headers.append(key)
file_manifest_template_data.append(value)
+ if key not in hlf_data_array[0]:
+ hlf_data_array[0].append(key)
return file_manifest_template_data
- def recursive_folder_traversal(folder, ds_struct_path, is_pennsieve):
- # Traverse files in the folder
+ def recursive_folder_traversal(folder, hlf_data_array, ds_struct_path, is_pennsieve):
if "files" in folder:
+ standard_manifest_columns = ["filename", "timestamp", "description", "file type", "entity", "data modality", "also in dataset", "data dictionary path", "entity is transitive", "Additional Metadata"]
+ if not hlf_data_array:
+ hlf_data_array.append(standard_manifest_columns)
+
for item in folder["files"]:
if item in ["manifest.xlsx", "manifest.csv"]:
continue
@@ -3761,28 +3762,33 @@ def recursive_folder_traversal(folder, ds_struct_path, is_pennsieve):
mtime = pathlib.Path(local_path_to_file).stat().st_mtime
timestamp_entry = datetime.fromtimestamp(mtime, tz=local_timezone).isoformat().replace(".", ",").replace("+00:00", "Z")
- manifest_data.append(build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name))
+ hlf_data_array.append(build_file_entry(item, folder, ds_struct_path, timestamp_entry, file_name))
- # Recursively traverse subfolders
if "folders" in folder:
- for subfolder_name, subfolder_content in folder["folders"].items():
- ds_struct_path.append(subfolder_name)
- recursive_folder_traversal(subfolder_content, ds_struct_path, is_pennsieve)
+ for item in folder["folders"]:
+ ds_struct_path.append(item)
+ recursive_folder_traversal(folder["folders"][item], hlf_data_array, ds_struct_path, is_pennsieve)
ds_struct_path.pop()
- # Begin recursive traversal from the top-level folders
- for high_level_folder, folder_content in dataset_structure_obj["folders"].items():
- is_pennsieve = "bfpath" in folder_content
- recursive_folder_traversal(folder_content, [high_level_folder], is_pennsieve)
+ hlf_manifest_data = {}
+
+ namespace_logger.info("Generating manifest file data")
+ namespace_logger.info(dataset_structure_obj)
+
+ for high_level_folder in dataset_structure_obj["folders"]:
+ hlf_data_array = []
+ relative_structure_path = []
+
+ is_pennsieve = "bfpath" in dataset_structure_obj["folders"][high_level_folder]
+ recursive_folder_traversal(dataset_structure_obj["folders"][high_level_folder], hlf_data_array, relative_structure_path, is_pennsieve)
- namespace_logger.info("Generated manifest data:")
- namespace_logger.info(manifest_data)
+ hlf_manifest_data[high_level_folder] = hlf_data_array
- return manifest_headers, manifest_data
+ return hlf_manifest_data
def handle_duplicate_package_name_error(e, soda_json_structure):
if "if-existing-files" in soda_json_structure["generate-dataset"] and (soda_json_structure["generate-dataset"]["if-existing-files"] == "create-duplicate") and (e.response.text== '{"type":"BadRequest","message":"package name must be unique","code":400}'):
return
- raise e
\ No newline at end of file
+ raise e
diff --git a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx
index 57e3da380..dd2040582 100644
--- a/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx
+++ b/src/renderer/src/components/shared/DatasetTreeViewRenderer/index.jsx
@@ -1,5 +1,5 @@
import { useState } from "react";
-import { Collapse, Text, Stack, UnstyledButton } from "@mantine/core";
+import { Collapse, Text, Stack, UnstyledButton, TextInput } from "@mantine/core";
import { useHover } from "@mantine/hooks";
import {
IconFolder,
@@ -16,8 +16,10 @@ import {
IconFileTypeXls,
IconFileTypeXml,
IconFileTypeZip,
+ IconSearch,
} from "@tabler/icons-react";
-import { getEntityForRelativePath } from "../../../stores/slices/manifestEntitySelectorSlice";
+import useGlobalStore from "../../../stores/globalStore";
+import { setDatasetstructureSearchFilter } from "../../../stores/slices/datasetTreeViewSlice";
// Constants
const FOLDER_ICON_COLOR = "#ADD8E6";
@@ -42,39 +44,42 @@ const fileIconMap = {
jp2: ,
};
+// Retrieve icon based on file extension
const getFileTypeIcon = (fileName) => {
const extension = fileName.split(".").pop().toLowerCase();
return fileIconMap[extension] || ;
};
// FileItem component
-const FileItem = ({ name, content, onFileClick, getFileBackgroundColor }) => {
- const fileBackgroundColor = getFileBackgroundColor(content.relativePath);
- return (
-
onFileClick(name, content)}
- >
- {getFileTypeIcon(name)}
- {name}
-
- );
-};
+const FileItem = ({ name, content, onFileClick, getFileBackgroundColor }) => (
+ onFileClick(name, content)}
+ >
+ {getFileTypeIcon(name)}
+ {name}
+
+);
// FolderItem component
-const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgroundColor }) => {
+const FolderItem = ({
+ name,
+ content,
+ onFolderClick,
+ onFileClick,
+ getFileBackgroundColor,
+ searchFilter,
+}) => {
const [isOpen, setIsOpen] = useState(false);
const { hovered, ref } = useHover();
- const toggleFolder = () => {
- setIsOpen((prev) => !prev);
- };
+ const toggleFolder = () => setIsOpen((prev) => !prev);
return (
@@ -100,22 +105,23 @@ const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgrou
{name}
-
- {Object.keys(content.folders || {}).map((folderName) => (
+
+ {content.filteredFolders?.map(([folderName, folderContent]) => (
))}
- {Object.keys(content.files || {}).map((fileName) => (
+ {content.filteredFiles?.map(([fileName, fileContent]) => (
@@ -125,35 +131,74 @@ const FolderItem = ({ name, content, onFolderClick, onFileClick, getFileBackgrou
);
};
+// Recursive function to filter folders and files, marking parent folders if a child matches
+const filterStructure = (structure, searchFilter) => {
+ const lowerCaseFilter = searchFilter.toLowerCase();
+
+ const folders = Object.entries(structure.folders || {}).reduce((acc, [name, content]) => {
+ const filteredContent = filterStructure(content, searchFilter);
+ if (filteredContent || name.toLowerCase().includes(lowerCaseFilter)) {
+ acc[name] = { ...content, ...filteredContent };
+ }
+ return acc;
+ }, {});
+
+ const files = Object.entries(structure.files || {}).filter(([name]) =>
+ name.toLowerCase().includes(lowerCaseFilter)
+ );
+
+ if (Object.keys(folders).length || files.length) {
+ return { filteredFolders: Object.entries(folders), filteredFiles: files };
+ }
+ return null;
+};
+
// Main component
const DatasetTreeView = ({
datasetStructure,
onFolderClick,
onFileClick,
getFileBackgroundColor,
-}) =>
- !datasetStructure?.folders && !datasetStructure?.files ? null : (
+}) => {
+ const datasetStructureSearchFilter = useGlobalStore(
+ (state) => state.datasetStructureSearchFilter
+ );
+
+ const handleSearchChange = (event) => setDatasetstructureSearchFilter(event.target.value);
+
+ const filteredStructure = filterStructure(datasetStructure, datasetStructureSearchFilter);
+
+ return (
- {Object.keys(datasetStructure.files || {}).map((fileName) => (
-
- ))}
- {Object.keys(datasetStructure.folders || {}).map((folderName) => (
+ }
+ />
+ {filteredStructure?.filteredFolders?.map(([folderName, folderContent]) => (
+ ))}
+ {filteredStructure?.filteredFiles?.map(([fileName, fileContent]) => (
+
))}
);
+};
export default DatasetTreeView;
diff --git a/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js b/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js
index 7b346aef4..07eaad262 100644
--- a/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js
+++ b/src/renderer/src/scripts/guided-mode/guided-curate-dataset.js
@@ -44,11 +44,8 @@ import {
setGuidedDatasetName,
setGuidedDatasetSubtitle,
} from "../../stores/slices/guidedModeSlice";
-import {
- setDatasetStructureJSONObj,
- setEntityList,
- setEntityType,
-} from "../../stores/slices/manifestEntitySelectorSlice";
+import { setEntityList, setEntityType } from "../../stores/slices/manifestEntitySelectorSlice";
+import { setTreeViewDatasetStructure } from "../../stores/slices/datasetTreeViewSlice";
import "bootstrap-select";
import Cropper from "cropperjs";
@@ -5467,7 +5464,7 @@ window.openPage = async (targetPageID) => {
if (targetPageID === "guided-manifest-subject-entity-selector-tab") {
//
setEntityList(window.getExistingSubjectNames());
- setDatasetStructureJSONObj(window.datasetStructureJSONObj);
+ setTreeViewDatasetStructure(window.datasetStructureJSONObj);
setEntityType("subjects");
}
diff --git a/src/renderer/src/stores/globalStore.js b/src/renderer/src/stores/globalStore.js
index c88ed057c..ec514ad5e 100644
--- a/src/renderer/src/stores/globalStore.js
+++ b/src/renderer/src/stores/globalStore.js
@@ -6,6 +6,7 @@ import { dropDownSlice } from "./slices/dropDownSlice";
import { singleColumnTableSlice } from "./slices/tableRowSlice";
import { backgroundServicesSlice } from "./slices/backgroundServicesSlice";
import { manifestEntitySelectorSlice } from "./slices/manifestEntitySelectorSlice";
+import { datasetTreeViewSlice } from "./slices/datasetTreeViewSlice";
const useGlobalStore = create(
immer((...a) => ({
@@ -15,6 +16,7 @@ const useGlobalStore = create(
...singleColumnTableSlice(...a),
...backgroundServicesSlice(...a),
...manifestEntitySelectorSlice(...a),
+ ...datasetTreeViewSlice(...a),
}))
);
diff --git a/src/renderer/src/stores/slices/datasetTreeViewSlice.js b/src/renderer/src/stores/slices/datasetTreeViewSlice.js
new file mode 100644
index 000000000..308c68499
--- /dev/null
+++ b/src/renderer/src/stores/slices/datasetTreeViewSlice.js
@@ -0,0 +1,43 @@
+import useGlobalStore from "../globalStore";
+
+const initialState = {
+ datasetStructureJSONObj: null,
+ datasetStructureSearchFilter: "",
+};
+
+export const datasetTreeViewSlice = (set) => ({
+ ...initialState,
+});
+
+export const setDatasetstructureSearchFilter = (datasetStructureSearchFilter) => {
+ useGlobalStore.setState((state) => ({
+ ...state,
+ datasetStructureSearchFilter,
+ }));
+};
+
+export const setTreeViewDatasetStructure = (datasetStructureJSONObj) => {
+ // Create a deep copy of the object so that the original object is not mutated
+ const datasetStructureJSONObjCopy = JSON.parse(JSON.stringify(datasetStructureJSONObj));
+ // Recursively add the relative paths to the datasetStructureJSONObjCopy object
+ // for use in the manifest entity selector component
+ const addRelativePaths = (obj, path = "") => {
+ const objFolders = Object.keys(obj.folders || {});
+ const objFiles = Object.keys(obj.files || {});
+ objFolders.forEach((folder) => {
+ const folderPath = path ? `${path}/${folder}` : folder;
+ obj.folders[folder].relativePath = folderPath;
+ addRelativePaths(obj.folders[folder], folderPath);
+ });
+ objFiles.forEach((file) => {
+ const filePath = path ? `${path}/${file}` : file;
+ obj.files[file].relativePath = filePath;
+ });
+ };
+ addRelativePaths(datasetStructureJSONObjCopy);
+ console.log("Dataset structure JSON object:", datasetStructureJSONObjCopy);
+ useGlobalStore.setState((state) => ({
+ ...state,
+ datasetStructureJSONObj: datasetStructureJSONObjCopy,
+ }));
+};
diff --git a/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js b/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js
index aec684590..98ac82a3f 100644
--- a/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js
+++ b/src/renderer/src/stores/slices/manifestEntitySelectorSlice.js
@@ -21,32 +21,6 @@ export const resetManifestEntitySelectorState = () => {
);
};
-export const setDatasetStructureJSONObj = (datasetStructureJSONObj) => {
- // Create a deep copy of the object so that the original object is not mutated
- const datasetStructureJSONObjCopy = JSON.parse(JSON.stringify(datasetStructureJSONObj));
- // Recursively add the relative paths to the datasetStructureJSONObjCopy object
- // for use in the manifest entity selector component
- const addRelativePaths = (obj, path = "") => {
- const objFolders = Object.keys(obj.folders || {});
- const objFiles = Object.keys(obj.files || {});
- objFolders.forEach((folder) => {
- const folderPath = path ? `${path}/${folder}` : folder;
- obj.folders[folder].relativePath = folderPath;
- addRelativePaths(obj.folders[folder], folderPath);
- });
- objFiles.forEach((file) => {
- const filePath = path ? `${path}/${file}` : file;
- obj.files[file].relativePath = filePath;
- });
- };
- addRelativePaths(datasetStructureJSONObjCopy);
- console.log("Dataset structure JSON object:", datasetStructureJSONObjCopy);
- useGlobalStore.setState((state) => ({
- ...state,
- datasetStructureJSONObj: datasetStructureJSONObjCopy,
- }));
-};
-
export const setEntityList = (entityList) => {
useGlobalStore.setState((state) => ({
...state,