Skip to content

Commit

Permalink
Merge branch 'staging' into pre-staging
Browse files Browse the repository at this point in the history
  • Loading branch information
@aaronm-2112
aaronm-2112 authored Oct 9, 2024
2 parents 910899b + 1c40d13 commit e958a83
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 150 deletions.
4 changes: 2 additions & 2 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 7 additions & 4 deletions src/renderer/src/scripts/organize-dataset/curate-functions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,15 +502,18 @@ window.uploadDatasetClickHandler = async (ev) => {
};

window.handleLocalDatasetImport = async (path) => {
console.log("Importing local dataset from path:", path);
const list = await getFilesAndFolders(path);
console.log("List of files and folders:", list);
const currentFileExplorerPath = window.organizeDSglobalPath.value.trim();
const buildDatasetStructure = await window.buildDatasetStructureJsonFromImportedData(
console.log("Current file explorer path:", currentFileExplorerPath);
const builtDatasetStructure = await window.buildDatasetStructureJsonFromImportedData(
list.folders,
currentFileExplorerPath,
"dataset_root/", // Use dataset_root as the root folder since we are importing the root in this case
true
);

window.sodaJSONObj["dataset-structure"] = buildDatasetStructure[0];
window.sodaJSONObj["dataset-structure"] = builtDatasetStructure[0];
window.sodaJSONObj["metadata-files"] = list.files;
const forbiddenFileNames = [];
const problematicFiles = [];
Expand Down Expand Up @@ -604,7 +607,7 @@ window.handleLocalDatasetImport = async (path) => {
window.sodaJSONObj = await window.addManifestDetailsToDatasetStructure(
window.sodaJSONObj,
list.manifestFiles,
buildDatasetStructure
builtDatasetStructure
);

return true;
Expand Down
251 changes: 110 additions & 141 deletions src/renderer/src/scripts/others/renderer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,8 @@ const initializeSODARenderer = async () => {
// Set the app version in the sidebar for the user to see
setSidebarAppVersion();

await warnUserIfBetaVersionAndDntNotEnabled();

// Launch announcements if the user has not seen them yet
await launchAnnouncements();

Expand Down Expand Up @@ -978,6 +980,22 @@ const setSidebarAppVersion = async () => {
version.innerText = currentAppVersion;
};

// Warn the user if they are on a beta version of the app
const warnUserIfBetaVersionAndDntNotEnabled = async () => {
try {
let currentAppVersion = await window.electron.ipcRenderer.invoke("app-version");
const dntFilePath = window.path.join(homeDirectory, ".soda-config", "dnt.soda");
if (currentAppVersion.includes("beta") && !window.fs.existsSync(dntFilePath)) {
await swalShowInfo(
"You are on a beta version of SODA",
"When you are finished using this special version of SODA, please download the latest stable version<a href='https://docs.sodaforsparc.io/' target='_blank'> by clicking here</a>"
);
}
} catch (err) {
console.error("Error determing if beta pop up should exist:", err);
}
};

// Check app version on current app and display in the side bar
window.electron.ipcRenderer.on("app_version", (event, arg) => {
window.electron.ipcRenderer.removeAllListeners("app_version");
Expand Down Expand Up @@ -4215,13 +4233,38 @@ window.electron.ipcRenderer.on(
/* ################################################################################## */
/* ################################################################################## */

const getNestedObjectsFromDatasetStructureByPath = (datasetStructure, path) => {
const pathArray = window.path.split("/").filter((item) => item !== "");
let currentObject = datasetStructure;
for (const item of pathArray) {
currentObject = currentObject["folders"][item];
// Function to check if a folder on the user's machine is empty or not
const localFolderPathAndSubFoldersHaveNoFiles = (localFolderPath) => {
try {
const items = window.fs.readdirSync(localFolderPath);
for (const item of items) {
const itemPath = window.path.join(localFolderPath, item);
const statsObj = window.fs.statSync(itemPath);

// Check if the item is a file
if (statsObj.isFile) {
// If a file with size > 0 is found, the folder is not considered empty
if (window.fs.fileSizeSync(itemPath) > 0) {
console.log("File is not empty");
return false; // Found a non-empty file
}
}
// Check if the item is a directory
else if (statsObj.isDirectory) {
// Recursively check the subdirectory
const isEmpty = localFolderPathAndSubFoldersHaveNoFiles(itemPath);
if (!isEmpty) {
return false; // Found a non-empty subdirectory
}
}
}

// If no files with size > 0 are found, the folder is considered empty
return true;
} catch (error) {
console.error(`Error reading folder: ${error.message}`);
return false; // Return false on error as we couldn't verify the folder
}
return currentObject;
};

const removeHiddenFilesFromDatasetStructure = (datasetStructure) => {
Expand Down Expand Up @@ -4388,47 +4431,56 @@ window.buildDatasetStructureJsonFromImportedData = async (
const problematicFileNames = [];
const datasetStructure = {};
const hiddenItems = [];
const emptyFolders = [];
const emptyFiles = [];

showFileImportLoadingSweetAlert(500);

// Function to traverse and build JSON structure
const traverseAndBuildJson = async (pathToExplore, currentStructure, currentStructurePath) => {
// Initialize the current structure if it does not exist
currentStructure["folders"] = currentStructure["folders"] || {};
currentStructure["files"] = currentStructure["files"] || {};

try {
if (await window.fs.isDirectory(pathToExplore)) {
const folderName = window.path.basename(pathToExplore);
const folderIsEmpty = localFolderPathAndSubFoldersHaveNoFiles(pathToExplore);
// If the folder is not empty, recursively traverse the folder and build the JSON structure

const folderNameIsValid = window.evaluateStringAgainstSdsRequirements(
folderName,
"folder-and-file-name-is-valid"
);
if (!folderNameIsValid) {
problematicFolderNames.push(`${currentStructurePath}${folderName}`);
}
if (folderIsEmpty) {
emptyFolders.push(pathToExplore);
} else {
const folderName = window.path.basename(pathToExplore);
const folderNameIsValid = window.evaluateStringAgainstSdsRequirements(
folderName,
"folder-and-file-name-is-valid"
);
if (!folderNameIsValid) {
problematicFolderNames.push(`${currentStructurePath}${folderName}`);
}

// Add the folder to the JSON structure
currentStructure["folders"][folderName] = {
path: pathToExplore,
type: "local",
files: {},
folders: {},
action: ["new"],
};
// Add the folder to the JSON structure
currentStructure["folders"][folderName] = {
path: pathToExplore,
type: "local",
files: {},
folders: {},
action: ["new"],
};

// Recursively traverse the folder and build the JSON structure
const folderContents = await window.fs.readdir(pathToExplore);
await Promise.all(
folderContents.map(async (item) => {
const itemPath = window.path.join(pathToExplore, item);
await traverseAndBuildJson(
itemPath,
currentStructure["folders"][folderName],
`${currentStructurePath}${folderName}/`
);
})
);
// Recursively traverse the folder and build the JSON structure
const folderContents = await window.fs.readdir(pathToExplore);
await Promise.all(
folderContents.map(async (item) => {
const itemPath = window.path.join(pathToExplore, item);
await traverseAndBuildJson(
itemPath,
currentStructure["folders"][folderName],
`${currentStructurePath}${folderName}/`
);
})
);
}
} else if (await window.fs.isFile(pathToExplore)) {
const fileName = window.path.basename(pathToExplore);
const fileExtension = window.path.extname(pathToExplore);
Expand All @@ -4444,8 +4496,13 @@ window.buildDatasetStructureJsonFromImportedData = async (
"file-is-in-forbidden-files-list"
);

const fileIsEmpty = window.fs.fileSizeSync(pathToExplore) === 0;

// If the file is not empty or forbidden, add it to the current dataset structure
if (fileIsInForbiddenFilesList) {
forbiddenFileNames.push(fileObject);
} else if (fileIsEmpty) {
emptyFiles.push(fileObject);
} else {
// Check if the file name has any characters that do not comply with SPARC naming requirements
const fileNameIsValid = window.evaluateStringAgainstSdsRequirements(
Expand Down Expand Up @@ -4508,6 +4565,24 @@ window.buildDatasetStructureJsonFromImportedData = async (
);
}

if (emptyFolders.length > 0) {
await swalFileListSingleAction(
emptyFolders,
"Empty folders detected",
"The following folders are empty or contain only other empty folders or files (0 KB). These will not be imported into SODA:",
false
);
}

if (emptyFiles.length > 0) {
await swalFileListSingleAction(
emptyFiles.map((file) => file.relativePath),
"Empty files detected",
"The files listed below are empty (0 KB) and will not be imported into SODA:",
false
);
}

if (problematicFolderNames.length > 0) {
const userResponse = await swalFileListTripleAction(
problematicFolderNames,
Expand Down Expand Up @@ -4692,29 +4767,6 @@ const mergeLocalAndRemoteDatasetStructure = async (
}
};

const checkForDuplicateFolderAndFileNames = async (importedFolders, itemsAtPath) => {
const duplicateFolderNames = [];
const duplicateFileNames = [];

const checkForDuplicateNames = async (importedFolders, itemsAtPath) => {
const currentFoldersAtPath = Object.keys(itemsAtPath.folders);
const currentFilesAtPath = Object.keys(itemsAtPath.files);
fg;
for (const folder of importedFolders) {
folderName = window.path.basename(folder);
if (currentFoldersAtPath.includes(folderName)) {
duplicateFolderNames.push(folderName);
}
const folderContents = await fs.readdir(folder);
}
for (const fileName of importedFiles) {
if (currentFilesAtPath.includes(fileName)) {
duplicateFileNames.push(fileName);
}
}
};
};

const addDataArrayToDatasetStructureAtPath = async (importedData) => {
// If no data was imported ()
const numberOfItemsToImport = importedData.length;
Expand Down Expand Up @@ -6375,47 +6427,6 @@ const divGenerateProgressBar = document.getElementById("div-new-curate-meter-pro
const generateProgressBar = document.getElementById("progress-bar-new-curate");
var progressStatus = document.getElementById("para-new-curate-progress-bar-status");

const checkEmptyFilesAndFolders = async (sodaJSONObj) => {
let emptyFilesFoldersResponse;
try {
emptyFilesFoldersResponse = await client.post(
`/curate_datasets/empty_files_and_folders`,
{
soda_json_structure: sodaJSONObj,
},
{ timeout: 0 }
);
} catch (error) {
clientError(error);
let emessage = userErrorMessage(error);
document.getElementById("para-new-curate-progress-bar-error-status").innerHTML =
"<span style='color: red;'> Error: " + emessage + "</span>";
document.getElementById("para-please-wait-new-curate").innerHTML = "";
$("#sidebarCollapse").prop("disabled", false);
return;
}

let { data } = emptyFilesFoldersResponse;

window.log.info("Continue with curate");
let errorMessage = "";
let error_files = data["empty_files"];
//bring duplicate outside
let error_folders = data["empty_folders"];

if (error_files.length > 0) {
const errorFilesHtml = generateHtmlListFromArray(error_files);
errorMessage += errorFilesHtml;
}

if (error_folders.length > 0) {
const errorFoldersHtml = generateHtmlListFromArray(error_folders);
errorMessage += errorFoldersHtml;
}

return errorMessage;
};

window.setSodaJSONStartingPoint = (sodaJSONObj) => {
if (window.sodaJSONObj["starting-point"]["type"] === "local") {
window.sodaJSONObj["starting-point"]["type"] = "new";
Expand Down Expand Up @@ -6551,49 +6562,7 @@ const preGenerateSetup = async (e, elementContext) => {

deleteTreeviewFiles(sodaJSONObj);

let errorMessage = await checkEmptyFilesAndFolders(sodaJSONObj);

if (errorMessage) {
errorMessage += "Would you like to continue?";
errorMessage = "<div style='text-align: left'>" + errorMessage + "</div>";
Swal.fire({
icon: "warning",
html: errorMessage,
showCancelButton: true,
cancelButtonText: "No, I want to review my files",
focusCancel: true,
confirmButtonText: "Yes, Continue",
backdrop: "rgba(0,0,0, 0.4)",
reverseButtons: window.reverseSwalButtons,
heightAuto: false,
showClass: {
popup: "animate__animated animate__zoomIn animate__faster",
},
hideClass: {
popup: "animate__animated animate__zoomOut animate__faster",
},
didOpen: () => {
document.getElementById("swal2-html-container").style.maxHeight = "19rem";
document.getElementById("swal2-html-container").style.overflowY = "auto";
},
}).then((result) => {
if (result.isConfirmed) {
initiate_generate(e);
} else {
$("#sidebarCollapse").prop("disabled", false);
$("#please-wait-new-curate-div").show();
document.getElementById("para-please-wait-new-curate").innerHTML = "Return to make changes";
document.getElementById("div-generate-comeback").style.display = "flex";
document.getElementById("guided_mode_view").style.pointerEvents = "";
// Allow documentation view to be clicked again
document.getElementById("documentation-view").style.pointerEvents = "";
// Allow contact us view to be clicked again
document.getElementById("contact-us-view").style.pointerEvents = "";
}
});
} else {
initiate_generate(e);
}
initiate_generate(e);
};

document.getElementById("button-generate").addEventListener("click", async function (e) {
Expand Down
Loading

0 comments on commit e958a83

Please sign in to comment.