From c139b4b09469805525f69652568c9f2f4e2fa55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Wollse=CC=81n?= Date: Mon, 6 Jan 2020 21:00:25 +0200 Subject: [PATCH] "Refresh surveys and combined topline listings" menu action working as expected --- docs/README.md | 10 +- src/gsheetsData/hardcodedConstants.ts | 57 ++- src/menuActions/common.ts | 50 ++- ...efreshSurveysAndCombinedToplineListings.ts | 388 ++++++++++++++---- 4 files changed, 420 insertions(+), 85 deletions(-) diff --git a/docs/README.md b/docs/README.md index e82e6e0..e4b117d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,17 +5,17 @@ ### Functions -* [menuRefreshCombinedToplineListing](README.md#menurefreshcombinedtoplinelisting) +* [menuRefreshSurveysAndCombinedToplineListings](README.md#menurefreshsurveysandcombinedtoplinelistings) ## Functions -### menuRefreshCombinedToplineListing +### menuRefreshSurveysAndCombinedToplineListings -▸ **menuRefreshCombinedToplineListing**(): *void* +▸ **menuRefreshSurveysAndCombinedToplineListings**(): *void* -*Defined in [menuActions/menuRefreshCombinedToplineListing.ts:27](https://github.com/Gapminder/gapminder-igno-survey-process-scripts/blob/v0.0.0/src/menuActions/menuRefreshCombinedToplineListing.ts#L27)* +*Defined in [menuActions/menuRefreshSurveysAndCombinedToplineListings.ts:37](https://github.com/Gapminder/gapminder-igno-survey-process-scripts/blob/v0.0.0/src/menuActions/menuRefreshSurveysAndCombinedToplineListings.ts#L37)* -Menu item action for "Gapminder Igno Survey Process -> Refresh combined topline listing" +Menu item action for "Gapminder Igno Survey Process -> Refresh surveys and combined topline listings" Notes: - Creates the `surveys` and `topline_combo` worksheets if they don't exist diff --git a/src/gsheetsData/hardcodedConstants.ts b/src/gsheetsData/hardcodedConstants.ts index 1122f24..706959d 100644 --- a/src/gsheetsData/hardcodedConstants.ts +++ b/src/gsheetsData/hardcodedConstants.ts @@ -22,7 +22,12 @@ export const gsResultsFolderName = "gs_results"; /** * @hidden */ -export const surveysSheetHeaders = ["survey_name", "file_name"]; +export const surveysSheetHeaders = [ + "survey_name", + "file_name", + "link_if_found_in_gs_results_folder", + "number_of_rows_in_topline_combo" +]; /** * @hidden @@ -36,3 +41,53 @@ export const combinedToplineSheetHeaders = [ "Metadata", "Weighted by" ]; + +/** + * @hidden + */ +export const surveysSheetValueRowToSurveyEntry = (surveysSheetRow: any[]) => { + return { + survey_name: surveysSheetRow[0], + file_name: surveysSheetRow[1], + link_if_found_in_gs_results_folder: surveysSheetRow[2] + }; +}; + +/** + * @hidden + */ +export const surveyEntryToSurveysSheetValueRow = updatedSurveyEntry => [ + updatedSurveyEntry.survey_name, + updatedSurveyEntry.file_name, + updatedSurveyEntry.link_if_found_in_gs_results_folder +]; + +/** + * @hidden + */ +export const combinedToplineSheetValueRowToToplineEntry = ( + combinedToplineSheetRow: any[] +) => { + return { + survey_id: combinedToplineSheetRow[0], + question_number: combinedToplineSheetRow[1], + question_text: combinedToplineSheetRow[2], + answer: combinedToplineSheetRow[3], + answer_by_percent: combinedToplineSheetRow[4], + metadata: combinedToplineSheetRow[5], + weighted_by: combinedToplineSheetRow[6] + }; +}; + +/** + * @hidden + */ +export const toplineEntryToCombinedToplineSheetValueRow = toplineEntry => [ + toplineEntry.survey_id, + toplineEntry.question_number, + toplineEntry.question_text, + toplineEntry.answer, + toplineEntry.answer_by_percent, + toplineEntry.metadata, + toplineEntry.weighted_by +]; diff --git a/src/menuActions/common.ts b/src/menuActions/common.ts index f5374e8..c08ec66 100644 --- a/src/menuActions/common.ts +++ b/src/menuActions/common.ts @@ -54,16 +54,14 @@ export function assertCorrectLeftmostSheetColumnHeaders( const headerRow = sheetDataWithHeaderRow.slice(0, 1); const firstSheetHeaders = headerRow[0].slice(0, sheetHeaders.length); if (JSON.stringify(firstSheetHeaders) !== JSON.stringify(sheetHeaders)) { - SpreadsheetApp.getUi().alert( + throw new Error( `The first column sheetHeaders in '${sheetName}' must be ${JSON.stringify( sheetHeaders )} but are currently ${JSON.stringify( firstSheetHeaders )}. Please adjust and try again` ); - return false; } - return true; } /** @@ -121,3 +119,49 @@ export function addGsheetConvertedVersionOfExcelFileToFolder( const createdFile = DriveApp.getFileById(createdFileDriveObject.id); return createdFile; } + +/** + * @hidden + */ +export function ensuredColumnIndex(headers, header: string) { + const index = headers.indexOf(header); + if (index < 0) { + throw new Error(`Header not found: '${header}'`); + } + return index; +} + +/** + * @hidden + */ +export function getColumnValuesRange(sheet: Sheet, headers, header) { + const columnIndex = ensuredColumnIndex(headers, header); + return sheet.getRange(2, columnIndex + 1, sheet.getMaxRows() - 1, 1); +} + +/** + * @hidden + */ +export function arrayOfASingleValue(value, len): any[] { + const arr = []; + for (let i = 0; i < len; i++) { + arr.push(value); + } + return arr; +} + +/** + * From https://stackoverflow.com/a/43046408/682317 + * @hidden + */ +export function unique(ar) { + const j = {}; + + ar.forEach(v => { + j[v + "::" + typeof v] = v; + }); + + return Object.keys(j).map(v => { + return j[v]; + }); +} diff --git a/src/menuActions/menuRefreshSurveysAndCombinedToplineListings.ts b/src/menuActions/menuRefreshSurveysAndCombinedToplineListings.ts index df2f74c..434c286 100644 --- a/src/menuActions/menuRefreshSurveysAndCombinedToplineListings.ts +++ b/src/menuActions/menuRefreshSurveysAndCombinedToplineListings.ts @@ -1,21 +1,31 @@ +import difference from "lodash/difference"; +import intersection from "lodash/intersection"; +import union from "lodash/union"; import { combinedToplineSheetHeaders, combinedToplineSheetName, + combinedToplineSheetValueRowToToplineEntry, gsResultsFolderName, + surveyEntryToSurveysSheetValueRow, surveysSheetHeaders, - surveysSheetName + surveysSheetName, + surveysSheetValueRowToSurveyEntry, + toplineEntryToCombinedToplineSheetValueRow } from "../gsheetsData/hardcodedConstants"; import { addGsheetConvertedVersionOfExcelFileToFolder, adjustSheetRowsAndColumnsCount, + arrayOfASingleValue, assertCorrectLeftmostSheetColumnHeaders, createSheet, + getColumnValuesRange, getSheetDataIncludingHeaderRow, gsheetMimeType, xlsxMimeType } from "./common"; import Folder = GoogleAppsScript.Drive.Folder; import File = GoogleAppsScript.Drive.File; +import Sheet = GoogleAppsScript.Spreadsheet.Sheet; /** * Menu item action for "Gapminder Igno Survey Process -> Refresh surveys and combined topline listings" @@ -25,11 +35,23 @@ import File = GoogleAppsScript.Drive.File; * - Verifies that the first headers of the `surveys` and `topline_combo` worksheets are as expected */ export function menuRefreshSurveysAndCombinedToplineListings() { - refreshCombinedToplineListing(); - - SpreadsheetApp.getUi().alert( - "Refreshed the surveys and combined topline listings (based on files in the gs_results folder)." - ); + try { + refreshSurveysAndCombinedToplineListings(); + SpreadsheetApp.getUi().alert( + "Refreshed the surveys and combined topline listing (based on files in the gs_results folder)." + ); + } catch (e) { + // Ignore "Timed out waiting for user response" since it just means that we let the script run and went for coffee + if (e.message === "Timed out waiting for user response") { + return; + } + // Friendly error notice + SpreadsheetApp.getUi().alert( + "Encountered an issue: \n\n" + e.message + "\n\n" + e.stack + ); + // Also throw the error so that it turns up in the error log + throw e; + } return; } @@ -37,7 +59,11 @@ export function menuRefreshSurveysAndCombinedToplineListings() { /** * @hidden */ -function refreshCombinedToplineListing() { +function refreshSurveysAndCombinedToplineListings() { + /* tslint:disable:no-console */ + console.info(`Start of refreshSurveysAndCombinedToplineListings()`); + + console.info(`Fetching and verifying existing worksheet contents`); const activeSpreadsheet = SpreadsheetApp.getActiveSpreadsheet(); let surveysSheet = activeSpreadsheet.getSheetByName(surveysSheetName); if (surveysSheet === null) { @@ -59,80 +85,57 @@ function refreshCombinedToplineListing() { ); } - const surveysSheetDataIncludingHeaderRow = getSheetDataIncludingHeaderRow( + const surveysSheetValuesIncludingHeaderRow = getSheetDataIncludingHeaderRow( surveysSheet, surveysSheetHeaders ); - const combinedToplineSheetDataIncludingHeaderRow = getSheetDataIncludingHeaderRow( + const combinedToplineSheetValuesIncludingHeaderRow = getSheetDataIncludingHeaderRow( combinedToplineSheet, combinedToplineSheetHeaders ); // Verify that the first headers are as expected - if ( - !assertCorrectLeftmostSheetColumnHeaders( - surveysSheetHeaders, - surveysSheetName, - surveysSheetDataIncludingHeaderRow - ) - ) { - return; - } - if ( - !assertCorrectLeftmostSheetColumnHeaders( - combinedToplineSheetHeaders, - combinedToplineSheetName, - combinedToplineSheetDataIncludingHeaderRow - ) - ) { - return; - } + assertCorrectLeftmostSheetColumnHeaders( + surveysSheetHeaders, + surveysSheetName, + surveysSheetValuesIncludingHeaderRow + ); + assertCorrectLeftmostSheetColumnHeaders( + combinedToplineSheetHeaders, + combinedToplineSheetName, + combinedToplineSheetValuesIncludingHeaderRow + ); - // Read files in the gs_results folder + // Read files in the (first found) folder called "gs_results", ensuring that there is a Gsheet version of each uploaded Excel file + console.info( + `Reading files in the folder called "gs_results", ensuring that there is a Gsheet version of each uploaded Excel file` + ); const folders = DriveApp.getFoldersByName(gsResultsFolderName); if (!folders.hasNext()) { throw Error(`No folder found called "${gsResultsFolderName}"`); } - - // Use the first matching folder const gsResultsFolder = folders.next(); - - // Load files, ensuring that there is a Gsheet version of each uploaded Excel file const filesByMimeType = ensureGsheetVersionsOfEachExcelFile(gsResultsFolder); + const gsResultsFolderGsheetFiles = filesByMimeType[gsheetMimeType]; - console.log(filesByMimeType); - - const combinedToplineSheetData = []; - - refreshCombinedToplineListingSheet( - combinedToplineSheet, - combinedToplineSheetData - ); - - // Read back the updated combined topline sheet data - const updatedCombinedToplineSheetDataIncludingHeaderRow = getSheetDataIncludingHeaderRow( - combinedToplineSheet, - combinedToplineSheetHeaders - ); - - // Limit the amount of rows of the surveys spreadsheet to the amount of surveys + a few extras for adding new ones - const surveysSheetData = surveysSheetDataIncludingHeaderRow.slice(1); - adjustSheetRowsAndColumnsCount( + console.info(`Refreshing survey listing`); + const { updatedSurveysSheetValues } = refreshSurveysSheetListing( surveysSheet, - surveysSheetData.length + 1 + 3, - surveysSheetDataIncludingHeaderRow[0].length + surveysSheetValuesIncludingHeaderRow, + gsResultsFolderGsheetFiles ); - // Limit the amount of rows of the surveys spreadsheet to the amount of entries - const updatedCombinedToplineSheetData = updatedCombinedToplineSheetDataIncludingHeaderRow.slice( - 1 - ); - adjustSheetRowsAndColumnsCount( + console.info(`Refreshing combined topline listing`); + refreshCombinedToplineSheetListing( + updatedSurveysSheetValues, combinedToplineSheet, - updatedCombinedToplineSheetData.length + 1, - updatedCombinedToplineSheetDataIncludingHeaderRow[0].length + combinedToplineSheetValuesIncludingHeaderRow, + gsResultsFolderGsheetFiles ); + + console.info(`End of refreshSurveysAndCombinedToplineListings()`); + /* tslint:disable:no-console */ } /** @@ -151,9 +154,13 @@ function ensureGsheetVersionsOfEachExcelFile(gsResultsFolder: Folder) { filesByMimeType[file.getMimeType()].push(file); } filesByMimeType[xlsxMimeType].map(excelFile => { - const targetFileName = excelFile.getName().replace(/.xlsx?/, ""); + // Note: Trimming at the end to ensure that the converted file name does not start or end with spaces + const targetFileName = excelFile + .getName() + .replace(/.xlsx?/, "") + .trim(); const existingGsheetFiles = filesByMimeType[gsheetMimeType].filter( - (gsheetFile: File) => gsheetFile.getName() === targetFileName + (gsheetFile: File) => gsheetFile.getName().trim() === targetFileName ); if (existingGsheetFiles.length === 0) { /* tslint:disable:no-console */ @@ -178,24 +185,253 @@ function ensureGsheetVersionsOfEachExcelFile(gsResultsFolder: Folder) { /** * @hidden */ -function refreshCombinedToplineListingSheet( - sheet, - gsResultsFolderListing: any[] +function refreshSurveysSheetListing( + surveysSheet: Sheet, + surveysSheetValuesIncludingHeaderRow: any[][], + gsResultsFolderGsheetFiles: File[] ) { - const gsResultsFolderListingValues = gsResultsFolderListing - .map((row: any) => { - return [row, row, row]; - }) - .reduce((a, b) => a.concat(b), []); // flattens the result - const combinedToplineValues = [combinedToplineSheetHeaders].concat( - gsResultsFolderListingValues - ); - sheet + const surveysSheetValues = surveysSheetValuesIncludingHeaderRow.slice(1); + + // Update existing entries + const existingSurveyEntries = surveysSheetValues.map( + surveysSheetValueRowToSurveyEntry + ); + const fileNamesEncounteredInExistingEntries = []; + const updatedSurveyEntries = existingSurveyEntries.map( + existingSurveyEntry => { + if (existingSurveyEntry.file_name === "") { + return existingSurveyEntry; + } + let link_if_found_in_gs_results_folder = + "(Not found in gs_results folder)"; + const matchingGsResultsFolderGsheetFiles = gsResultsFolderGsheetFiles.filter( + (gsResultsFolderGsheetFile: File) => + existingSurveyEntry.file_name === gsResultsFolderGsheetFile.getName() + ); + if (matchingGsResultsFolderGsheetFiles.length > 0) { + const matchingGsResultsFolderGsheetFile = + matchingGsResultsFolderGsheetFiles[0]; + fileNamesEncounteredInExistingEntries.push( + matchingGsResultsFolderGsheetFile.getName() + ); + link_if_found_in_gs_results_folder = matchingGsResultsFolderGsheetFile.getUrl(); + } + return { + ...existingSurveyEntry, + link_if_found_in_gs_results_folder + }; + } + ); + + // Add previously unencountered gsheet files to the bottom of the surveys sheet listing + const gsResultsFolderGsheetFileNames = gsResultsFolderGsheetFiles.map( + gsResultsFolderGsheetFile => gsResultsFolderGsheetFile.getName() + ); + const newGsResultsFolderGsheetFileNames = difference( + gsResultsFolderGsheetFileNames, + fileNamesEncounteredInExistingEntries + ); + const newGsResultsFolderGsheetFiles = gsResultsFolderGsheetFiles.filter( + (gsResultsFolderGsheetFile: File) => + newGsResultsFolderGsheetFileNames.includes( + gsResultsFolderGsheetFile.getName() + ) + ); + if (newGsResultsFolderGsheetFiles.length > 0) { + newGsResultsFolderGsheetFiles.map((gsResultsFolderGsheetFile: File) => { + updatedSurveyEntries.push({ + file_name: gsResultsFolderGsheetFile.getName(), + link_if_found_in_gs_results_folder: gsResultsFolderGsheetFile.getUrl(), + survey_name: "" + }); + }); + } + + // Save the updated values to the surveys worksheet + const updatedSurveysSheetValues = updatedSurveyEntries.map( + surveyEntryToSurveysSheetValueRow + ); + surveysSheet .getRange( + 2, 1, + updatedSurveysSheetValues.length, + updatedSurveysSheetValues[0].length + ) + .setValues(updatedSurveysSheetValues); + + // Limit the amount of rows of the surveys spreadsheet to the amount of surveys + a few extras for adding new ones + const extraBlankRows = 3; + adjustSheetRowsAndColumnsCount( + surveysSheet, + updatedSurveysSheetValues.length + extraBlankRows + 1, + surveysSheetValuesIncludingHeaderRow[0].length + ); + + // Fill the "number_of_rows_in_topline_combo" column with a formula + const catalogStatusRange = getColumnValuesRange( + surveysSheet, + surveysSheetHeaders, + "number_of_rows_in_topline_combo" + ); + const sheetValueRowsCount = surveysSheet.getMaxRows() - 1; + const formulas = arrayOfASingleValue( + '=IF(INDIRECT("R[0]C[-2]", FALSE)="","",COUNTIF(topline_combo!$A$2:$A, SUBSTITUTE(INDIRECT("R[0]C[-2]", FALSE),"survey-","")))', + sheetValueRowsCount + ); + const formulaRows = formulas.map(formula => [formula]); + catalogStatusRange.setFormulas(formulaRows); + + return { updatedSurveysSheetValues }; +} + +/** + * @hidden + */ +function fileNameToSurveyId(fileName) { + return fileName.replace("survey-", ""); +} + +/** + * @hidden + */ +function refreshCombinedToplineSheetListing( + updatedSurveysSheetValues: any[][], + combinedToplineSheet: Sheet, + combinedToplineSheetValuesIncludingHeaderRow: any[][], + gsResultsFolderGsheetFiles: File[] +) { + /* tslint:disable:no-console */ + console.info(`Start of refreshCombinedToplineSheetListing()`); + + // Clear all rows except the header row + console.info(`Clearing all rows except the header row`); + combinedToplineSheet + .getRange( + 2, 1, - combinedToplineValues.length, - combinedToplineSheetHeaders.length + combinedToplineSheetValuesIncludingHeaderRow.length, + combinedToplineSheetValuesIncludingHeaderRow[0].length + ) + .clearContent(); + + // From the existing sheet contents, purge entries that does not have an entry in the surveys sheet + // so that the combined topline listing only contains rows that are relevant for analysis + console.info(`Purging orphaned rows from the combined topline listing`); + const combinedToplineSheetValues = combinedToplineSheetValuesIncludingHeaderRow.slice( + 1 + ); + const existingSurveyEntries = updatedSurveysSheetValues.map( + surveysSheetValueRowToSurveyEntry + ); + const existingToplineEntries = combinedToplineSheetValues.map( + combinedToplineSheetValueRowToToplineEntry + ); + const existingSurveysSurveyIds = existingSurveyEntries.map( + existingSurveyEntry => fileNameToSurveyId(existingSurveyEntry.file_name) + ); + const existingToplineSurveyIds = union( + existingToplineEntries.map( + existingToplineEntry => existingToplineEntry.survey_id + ) + ); + + const surveyIdsInBothListings = intersection( + existingSurveysSurveyIds, + existingToplineSurveyIds + ); + + const toplineEntriesWithSurveyEntry = existingToplineEntries.filter( + toplineEntry => surveyIdsInBothListings.includes(toplineEntry.survey_id) + ); + + // Write the purged/trimmed contents back to the sheet + console.info(`Writing the purged/trimmed contents back to the sheet`); + if (toplineEntriesWithSurveyEntry.length > 0) { + combinedToplineSheet + .getRange( + 2, + 1, + toplineEntriesWithSurveyEntry.length, + combinedToplineSheetValuesIncludingHeaderRow[0].length + ) + .setValues( + toplineEntriesWithSurveyEntry.map( + toplineEntryToCombinedToplineSheetValueRow + ) + ); + } + let rowsWritten = toplineEntriesWithSurveyEntry.length; + + console.info( + `Finding which gsheet files are not-yet-included in the combined topline listing` + ); + const gsResultsFolderGsheetFilesSurveyIds = union( + gsResultsFolderGsheetFiles.map(gsResultsFolderGsheetFile => + fileNameToSurveyId(gsResultsFolderGsheetFile.getName()) ) - .setValues(combinedToplineValues); + ); + const notYetIncludedGsResultsFolderGsheetFilesSurveyIds = difference( + gsResultsFolderGsheetFilesSurveyIds, + existingToplineSurveyIds + ); + const notYetIncludedGsResultsFolderGsheetFiles = gsResultsFolderGsheetFiles.filter( + (gsResultsFolderGsheetFile: File) => { + const surveyId = fileNameToSurveyId(gsResultsFolderGsheetFile.getName()); + return notYetIncludedGsResultsFolderGsheetFilesSurveyIds.includes( + surveyId + ); + } + ); + // Open each not-yet-included gsheet file and add rows to the end of the sheet continuously + if (notYetIncludedGsResultsFolderGsheetFiles.length > 0) { + console.info( + `Opening the ${notYetIncludedGsResultsFolderGsheetFiles.length} not-yet-included gsheet file(s) and adding them to the end of the sheet` + ); + // console.log({ notYetIncludedGsResultsFolderGsheetFiles }); + notYetIncludedGsResultsFolderGsheetFiles.map( + (gsResultsFolderGsheetFile: File) => { + const gsResultsFolderGsheet = SpreadsheetApp.openById( + gsResultsFolderGsheetFile.getId() + ); + const sourceSheet = gsResultsFolderGsheet.getSheetByName("Topline"); + const sourceDataRange = sourceSheet.getDataRange(); + const sourceValuesIncludingHeaderRow = sourceDataRange.getValues(); + const sourceHeaderRows = sourceValuesIncludingHeaderRow.slice(0, 1); + const sourceValues = sourceValuesIncludingHeaderRow.slice(1); + combinedToplineSheet + .getRange( + rowsWritten + 2, + 1, + sourceValues.length, + sourceHeaderRows[0].length + ) + .setValues(sourceValues); + rowsWritten += sourceValues.length; + } + ); + } + + // Read back the updated combined topline sheet data + console.info(`Reading back the updated combined topline sheet data`); + const updatedCombinedToplineSheetValuesIncludingHeaderRow = getSheetDataIncludingHeaderRow( + combinedToplineSheet, + combinedToplineSheetHeaders + ); + + // Limit the amount of rows of the surveys spreadsheet to the amount of entries + console.info( + `Limiting the amount of rows of the surveys spreadsheet to the amount of entries` + ); + const updatedCombinedToplineSheetData = updatedCombinedToplineSheetValuesIncludingHeaderRow.slice( + 1 + ); + adjustSheetRowsAndColumnsCount( + combinedToplineSheet, + updatedCombinedToplineSheetData.length + 1, + updatedCombinedToplineSheetValuesIncludingHeaderRow[0].length + ); + + console.info(`End of refreshCombinedToplineSheetListing()`); + /* tslint:enable:no-console */ }