From a77e89db20a77bcc8e50e243d3fe435f28598e17 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Fri, 13 Nov 2020 14:56:18 +0000
Subject: [PATCH 01/20] updated readme with error information

---
 README.md                | 18 ++++++++++++++++++
 app.js                   |  9 ++++++++-
 screenshot/screenshot.js | 31 ++++++++++++++++++++++++++-----
 3 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 3f178dc..77a3857 100644
--- a/README.md
+++ b/README.md
@@ -52,6 +52,24 @@ When no arguments are passed the application will assume your RAMPART project is
 
 ---
 
+## Compatibility with Rampart version
+
+This application has been tested against Rampart version 1.1.0. This application relies on the structure of the Rampart page being constant, therefore future updates to Rampart may break this application.
+
+---
+
+## Timeout error
+
+When running the application, you may occasionally experience this error: "TimeoutError: waiting for selector "[selector]" failed". This could be related to a potential puppeteer bug: [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). This error can usually be resolved by running the application again.
+
+---
+
+## Why does the application sometimes fail to screenshot every chart?
+
+When viewing your image outputs you might occasionally notice that the application skipped a screenshot, or just took a screenshot of the tab bar containing the chart (and not the chart itself). This may be related to the above mentioned error [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). If this happens, try running the application again or take a manual screenshot of the charts that were missed.
+
+---
+
 ## Additional Project Notes
 
 This project uses puppeteer [https://github.com/puppeteer/puppeteer](https://github.com/puppeteer/puppeteer) to programatically interact with the browser.
diff --git a/app.js b/app.js
index f43a5c8..f6dc321 100644
--- a/app.js
+++ b/app.js
@@ -34,6 +34,13 @@ const puppeteerConnect = async (url) => {
 		return;
 	}
 
+	// helper function to wait for a specified period of time
+	const delay = (time) => {
+		return new Promise(function (resolve) {
+			setTimeout(resolve, time);
+		});
+	};
+
 	// get the current date in YYYY-MM-DD format (for creating directories)
 	let timeStamp = Date.now();
 	let dateObject = new Date(timeStamp);
@@ -48,7 +55,7 @@ const puppeteerConnect = async (url) => {
 		fs.mkdirSync(directory, { recursive: true });
 	}
 
-	await screenshot.takeScreenshots(page, directory);
+	await screenshot.takeScreenshots(page, directory, delay);
 	// await saveReport.saveReport(page, directory);
 	console.log("All operations completed");
 
diff --git a/screenshot/screenshot.js b/screenshot/screenshot.js
index 1b3a465..f69a54c 100644
--- a/screenshot/screenshot.js
+++ b/screenshot/screenshot.js
@@ -1,7 +1,7 @@
 const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 
-const takeScreenshots = async (page, directory) => {
+const takeScreenshots = async (page, directory, delay) => {
 	// count how many divs are clickable buttons (to open the chart tabs)
 	let i = 1;
 	let blocksLeftToCount = true;
@@ -27,10 +27,29 @@ const takeScreenshots = async (page, directory) => {
 			// element will be null if it doesn't contain the button to open the chart tab
 			continue;
 		}
+
 		await element.click(); // open the chart tab and wait for the element to appear in the dom
-		await page.waitForSelector(
-			`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
-		);
+		console.log("tab button clicked");
+		await delay(700);
+		try {
+			const tabContent = await page.$(
+				`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
+			);
+			let parsedTabContent = await page.evaluate(
+				(element) => element.className,
+				tabContent
+			);
+			console.log("element exists, classname = " + parsedTabContent);
+		} catch (err) {
+			console.log("could not get tab content");
+			continue;
+		}
+
+		// console.log("waiting for selector");
+		// await page.waitForSelector(
+		// 	`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
+		// );
+		// console.log("successfully waited for content");
 
 		// remove this repetition
 		const chartTabName = await page.$(
@@ -41,7 +60,9 @@ const takeScreenshots = async (page, directory) => {
 			chartTabName
 		);
 
-		console.log(`Opening chart tab: ${chartTabNameText}, please wait...`);
+		console.log(
+			`Opening chart tab: ${chartTabNameText}, please wait...` + "\n"
+		);
 	}
 
 	// store an array of elements pointing towards the tabs containing charts, also store the tab names

From 349d75205b1b54dc3498fa2e4fbb0f51098fba60 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Fri, 13 Nov 2020 16:47:56 +0000
Subject: [PATCH 02/20] added try catch blocks to functions

---
 README.md                 | 22 +++++++---------
 app.js                    | 22 +++++++++++++---
 screenshot/save_report.js |  2 +-
 screenshot/screenshot.js  | 55 +++++++++++++++++++++------------------
 4 files changed, 58 insertions(+), 43 deletions(-)

diff --git a/README.md b/README.md
index 77a3857..0043bf9 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-This project allows users of the RAMPART project [https://github.com/artic-network/rampart](https://github.com/artic-network/rampart) to take automatic screenshots of RAMPART chart outputs.
+This application allows users of the RAMPART project [https://github.com/artic-network/rampart](https://github.com/artic-network/rampart) to take automatic screenshots of RAMPART chart outputs using puppeteer https://github.com/puppeteer/puppeteer](https://github.com/puppeteer/puppeteer).
 
 ---
 
@@ -28,9 +28,7 @@ Once RAMPART is running, navigate to the root of this project and run:
 node app.js
 ```
 
-The application will then automatically take screenshots of the charts in your active RAMPART session. Screenshots are saved as .png files in the project root under `/images`. Depending on how many charts you have in your project, it may take some time to finish taking all of the screenshots.
-
-Once the program has started, puppeteer will open a temporary Chrome window to start taking screenshots. The window will automatically close again when the program has finished.
+The application will then automatically take screenshots of the charts in your active RAMPART session. Screenshots are saved as .png files in the project root under `/outputs`. Depending on how many charts you have in your project, it may take some time to finish taking all of the screenshots. Any screenshots that fail (see below for possible errors) will have "-failed" appended to the filename.
 
 ---
 
@@ -52,26 +50,24 @@ When no arguments are passed the application will assume your RAMPART project is
 
 ---
 
-## Compatibility with Rampart version
+## Timeout error
 
-This application has been tested against Rampart version 1.1.0. This application relies on the structure of the Rampart page being constant, therefore future updates to Rampart may break this application.
+When running the application, you may occasionally experience this error: "TimeoutError: waiting for selector "[selector]" failed". This could be related to a potential puppeteer bug: [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072).
 
----
-
-## Timeout error
+When this error occurs the application will try to continue taking screenshots of the remaining charts. At this point you may wish to quit the running process and start it again (this can fix the error) or leave it running to take the rest of the screenshots. If you leave the process running, note that the chart causing the error will not produce a successful screenshot so you will need to take the screenshot of that particular chart manually.
 
-When running the application, you may occasionally experience this error: "TimeoutError: waiting for selector "[selector]" failed". This could be related to a potential puppeteer bug: [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). This error can usually be resolved by running the application again.
+All charts that fail to produce a successful screenshot will have "-failed" appened to the image name in the outputs folder (e.g. Mayinga-failed.png).
 
 ---
 
 ## Why does the application sometimes fail to screenshot every chart?
 
-When viewing your image outputs you might occasionally notice that the application skipped a screenshot, or just took a screenshot of the tab bar containing the chart (and not the chart itself). This may be related to the above mentioned error [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). If this happens, try running the application again or take a manual screenshot of the charts that were missed.
+When viewing your image outputs you might occasionally notice that the application did not take a screenshot of every chart, or just took a screenshot of the tab bar containing the chart (in which case the image name will have "-failed" appended). This is related to the above mentioned timeout error [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). If this happens, try running the application again or take a manual screenshot of the charts that were missed.
 
 ---
 
-## Additional Project Notes
+## Compatibility
 
-This project uses puppeteer [https://github.com/puppeteer/puppeteer](https://github.com/puppeteer/puppeteer) to programatically interact with the browser.
+All features have been tested against Rampart version 1.1.0. This application relies on the structure of the Rampart page being constant, therefore future updates to Rampart may cause errors.
 
 ---
diff --git a/app.js b/app.js
index f6dc321..5ff2fe0 100644
--- a/app.js
+++ b/app.js
@@ -55,9 +55,25 @@ const puppeteerConnect = async (url) => {
 		fs.mkdirSync(directory, { recursive: true });
 	}
 
-	await screenshot.takeScreenshots(page, directory, delay);
-	// await saveReport.saveReport(page, directory);
-	console.log("All operations completed");
+	try {
+		await screenshot.takeScreenshots(page, directory, delay);
+	} catch (err) {
+		console.log(
+			"\x1b[36m%s\x1b[0m",
+			" \n Something went wrong taking screenshots. Printing error... \n"
+		);
+		console.log(err);
+	}
+
+	// try {
+	// 	await saveReport.saveReport(page, directory, delay);
+	// } catch (err) {
+	// 	console.log(
+	// 		"\x1b[36m%s\x1b[0m",
+	// 		"Something went wrong taking screenshots. Printing error... \n"
+	// 	);
+	// 	console.log(err);
+	// }
 
 	browser.close();
 };
diff --git a/screenshot/save_report.js b/screenshot/save_report.js
index deae8eb..a9272b4 100644
--- a/screenshot/save_report.js
+++ b/screenshot/save_report.js
@@ -3,7 +3,7 @@ const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 const createCsvWriter = require("csv-writer").createObjectCsvWriter;
 
-const saveReport = async (page, directory) => {
+const saveReport = async (page, directory, delay) => {
 	const reportButtonElement = await page.$(
 		`#root > div > div > div.sc-AxirZ.aHDei > div.buttons > button`
 	);
diff --git a/screenshot/screenshot.js b/screenshot/screenshot.js
index f69a54c..b6c4f85 100644
--- a/screenshot/screenshot.js
+++ b/screenshot/screenshot.js
@@ -1,3 +1,4 @@
+const { fail } = require("assert");
 const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 
@@ -17,8 +18,10 @@ const takeScreenshots = async (page, directory, delay) => {
 	}
 	const foundElementsCount = i;
 
+	const failedScreenshots = [];
 	// open all of the tabs containing charts (necessary to take screenshots)
 	for (let i = 1; i < foundElementsCount + 1; i++) {
+		// get the element containing the button to open the chart tab
 		const element = await page.$(
 			`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > svg`
 		);
@@ -28,30 +31,7 @@ const takeScreenshots = async (page, directory, delay) => {
 			continue;
 		}
 
-		await element.click(); // open the chart tab and wait for the element to appear in the dom
-		console.log("tab button clicked");
-		await delay(700);
-		try {
-			const tabContent = await page.$(
-				`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
-			);
-			let parsedTabContent = await page.evaluate(
-				(element) => element.className,
-				tabContent
-			);
-			console.log("element exists, classname = " + parsedTabContent);
-		} catch (err) {
-			console.log("could not get tab content");
-			continue;
-		}
-
-		// console.log("waiting for selector");
-		// await page.waitForSelector(
-		// 	`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
-		// );
-		// console.log("successfully waited for content");
-
-		// remove this repetition
+		// get the name of the current chart tab
 		const chartTabName = await page.$(
 			`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > span`
 		);
@@ -60,8 +40,26 @@ const takeScreenshots = async (page, directory, delay) => {
 			chartTabName
 		);
 
+		await element.click(); // open the chart tab
+		console.log("chart tab button clicked");
+
+		try {
+			console.log("waiting for selector");
+			await page.waitForSelector(
+				`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
+			);
+		} catch (err) {
+			console.log(
+				`the application experienced a timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart` +
+					"\n"
+			);
+			failedScreenshots.push(chartTabNameText);
+			continue;
+		}
+
 		console.log(
-			`Opening chart tab: ${chartTabNameText}, please wait...` + "\n"
+			`Successfully opened chart tab: ${chartTabNameText}, please wait...` +
+				"\n"
 		);
 	}
 
@@ -102,8 +100,13 @@ const takeScreenshots = async (page, directory, delay) => {
 	for (let i = 0; i < chartTabGroupElements.length; i++) {
 		console.log(`Taking screenshot: ${chartTabGroupNames[i]}, please wait...`);
 		const chartTab = await page.$(chartTabGroupElements[i]);
+
+		const appendToImageName = failedScreenshots.includes(chartTabGroupNames[i])
+			? "-failed"
+			: "";
+
 		await chartTab.screenshot({
-			path: `${directory}/${chartTabGroupNames[i]}.png`,
+			path: `${directory}/${chartTabGroupNames[i]}${appendToImageName}.png`,
 			omitBackground: true,
 		});
 	}

From 3de20ce1e00b6bbbfa083ed7cc113934f9821cb6 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Fri, 13 Nov 2020 17:29:21 +0000
Subject: [PATCH 03/20] updated directory structure. Improved error logging

---
 app.js                             | 26 +++++++++++++-------------
 {screenshot => src}/save_report.js |  2 --
 {screenshot => src}/screenshot.js  |  7 +++----
 3 files changed, 16 insertions(+), 19 deletions(-)
 rename {screenshot => src}/save_report.js (97%)
 rename {screenshot => src}/screenshot.js (95%)

diff --git a/app.js b/app.js
index 5ff2fe0..cd2adb4 100644
--- a/app.js
+++ b/app.js
@@ -1,13 +1,13 @@
 const fs = require("fs");
 const puppeteer = require("puppeteer");
-const screenshot = require("./screenshot/screenshot");
-const saveReport = require("./screenshot/save_report");
+const screenshot = require("./src/screenshot");
+const saveReport = require("./src/save_report");
 
 // process arguments
 let url = process.argv[2];
 if (!url) {
 	console.log(
-		"No arguments passed to specify url. Defaulting to http://localhost:3000"
+		"No arguments passed to specify url. Defaulting to http://localhost:3000 \n"
 	);
 	url = "http://localhost:3000";
 }
@@ -65,17 +65,17 @@ const puppeteerConnect = async (url) => {
 		console.log(err);
 	}
 
-	// try {
-	// 	await saveReport.saveReport(page, directory, delay);
-	// } catch (err) {
-	// 	console.log(
-	// 		"\x1b[36m%s\x1b[0m",
-	// 		"Something went wrong taking screenshots. Printing error... \n"
-	// 	);
-	// 	console.log(err);
-	// }
+	try {
+		// await saveReport.saveReport(page, directory, delay);
+	} catch (err) {
+		console.log(
+			"\x1b[36m%s\x1b[0m",
+			"Something went wrong taking screenshots. Printing error... \n"
+		);
+		console.log(err);
+	}
 
-	browser.close();
+	//browser.close();
 };
 
 puppeteerConnect(url);
diff --git a/screenshot/save_report.js b/src/save_report.js
similarity index 97%
rename from screenshot/save_report.js
rename to src/save_report.js
index a9272b4..9b7ae21 100644
--- a/screenshot/save_report.js
+++ b/src/save_report.js
@@ -1,5 +1,3 @@
-const { table } = require("console");
-const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 const createCsvWriter = require("csv-writer").createObjectCsvWriter;
 
diff --git a/screenshot/screenshot.js b/src/screenshot.js
similarity index 95%
rename from screenshot/screenshot.js
rename to src/screenshot.js
index b6c4f85..98dc99d 100644
--- a/screenshot/screenshot.js
+++ b/src/screenshot.js
@@ -1,4 +1,3 @@
-const { fail } = require("assert");
 const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 
@@ -41,16 +40,16 @@ const takeScreenshots = async (page, directory, delay) => {
 		);
 
 		await element.click(); // open the chart tab
-		console.log("chart tab button clicked");
+		console.log(`chart tab button ${chartTabNameText} clicked`);
 
 		try {
-			console.log("waiting for selector");
+			console.log("waiting for element to appear in the dom, please wait...");
 			await page.waitForSelector(
 				`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
 			);
 		} catch (err) {
 			console.log(
-				`the application experienced a timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart` +
+				`timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart...` +
 					"\n"
 			);
 			failedScreenshots.push(chartTabNameText);

From c646b29e3fb567d8756fb2310687933af29c3d33 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Wed, 18 Nov 2020 15:54:21 +0000
Subject: [PATCH 04/20] adding a recursive function to make multiple attempts
 at opening chart tabs

---
 app.js            | 12 +++++++--
 src/screenshot.js | 69 +++++++++++++++++++++++++++++++----------------
 2 files changed, 56 insertions(+), 25 deletions(-)

diff --git a/app.js b/app.js
index cd2adb4..c9def0a 100644
--- a/app.js
+++ b/app.js
@@ -21,6 +21,14 @@ const puppeteerConnect = async (url) => {
 	const page = await browser.newPage();
 	await page.setViewport({ width: 1980, height: 50000 }); // setting large height to account for case where there are many charts
 
+	// minimise the window if running with headless mode as false
+	// const session = await page.target().createCDPSession();
+	// const { windowId } = await session.send("Browser.getWindowForTarget");
+	// await session.send("Browser.setWindowBounds", {
+	// 	windowId,
+	// 	bounds: { windowState: "minimized" },
+	// });
+
 	try {
 		await page.goto(url, {
 			waitUntil: "networkidle2",
@@ -70,12 +78,12 @@ const puppeteerConnect = async (url) => {
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
-			"Something went wrong taking screenshots. Printing error... \n"
+			"Something went wrong saving the chart. Printing error... \n"
 		);
 		console.log(err);
 	}
 
-	//browser.close();
+	browser.close();
 };
 
 puppeteerConnect(url);
diff --git a/src/screenshot.js b/src/screenshot.js
index 98dc99d..8a0dddf 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -20,10 +20,11 @@ const takeScreenshots = async (page, directory, delay) => {
 	const failedScreenshots = [];
 	// open all of the tabs containing charts (necessary to take screenshots)
 	for (let i = 1; i < foundElementsCount + 1; i++) {
-		// get the element containing the button to open the chart tab
-		const element = await page.$(
-			`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > svg`
-		);
+		// a reference to the dom element containing the button to open chart tabs
+		const tabButtonRef = `#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > svg`;
+
+		// give puppeteer access to the element containing the button to open the chart tab
+		const element = await page.$(tabButtonRef);
 
 		if (element === null) {
 			// element will be null if it doesn't contain the button to open the chart tab
@@ -39,27 +40,49 @@ const takeScreenshots = async (page, directory, delay) => {
 			chartTabName
 		);
 
-		await element.click(); // open the chart tab
-		console.log(`chart tab button ${chartTabNameText} clicked`);
-
-		try {
-			console.log("waiting for element to appear in the dom, please wait...");
-			await page.waitForSelector(
-				`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
-			);
-		} catch (err) {
-			console.log(
-				`timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart...` +
-					"\n"
-			);
-			failedScreenshots.push(chartTabNameText);
+		// a recursive function that will attempt to open the chart tab 2 times if it fails for some reason
+		let attempts = 0;
+		const openAndWait = async () => {
+			console.log(attempts);
+
+			// get the button to open the tab again (else the node detaches on the second attempt)
+			const element = await page.$(tabButtonRef);
+
+			await element.click(); // open the chart tab
+			console.log(`chart tab button ${chartTabNameText} clicked`);
+
+			try {
+				console.log("waiting for element to appear in the dom, please wait...");
+				await page.waitForSelector(
+					`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
+				);
+				console.log(
+					`Successfully opened chart tab: ${chartTabNameText}, please wait...` +
+						"\n"
+				);
+				return "success";
+			} catch (err) {
+				attempts++;
+				if (attempts <= 1) {
+					console.log(`trying again to open tab ${chartTabNameText}`);
+					await openAndWait();
+				} else {
+					console.log(
+						`timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart...` +
+							"\n"
+					);
+					failedScreenshots.push(chartTabNameText);
+					return "failed to open tab";
+				}
+			}
+		};
+
+		const res = await openAndWait();
+		console.log(res);
+
+		if (res === "failed to open tab") {
 			continue;
 		}
-
-		console.log(
-			`Successfully opened chart tab: ${chartTabNameText}, please wait...` +
-				"\n"
-		);
 	}
 
 	// store an array of elements pointing towards the tabs containing charts, also store the tab names

From b92e23b46c99bac839f99040025c2b51d1db7679 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Wed, 18 Nov 2020 17:42:02 +0000
Subject: [PATCH 05/20] refactored screenshot module into separate functions.
 Recursive function for opening tabs now working.

---
 src/screenshot.js | 325 +++++++++++++++++++++++++---------------------
 1 file changed, 177 insertions(+), 148 deletions(-)

diff --git a/src/screenshot.js b/src/screenshot.js
index 8a0dddf..d807487 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -1,178 +1,207 @@
 const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 
-const takeScreenshots = async (page, directory, delay) => {
-	// count how many divs are clickable buttons (to open the chart tabs)
-	let i = 1;
-	let blocksLeftToCount = true;
-	while (blocksLeftToCount) {
-		const element = await page.$(`#root > div > div > div:nth-child(${i})`);
-
-		if (element === null) {
-			// element will be null when it finds no more dom elements
-			i--; // do this to avoid adding an extra element to the count
-			blocksLeftToCount = false;
-		}
-		i++;
-	}
-	const foundElementsCount = i;
-
-	const failedScreenshots = [];
-	// open all of the tabs containing charts (necessary to take screenshots)
-	for (let i = 1; i < foundElementsCount + 1; i++) {
-		// a reference to the dom element containing the button to open chart tabs
-		const tabButtonRef = `#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > svg`;
-
-		// give puppeteer access to the element containing the button to open the chart tab
-		const element = await page.$(tabButtonRef);
-
-		if (element === null) {
-			// element will be null if it doesn't contain the button to open the chart tab
-			continue;
-		}
-
+const takeScreenshots = async (page, directory) => {
+	const getElementText = async (elementRef) => {
 		// get the name of the current chart tab
-		const chartTabName = await page.$(
-			`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > span`
-		);
-		let chartTabNameText = await page.evaluate(
+		const elementTextRef = await page.$(elementRef);
+		let text = await page.evaluate(
 			(element) => element.innerHTML,
-			chartTabName
+			elementTextRef
 		);
+		return text;
+	};
+
+	const countElements = async () => {
+		// count how many divs exist in the dom (so we know what to iterate over when opening chart tabs)
+		let i = 1;
+		let blocksLeftToCount = true;
+		while (blocksLeftToCount) {
+			const divRef = `#root > div > div > div:nth-child(${i})`;
+			const element = await page.$(divRef);
+
+			if (element === null) {
+				// element will be null when it finds no more dom elements
+				i--; // do this to avoid adding an extra element to the count
+				blocksLeftToCount = false;
+			}
+			i++;
+		}
+		return i;
+	};
 
-		// a recursive function that will attempt to open the chart tab 2 times if it fails for some reason
-		let attempts = 0;
-		const openAndWait = async () => {
-			console.log(attempts);
+	// open all of the tabs containing charts (necessary to take screenshots)
+	const openChartTabs = async (domElementCount) => {
+		for (let i = 1; i < domElementCount + 1; i++) {
+			// a recursive function that will attempt to open the chart tab 2 times if it fails for some reason
+			let attempts = 0;
+			const openTabAndWait = async () => {
+				console.log(attempts);
 
-			// get the button to open the tab again (else the node detaches on the second attempt)
-			const element = await page.$(tabButtonRef);
+				// get the button to open the tab again (else the node detaches on the second attempt)
+				const element = await page.$(tabButtonRef);
+
+				await element.click(); // open the chart tab
+				console.log(`chart tab button ${chartTabNameText} clicked`);
 
-			await element.click(); // open the chart tab
-			console.log(`chart tab button ${chartTabNameText} clicked`);
-
-			try {
-				console.log("waiting for element to appear in the dom, please wait...");
-				await page.waitForSelector(
-					`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
-				);
-				console.log(
-					`Successfully opened chart tab: ${chartTabNameText}, please wait...` +
-						"\n"
-				);
-				return "success";
-			} catch (err) {
-				attempts++;
-				if (attempts <= 1) {
-					console.log(`trying again to open tab ${chartTabNameText}`);
-					await openAndWait();
-				} else {
+				try {
 					console.log(
-						`timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart...` +
+						"waiting for element to appear in the dom, please wait..."
+					);
+					await page.waitForSelector(
+						`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
+					);
+					console.log(
+						`Successfully opened chart tab: ${chartTabNameText}, please wait...` +
 							"\n"
 					);
-					failedScreenshots.push(chartTabNameText);
-					return "failed to open tab";
+					return true;
+				} catch (err) {
+					attempts++;
+					if (attempts <= 1) {
+						console.log(`trying again to open tab ${chartTabNameText}`);
+						await openTabAndWait();
+					} else {
+						console.log(
+							`timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart...` +
+								"\n"
+						);
+						failedScreenshots.push(chartTabNameText);
+						return false;
+					}
 				}
-			}
-		};
+			};
 
-		const res = await openAndWait();
-		console.log(res);
+			const tabButtonRef = `#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > svg`;
+			const element = await page.$(tabButtonRef);
+			if (element === null) continue; // element will be null if it doesn't contain the button to open the chart tab
 
-		if (res === "failed to open tab") {
-			continue;
-		}
-	}
-
-	// store an array of elements pointing towards the tabs containing charts, also store the tab names
-	let chartTabGroupElements = [];
-	let chartTabGroupNames = [];
-	for (let i = 1; i < foundElementsCount + 1; i++) {
-		const element = await page.$(
-			`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > svg`
-		);
+			const chartTabNameText = await getElementText(
+				`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > span`
+			);
 
-		if (element === null) {
-			// element will be null if it doesn't contain the svg button to open the chart tab
-			continue;
+			const tabOpened = await openTabAndWait();
+			if (tabOpened === false) {
+				continue;
+			}
 		}
+	};
 
-		const chartTabName = await page.$(
-			`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > span`
-		);
-		let chartTabNameText = await page.evaluate(
-			(element) => element.innerHTML,
-			chartTabName
-		);
+	// store references to the elements containing the chart tabs
+	// also store the names of the chart tabs
+	const storeChartElementRefs = async (domElementCount) => {
+		for (let i = 1; i < domElementCount + 1; i++) {
+			const element = await page.$(
+				`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > svg`
+			);
 
-		// if there are duplicate tab names, append a unique id to the end of the name
-		if (
-			chartTabGroupNames.includes(chartTabNameText) ||
-			fs.existsSync(`./${directory}/${chartTabNameText}.png`)
-		) {
-			chartTabNameText += `_${uuidv4()}`;
-		}
+			// element will be null if it doesn't contain the svg button to open the chart tab
+			if (element === null) continue;
+
+			const chartTabNameText = await getElementText(
+				`#root > div > div > div:nth-child(${i}) > div > div.infoRow > div:nth-child(1) > span`
+			);
+
+			// if there are duplicate tab names, append a unique id to the end of the name
+			if (
+				chartTabGroupNames.includes(chartTabNameText) ||
+				fs.existsSync(`./${directory}/${chartTabNameText}.png`)
+			) {
+				chartTabNameText += `_${uuidv4()}`;
+			}
 
-		chartTabGroupNames.push(chartTabNameText);
-		chartTabGroupElements.push(`#root > div > div > div:nth-child(${i}) > div`);
-	}
+			chartTabGroupNames.push(chartTabNameText);
+			chartTabGroupElements.push(
+				`#root > div > div > div:nth-child(${i}) > div`
+			);
+		}
+	};
 
 	// screenshot each tab containing charts
-	for (let i = 0; i < chartTabGroupElements.length; i++) {
-		console.log(`Taking screenshot: ${chartTabGroupNames[i]}, please wait...`);
-		const chartTab = await page.$(chartTabGroupElements[i]);
-
-		const appendToImageName = failedScreenshots.includes(chartTabGroupNames[i])
-			? "-failed"
-			: "";
+	const takeTabScreenshots = async (
+		chartTabGroupElements,
+		chartTabGroupNames,
+		failedScreenshots
+	) => {
+		for (let i = 0; i < chartTabGroupElements.length; i++) {
+			console.log(
+				`Taking screenshot: ${chartTabGroupNames[i]}, please wait...`
+			);
+			const chartTab = await page.$(chartTabGroupElements[i]);
+
+			// append 'failed' to image name if the chart tab failed to open
+			const appendToImageName = failedScreenshots.includes(
+				chartTabGroupNames[i]
+			)
+				? "-failed"
+				: "";
+
+			await chartTab.screenshot({
+				path: `${directory}/${chartTabGroupNames[i]}${appendToImageName}.png`,
+				omitBackground: true,
+			});
+		}
+	};
+
+	const takeHeaderScreenshot = async () => {
+		// screenshot the top header
+		const topHeader = await page.$("#root > div > div > div");
+		await topHeader.screenshot({
+			path: fs.existsSync(`./${directory}/top_header.png`)
+				? `${directory}/top_header${uuidv4()}.png`
+				: `${directory}/top_header.png`,
+			omitBackground: true,
+		});
 
-		await chartTab.screenshot({
-			path: `${directory}/${chartTabGroupNames[i]}${appendToImageName}.png`,
+		// screenshot the first set of charts in the top header (append unique id to name if it already exists in the directory)
+		const headerCharts = await page.$(
+			"#root > div > div > div:nth-child(2) > div"
+		);
+		await headerCharts.screenshot({
+			path: fs.existsSync(`./${directory}/header_charts.png`)
+				? `${directory}/header_charts_${uuidv4()}.png`
+				: `${directory}/header_charts.png`,
 			omitBackground: true,
 		});
-	}
+	};
+
+	const takeFullpageScreenshot = async () => {
+		// detect the size of the page when all chart tabs are open so we can take a full page screenshot
+		// commented out setViewport as the chart tabs get closed when changing the viewport size (this is an issue with RAMPART not this app)
+		const rootElement = await page.$("#root");
+		const boundingBox = await rootElement.boundingBox();
+		const { width, height } = boundingBox;
+
+		// await page.setViewport({ width: width, height: height });
+		if (height > page.viewport().height) {
+			console.log(
+				`Viewport too small to fit images on screen. You need to increase the viewport height in app.js line 20. Please set the height larger than ${height}`
+			);
+		}
 
-	// screenshot the first set of charts in the top header (append unique id to name if it already exists in the directory)
-	const headerCharts = await page.$(
-		"#root > div > div > div:nth-child(2) > div"
+		// screenshot the entire page (append unique id to name if it already exists in the directory)
+		await rootElement.screenshot({
+			path: fs.existsSync(`./${directory}/full_page.png`)
+				? `${directory}/full_page${uuidv4()}.png`
+				: `${directory}/full_page.png`,
+			omitBackground: true,
+		});
+	};
+
+	const chartTabGroupElements = []; // stores the chart tab dom elements
+	const chartTabGroupNames = []; // stores the name of each chart tab
+	const failedScreenshots = []; // stores the names of any charts that failed to screenshot
+
+	const domElementCount = await countElements();
+	await openChartTabs(domElementCount);
+	await storeChartElementRefs(domElementCount);
+	await takeTabScreenshots(
+		chartTabGroupElements,
+		chartTabGroupNames,
+		failedScreenshots
 	);
-	await headerCharts.screenshot({
-		path: fs.existsSync(`./${directory}/header_charts.png`)
-			? `${directory}/header_charts_${uuidv4()}.png`
-			: `${directory}/header_charts.png`,
-		omitBackground: true,
-	});
-
-	// screenshot the top header
-	const topHeader = await page.$("#root > div > div > div");
-	await topHeader.screenshot({
-		path: fs.existsSync(`./${directory}/top_header.png`)
-			? `${directory}/top_header${uuidv4()}.png`
-			: `${directory}/top_header.png`,
-		omitBackground: true,
-	});
-
-	// detect the size of the page when all chart tabs are open so we can take a full page screenshot
-	// commented out setViewport as the chart tabs get closed when changing the viewport size (this is an issue with RAMPART not this app)
-	const rootElement = await page.$("#root");
-	const boundingBox = await rootElement.boundingBox();
-	const { width, height } = boundingBox;
-
-	// await page.setViewport({ width: width, height: height });
-	if (height > page.viewport().height) {
-		console.log(
-			`Viewport too small to fit images on screen. You need to increase the viewport height in app.js line 20. Please set the height larger than ${height}`
-		);
-	}
-
-	// screenshot the entire page (append unique id to name if it already exists in the directory)
-	await rootElement.screenshot({
-		path: fs.existsSync(`./${directory}/full_page.png`)
-			? `${directory}/full_page${uuidv4()}.png`
-			: `${directory}/full_page.png`,
-		omitBackground: true,
-	});
+	await takeHeaderScreenshot();
+	await takeFullpageScreenshot();
 };
 
 exports.takeScreenshots = takeScreenshots;

From 365ae23478ec60edba1efb8460e297990ae72347 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Wed, 18 Nov 2020 17:57:56 +0000
Subject: [PATCH 06/20] updated readme. Tidied up code

---
 README.md         |  8 ++++----
 src/screenshot.js | 14 +++++---------
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 0043bf9..95d831d 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ This application allows users of the RAMPART project [https://github.com/artic-n
 
 ## Requirements
 
-You need to have Node.js installed on your local machine, this is required to handle npm packages.
+You need to have Node.js (at least version 10.18.1) installed on your local machine, this is required to handle npm packages. Note that RAMPART also requires Node.js to be at least version 10 or above so you should be able to rely on the version of Node.js that is already running your rampart project.
 
 https://nodejs.org/en/download/
 
@@ -54,9 +54,7 @@ When no arguments are passed the application will assume your RAMPART project is
 
 When running the application, you may occasionally experience this error: "TimeoutError: waiting for selector "[selector]" failed". This could be related to a potential puppeteer bug: [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072).
 
-When this error occurs the application will try to continue taking screenshots of the remaining charts. At this point you may wish to quit the running process and start it again (this can fix the error) or leave it running to take the rest of the screenshots. If you leave the process running, note that the chart causing the error will not produce a successful screenshot so you will need to take the screenshot of that particular chart manually.
-
-All charts that fail to produce a successful screenshot will have "-failed" appened to the image name in the outputs folder (e.g. Mayinga-failed.png).
+When this error occurs the application will try to continue taking screenshots of the remaining charts so you will need to take the screenshot of the failed chart manually.
 
 ---
 
@@ -64,6 +62,8 @@ All charts that fail to produce a successful screenshot will have "-failed" appe
 
 When viewing your image outputs you might occasionally notice that the application did not take a screenshot of every chart, or just took a screenshot of the tab bar containing the chart (in which case the image name will have "-failed" appended). This is related to the above mentioned timeout error [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). If this happens, try running the application again or take a manual screenshot of the charts that were missed.
 
+Any charts that fail to produce a successful screenshot will have "-failed" appened to the image name in the outputs folder (e.g. Mayinga-failed.png) and you will therefore need to take a screenshot of this image manually.
+
 ---
 
 ## Compatibility
diff --git a/src/screenshot.js b/src/screenshot.js
index d807487..da15f30 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -36,8 +36,6 @@ const takeScreenshots = async (page, directory) => {
 			// a recursive function that will attempt to open the chart tab 2 times if it fails for some reason
 			let attempts = 0;
 			const openTabAndWait = async () => {
-				console.log(attempts);
-
 				// get the button to open the tab again (else the node detaches on the second attempt)
 				const element = await page.$(tabButtonRef);
 
@@ -51,20 +49,18 @@ const takeScreenshots = async (page, directory) => {
 					await page.waitForSelector(
 						`#root > div > div > div:nth-child(${i}) > div > div:nth-child(2)`
 					);
-					console.log(
-						`Successfully opened chart tab: ${chartTabNameText}, please wait...` +
-							"\n"
-					);
+					console.log(`Successfully opened chart tab ${chartTabNameText} \n`);
 					return true;
 				} catch (err) {
 					attempts++;
 					if (attempts <= 1) {
-						console.log(`trying again to open tab ${chartTabNameText}`);
+						console.log(
+							`experienced a timeout. Trying to open tab ${chartTabNameText} again`
+						);
 						await openTabAndWait();
 					} else {
 						console.log(
-							`timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart...` +
-								"\n"
+							`timeout waiting for tab ${chartTabNameText} to open. Moving on to next chart... \n`
 						);
 						failedScreenshots.push(chartTabNameText);
 						return false;

From b5e6e94021e7a253d1688de522f17f664357718a Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 19 Nov 2020 11:13:18 +0000
Subject: [PATCH 07/20] further tidy up. Updating readme with information about
 fullpage screenshot bug

---
 README.md         | 16 ++++++++++++++++
 app.js            |  2 ++
 src/screenshot.js | 39 ++++++++++++++++++++++++++++-----------
 3 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 95d831d..195a149 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,22 @@ When no arguments are passed the application will assume your RAMPART project is
 
 ---
 
+## Fullpage screenshot issues
+
+There is currently a known chromium bug that is affecting the behaviour of some fullpage screenshots, see here for more information [https://github.com/puppeteer/puppeteer/issues/1576](https://github.com/puppeteer/puppeteer/issues/1576). This seems to affect pages that have a large viewport height and causes the fullpage screenshot to duplicate content.
+
+If you are running into this issue there are currently two options: 1. set headless mode to false (app.js line 17); 2. take the fullpage screenshot manually.
+
+---
+
+## Why does the application sometimes fail to screenshot every chart?
+
+When viewing your image outputs you might occasionally notice that the application did not take a screenshot of every chart, or just took a screenshot of the tab bar containing the chart (in which case the image name will have "-failed" appended). This is related to the above mentioned timeout error [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). If this happens, try running the application again or take a manual screenshot of the charts that were missed.
+
+Any charts that fail to produce a successful screenshot will have "-failed" appened to the image name in the outputs folder (e.g. Mayinga-failed.png) and you will therefore need to take a screenshot of this image manually.
+
+---
+
 ## Timeout error
 
 When running the application, you may occasionally experience this error: "TimeoutError: waiting for selector "[selector]" failed". This could be related to a potential puppeteer bug: [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072).
diff --git a/app.js b/app.js
index c9def0a..33e6434 100644
--- a/app.js
+++ b/app.js
@@ -63,6 +63,7 @@ const puppeteerConnect = async (url) => {
 		fs.mkdirSync(directory, { recursive: true });
 	}
 
+	// take screenshots of charts
 	try {
 		await screenshot.takeScreenshots(page, directory, delay);
 	} catch (err) {
@@ -73,6 +74,7 @@ const puppeteerConnect = async (url) => {
 		console.log(err);
 	}
 
+	// save the information in the 'reports' tab as a .csv
 	try {
 		// await saveReport.saveReport(page, directory, delay);
 	} catch (err) {
diff --git a/src/screenshot.js b/src/screenshot.js
index da15f30..2594001 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -161,29 +161,44 @@ const takeScreenshots = async (page, directory) => {
 		});
 	};
 
-	const takeFullpageScreenshot = async () => {
-		// detect the size of the page when all chart tabs are open so we can take a full page screenshot
-		// commented out setViewport as the chart tabs get closed when changing the viewport size (this is an issue with RAMPART not this app)
-		const rootElement = await page.$("#root");
-		const boundingBox = await rootElement.boundingBox();
-		const { width, height } = boundingBox;
-
-		// await page.setViewport({ width: width, height: height });
-		if (height > page.viewport().height) {
+	const takeFullpageScreenshot = async (viewport) => {
+		if (viewport.height > page.viewport().height) {
 			console.log(
 				`Viewport too small to fit images on screen. You need to increase the viewport height in app.js line 20. Please set the height larger than ${height}`
 			);
 		}
 
+		// await page.screenshot({
+		// 	path: fs.existsSync(`./${directory}/full_page.png`)
+		// 		? `${directory}/full_page${uuidv4()}.png`
+		// 		: `${directory}/full_page.png`,
+		// 	fullPage: true,
+		// });
+
 		// screenshot the entire page (append unique id to name if it already exists in the directory)
+		const rootElement = await page.$("#root");
 		await rootElement.screenshot({
 			path: fs.existsSync(`./${directory}/full_page.png`)
-				? `${directory}/full_page${uuidv4()}.png`
+				? `${directory}/full_page_${uuidv4()}.png`
 				: `${directory}/full_page.png`,
 			omitBackground: true,
 		});
 	};
 
+	const getViewport = async () => {
+		// detect the size of the page when all chart tabs are open so we can take a full page screenshot
+		const rootElement = await page.$("#root");
+		const boundingBox = await rootElement.boundingBox();
+		const { width, height } = boundingBox;
+		return boundingBox;
+	};
+
+	const setViewport = async (viewport) => {
+		console.log(viewport);
+		// commented out setViewport as the chart tabs get closed when changing the viewport size (this is an issue with RAMPART not this app)
+		//await page.setViewport({ width: viewport.width, height: viewport.height });
+	};
+
 	const chartTabGroupElements = []; // stores the chart tab dom elements
 	const chartTabGroupNames = []; // stores the name of each chart tab
 	const failedScreenshots = []; // stores the names of any charts that failed to screenshot
@@ -197,7 +212,9 @@ const takeScreenshots = async (page, directory) => {
 		failedScreenshots
 	);
 	await takeHeaderScreenshot();
-	await takeFullpageScreenshot();
+	const viewport = await getViewport();
+	// await setViewport(viewport);
+	await takeFullpageScreenshot(viewport);
 };
 
 exports.takeScreenshots = takeScreenshots;

From 9e711732dfb54e9a13df73283322dca58674ebe2 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 19 Nov 2020 14:35:22 +0000
Subject: [PATCH 08/20] setting up save_report to dynamically save content from
 all existing tables in the dom (not finished)

---
 app.js             |   4 +-
 src/save_report.js | 201 ++++++++++++++++++++++++++++-----------------
 2 files changed, 126 insertions(+), 79 deletions(-)

diff --git a/app.js b/app.js
index 33e6434..472262e 100644
--- a/app.js
+++ b/app.js
@@ -65,7 +65,7 @@ const puppeteerConnect = async (url) => {
 
 	// take screenshots of charts
 	try {
-		await screenshot.takeScreenshots(page, directory, delay);
+		// await screenshot.takeScreenshots(page, directory, delay);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
@@ -76,7 +76,7 @@ const puppeteerConnect = async (url) => {
 
 	// save the information in the 'reports' tab as a .csv
 	try {
-		// await saveReport.saveReport(page, directory, delay);
+		await saveReport.saveReport(page, directory, delay);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
diff --git a/src/save_report.js b/src/save_report.js
index 9b7ae21..fe7fe36 100644
--- a/src/save_report.js
+++ b/src/save_report.js
@@ -2,89 +2,136 @@ const { v4: uuidv4 } = require("uuid");
 const createCsvWriter = require("csv-writer").createObjectCsvWriter;
 
 const saveReport = async (page, directory, delay) => {
-	const reportButtonElement = await page.$(
-		`#root > div > div > div.sc-AxirZ.aHDei > div.buttons > button`
-	);
+	const openReportTab = async () => {
+		// click the report button to open the tab
+		const reportButtonElement = await page.$(
+			`#root > div > div > div:nth-child(1) > div.buttons > button`
+		);
+		if (reportButtonElement === null) {
+			console.log("No report tab was found");
+			return;
+		}
+		await reportButtonElement.click(); // open the report tab
 
-	if (reportButtonElement === null) {
-		console.log("No report tab was found");
-		return;
-	}
+		// wait for the table content to appear in the dom
+		await page.waitForSelector(`#root > div > div > .open > div`);
+	};
+
+	const countTables = async () => {
+		const tableData = await page.$$(
+			`#root > div > div > .open > div > :nth-child(2) > table`
+		);
+		return tableData.length;
+	};
 
-	await reportButtonElement.click(); // open the report tab
+	const getTableHeaderData = async (tableRef) => {
+		// get the headers from the first table
+		// const tableHeadData = await page.evaluate(() => {
+		// 	const thead = Array.from(
+		// 		document.querySelectorAll(
+		// 			"#root > div > div > div.sc-fzqBZW.ksbfDg.open > div > div.sc-fzqNJr.hSLfNI > table:nth-child(2) > thead > tr > th"
+		// 		)
+		// 	);
+		// 	return thead.map((th) => th.innerText);
+		// });
 
-	await page.waitForSelector(
-		`#root > div > div > div.sc-fzqBZW.ksbfDg.open > div`
-	);
+		const tableHeadData = await page.evaluate((tableRef) => {
+			const thead = Array.from(
+				document.querySelectorAll(`${tableRef} > thead > tr > th`)
+			);
+			return thead.map((th) => th.innerText);
+		}, tableRef);
 
-	// get the headers from the first table
-	const tableHeadData = await page.evaluate(() => {
-		const thead = Array.from(
-			document.querySelectorAll(
-				"#root > div > div > div.sc-fzqBZW.ksbfDg.open > div > div.sc-fzqNJr.hSLfNI > table:nth-child(2) > thead > tr > th"
-			)
-		);
-		return thead.map((th) => th.innerText);
-	});
-
-	// format header data to be inserted into csv
-	const csvHeaders = tableHeadData.map((header, index) => {
-		return {
-			id: index,
-			title: header,
-		};
-	});
-	console.log(tableHeadData);
-	console.log(csvHeaders);
-
-	// get the rows from the first table
-	const tableRowData = await page.evaluate(() => {
-		const tbody = Array.from(
-			document.querySelectorAll(
-				"#root > div > div > div.sc-fzqBZW.ksbfDg.open > div > div.sc-fzqNJr.hSLfNI > table:nth-child(2) > tbody > tr"
-			)
-		);
-		return tbody.map((tr) => tr.innerText);
-	});
-	//console.log(tableRowData);
-
-	// extract content from each table row
-	const parsedRowData = [];
-	tableRowData.forEach((element) => {
-		const splitRow = element.split("\t");
-		parsedRowData.push(splitRow);
-	});
-	console.log(parsedRowData);
-
-	const csvRows = [];
-	parsedRowData.map((row) => {
-		let obj = {};
-		row.map((inner, index) => {
-			obj[index] = inner;
+		// format header data to be inserted into csv
+		const csvHeaders = tableHeadData.map((header, index) => {
+			return {
+				id: index,
+				title: header,
+			};
+		});
+		console.log(tableHeadData);
+		console.log(csvHeaders);
+
+		return csvHeaders;
+	};
+
+	const getTableRowData = async () => {
+		// get the rows from the first table
+		const tableRowData = await page.evaluate(() => {
+			const tbody = Array.from(
+				document.querySelectorAll(
+					"#root > div > div > div.sc-fzqBZW.ksbfDg.open > div > div.sc-fzqNJr.hSLfNI > table:nth-child(2) > tbody > tr"
+				)
+			);
+			return tbody.map((tr) => tr.innerText);
+		});
+
+		// console.log(tableRowData);
+		return tableRowData;
+	};
+
+	const parseTableData1 = async (tableData) => {
+		// extract content from each table row
+		const parsedRowData = [];
+		tableData.forEach((element) => {
+			const splitRow = element.split("\t");
+			parsedRowData.push(splitRow);
 		});
-		csvRows.push(obj);
-	});
-	console.log(csvRows);
-
-	// write data to csv
-	const csvWriter = createCsvWriter({
-		path: `${directory}/report.csv`,
-		header: csvHeaders,
-	});
-
-	// const csvRows = [
-	// 	{ 1: "test1", 2: "test2", 3: "test3", 4: "test4", 5: "test5", 6: "test6" },
-	// 	{ 1: "test1", 2: "test2", 3: "test3", 4: "test4", 5: "test5", 6: "test6" },
-	// ];
-
-	await csvWriter
-		.writeRecords(csvRows) // returns a promise
-		.then(() => {
-			console.log("saved report as .csv");
-		})
-		.catch((error) => {
-			console.error(error);
+		console.log(parsedRowData);
+
+		const csvRows = [];
+		parsedRowData.map((row) => {
+			let obj = {};
+			row.map((inner, index) => {
+				obj[index] = inner;
+			});
+			csvRows.push(obj);
+		});
+
+		console.log(csvRows);
+		return csvRows;
+	};
+
+	const saveTableToCsv = async (csvHeaders, csvRows) => {
+		// write data to csv
+		const csvWriter = createCsvWriter({
+			path: `${directory}/report.csv`,
+			header: csvHeaders,
 		});
+
+		// const csvRows = [
+		// 	{ 1: "test1", 2: "test2", 3: "test3", 4: "test4", 5: "test5", 6: "test6" },
+		// 	{ 1: "test1", 2: "test2", 3: "test3", 4: "test4", 5: "test5", 6: "test6" },
+		// ];
+
+		await csvWriter
+			.writeRecords(csvRows)
+			.then(() => {
+				console.log("saved report as .csv");
+			})
+			.catch((error) => {
+				console.error(error);
+			});
+	};
+
+	await openReportTab();
+	const tableCount = await countTables();
+
+	for (let i = 1; i < tableCount + 1; i++) {
+		const tableRef = `#root > div > div > .open > div > div:nth-child(2) > table:nth-of-type(${i})`;
+		// const element = await page.$(tableRef);
+
+		const csvHeaders = await getTableHeaderData(tableRef);
+
+		// const tableRowData = await getTableRowData();
+		// const csvRows = await parseTableData1(tableRowData);
+		// await saveTableToCsv(csvHeaders, csvRows);
+	}
+
+	// const csvHeaders = await getTableHeaderData();
+	// const tableRowData = await getTableRowData();
+	// const csvRows = await parseTableData1(tableRowData);
+	// await saveTableToCsv(csvHeaders, csvRows);
 };
 
 exports.saveReport = saveReport;

From c9fae5cf5e2c53294947190ebd0e0020bb04621c Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 19 Nov 2020 15:37:07 +0000
Subject: [PATCH 09/20] several bits of tidy up. Removed duplicated content in
 readme

---
 README.md          |  8 ------
 src/save_report.js | 61 +++++++++++++++-------------------------------
 2 files changed, 20 insertions(+), 49 deletions(-)

diff --git a/README.md b/README.md
index 195a149..8937130 100644
--- a/README.md
+++ b/README.md
@@ -58,14 +58,6 @@ If you are running into this issue there are currently two options: 1. set headl
 
 ---
 
-## Why does the application sometimes fail to screenshot every chart?
-
-When viewing your image outputs you might occasionally notice that the application did not take a screenshot of every chart, or just took a screenshot of the tab bar containing the chart (in which case the image name will have "-failed" appended). This is related to the above mentioned timeout error [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072). If this happens, try running the application again or take a manual screenshot of the charts that were missed.
-
-Any charts that fail to produce a successful screenshot will have "-failed" appened to the image name in the outputs folder (e.g. Mayinga-failed.png) and you will therefore need to take a screenshot of this image manually.
-
----
-
 ## Timeout error
 
 When running the application, you may occasionally experience this error: "TimeoutError: waiting for selector "[selector]" failed". This could be related to a potential puppeteer bug: [https://github.com/puppeteer/puppeteer/issues/4072](https://github.com/puppeteer/puppeteer/issues/4072).
diff --git a/src/save_report.js b/src/save_report.js
index fe7fe36..398a258 100644
--- a/src/save_report.js
+++ b/src/save_report.js
@@ -25,16 +25,6 @@ const saveReport = async (page, directory, delay) => {
 	};
 
 	const getTableHeaderData = async (tableRef) => {
-		// get the headers from the first table
-		// const tableHeadData = await page.evaluate(() => {
-		// 	const thead = Array.from(
-		// 		document.querySelectorAll(
-		// 			"#root > div > div > div.sc-fzqBZW.ksbfDg.open > div > div.sc-fzqNJr.hSLfNI > table:nth-child(2) > thead > tr > th"
-		// 		)
-		// 	);
-		// 	return thead.map((th) => th.innerText);
-		// });
-
 		const tableHeadData = await page.evaluate((tableRef) => {
 			const thead = Array.from(
 				document.querySelectorAll(`${tableRef} > thead > tr > th`)
@@ -49,35 +39,31 @@ const saveReport = async (page, directory, delay) => {
 				title: header,
 			};
 		});
-		console.log(tableHeadData);
-		console.log(csvHeaders);
+		// console.log(tableHeadData);
+		// console.log(csvHeaders);
 
 		return csvHeaders;
 	};
 
-	const getTableRowData = async () => {
-		// get the rows from the first table
-		const tableRowData = await page.evaluate(() => {
+	const getTableRowData = async (tableRef) => {
+		const tableRowData = await page.evaluate((tableRef) => {
 			const tbody = Array.from(
-				document.querySelectorAll(
-					"#root > div > div > div.sc-fzqBZW.ksbfDg.open > div > div.sc-fzqNJr.hSLfNI > table:nth-child(2) > tbody > tr"
-				)
+				document.querySelectorAll(`${tableRef} > tbody > tr`)
 			);
 			return tbody.map((tr) => tr.innerText);
-		});
+		}, tableRef);
 
-		// console.log(tableRowData);
 		return tableRowData;
 	};
 
-	const parseTableData1 = async (tableData) => {
+	const parseTableData = async (tableData) => {
 		// extract content from each table row
 		const parsedRowData = [];
 		tableData.forEach((element) => {
 			const splitRow = element.split("\t");
 			parsedRowData.push(splitRow);
 		});
-		console.log(parsedRowData);
+		// console.log(parsedRowData);
 
 		const csvRows = [];
 		parsedRowData.map((row) => {
@@ -88,50 +74,43 @@ const saveReport = async (page, directory, delay) => {
 			csvRows.push(obj);
 		});
 
-		console.log(csvRows);
+		// console.log(csvRows);
 		return csvRows;
 	};
 
-	const saveTableToCsv = async (csvHeaders, csvRows) => {
+	const saveTableToCsv = async (csvHeaders, csvRows, index) => {
 		// write data to csv
 		const csvWriter = createCsvWriter({
-			path: `${directory}/report.csv`,
+			path: `${directory}/report_${index}.csv`,
 			header: csvHeaders,
 		});
 
-		// const csvRows = [
-		// 	{ 1: "test1", 2: "test2", 3: "test3", 4: "test4", 5: "test5", 6: "test6" },
-		// 	{ 1: "test1", 2: "test2", 3: "test3", 4: "test4", 5: "test5", 6: "test6" },
-		// ];
-
 		await csvWriter
 			.writeRecords(csvRows)
 			.then(() => {
-				console.log("saved report as .csv");
+				console.log(`saved report_${index} as .csv`);
 			})
 			.catch((error) => {
+				console.log(
+					`something went wrong trying to write report_${index} as .csv`
+				);
 				console.error(error);
 			});
 	};
 
+	// App flow starts here
 	await openReportTab();
 	const tableCount = await countTables();
 
+	// loop through each table and save the data as csv
 	for (let i = 1; i < tableCount + 1; i++) {
 		const tableRef = `#root > div > div > .open > div > div:nth-child(2) > table:nth-of-type(${i})`;
-		// const element = await page.$(tableRef);
 
 		const csvHeaders = await getTableHeaderData(tableRef);
-
-		// const tableRowData = await getTableRowData();
-		// const csvRows = await parseTableData1(tableRowData);
-		// await saveTableToCsv(csvHeaders, csvRows);
+		const tableRowData = await getTableRowData(tableRef);
+		const csvRows = await parseTableData(tableRowData);
+		await saveTableToCsv(csvHeaders, csvRows, i);
 	}
-
-	// const csvHeaders = await getTableHeaderData();
-	// const tableRowData = await getTableRowData();
-	// const csvRows = await parseTableData1(tableRowData);
-	// await saveTableToCsv(csvHeaders, csvRows);
 };
 
 exports.saveReport = saveReport;

From e38704fb469b5e9d89e2696598a4e2fa0a3fbea2 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 19 Nov 2020 17:14:59 +0000
Subject: [PATCH 10/20] further tidy up. Now saving .csv files with correct
 table names

---
 app.js             |  3 ++-
 src/save_report.js | 30 +++++++++++++++++++++---------
 src/screenshot.js  |  1 +
 3 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/app.js b/app.js
index 472262e..905f623 100644
--- a/app.js
+++ b/app.js
@@ -20,6 +20,7 @@ const puppeteerConnect = async (url) => {
 
 	const page = await browser.newPage();
 	await page.setViewport({ width: 1980, height: 50000 }); // setting large height to account for case where there are many charts
+	page.setDefaultTimeout(10000);
 
 	// minimise the window if running with headless mode as false
 	// const session = await page.target().createCDPSession();
@@ -65,7 +66,7 @@ const puppeteerConnect = async (url) => {
 
 	// take screenshots of charts
 	try {
-		// await screenshot.takeScreenshots(page, directory, delay);
+		await screenshot.takeScreenshots(page, directory, delay);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
diff --git a/src/save_report.js b/src/save_report.js
index 398a258..748e1e2 100644
--- a/src/save_report.js
+++ b/src/save_report.js
@@ -24,6 +24,14 @@ const saveReport = async (page, directory, delay) => {
 		return tableData.length;
 	};
 
+	const getTableName = async (tableRef) => {
+		const tableName = await page.$eval(
+			`${tableRef} > caption`,
+			(el) => el.innerText
+		);
+		return tableName;
+	};
+
 	const getTableHeaderData = async (tableRef) => {
 		const tableHeadData = await page.evaluate((tableRef) => {
 			const thead = Array.from(
@@ -32,6 +40,12 @@ const saveReport = async (page, directory, delay) => {
 			return thead.map((th) => th.innerText);
 		}, tableRef);
 
+		// an alternative method to above (more abstract and less clear what it does)
+		// const tableHeadData = await page.$$eval(
+		// 	`${tableRef} > thead > tr > th`,
+		// 	(theads) => theads.map((th) => th.innerText)
+		// );
+
 		// format header data to be inserted into csv
 		const csvHeaders = tableHeadData.map((header, index) => {
 			return {
@@ -39,9 +53,6 @@ const saveReport = async (page, directory, delay) => {
 				title: header,
 			};
 		});
-		// console.log(tableHeadData);
-		// console.log(csvHeaders);
-
 		return csvHeaders;
 	};
 
@@ -56,10 +67,10 @@ const saveReport = async (page, directory, delay) => {
 		return tableRowData;
 	};
 
-	const parseTableData = async (tableData) => {
+	const parseTableData = async (tableRowData) => {
 		// extract content from each table row
 		const parsedRowData = [];
-		tableData.forEach((element) => {
+		tableRowData.forEach((element) => {
 			const splitRow = element.split("\t");
 			parsedRowData.push(splitRow);
 		});
@@ -78,17 +89,17 @@ const saveReport = async (page, directory, delay) => {
 		return csvRows;
 	};
 
-	const saveTableToCsv = async (csvHeaders, csvRows, index) => {
+	const saveTableToCsv = async (tableName, csvHeaders, csvRows, index) => {
 		// write data to csv
 		const csvWriter = createCsvWriter({
-			path: `${directory}/report_${index}.csv`,
+			path: `${directory}/${tableName}.csv`,
 			header: csvHeaders,
 		});
 
 		await csvWriter
 			.writeRecords(csvRows)
 			.then(() => {
-				console.log(`saved report_${index} as .csv`);
+				console.log(`saved ${tableName} as .csv`);
 			})
 			.catch((error) => {
 				console.log(
@@ -106,10 +117,11 @@ const saveReport = async (page, directory, delay) => {
 	for (let i = 1; i < tableCount + 1; i++) {
 		const tableRef = `#root > div > div > .open > div > div:nth-child(2) > table:nth-of-type(${i})`;
 
+		const tableName = await getTableName(tableRef);
 		const csvHeaders = await getTableHeaderData(tableRef);
 		const tableRowData = await getTableRowData(tableRef);
 		const csvRows = await parseTableData(tableRowData);
-		await saveTableToCsv(csvHeaders, csvRows, i);
+		await saveTableToCsv(tableName, csvHeaders, csvRows, i);
 	}
 };
 
diff --git a/src/screenshot.js b/src/screenshot.js
index 2594001..6c35f61 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -9,6 +9,7 @@ const takeScreenshots = async (page, directory) => {
 			(element) => element.innerHTML,
 			elementTextRef
 		);
+
 		return text;
 	};
 

From 2fd6dc975a98c7bbea30bb72b1eadabb4b3ccdfb Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 26 Nov 2020 09:34:16 +0000
Subject: [PATCH 11/20] tidying things up (cleaning up comments and shortening
 functions)

---
 src/save_report.js | 49 +++++++++++++++++++---------------------------
 src/screenshot.js  |  3 +--
 2 files changed, 21 insertions(+), 31 deletions(-)

diff --git a/src/save_report.js b/src/save_report.js
index 748e1e2..d7508cd 100644
--- a/src/save_report.js
+++ b/src/save_report.js
@@ -3,7 +3,6 @@ const createCsvWriter = require("csv-writer").createObjectCsvWriter;
 
 const saveReport = async (page, directory, delay) => {
 	const openReportTab = async () => {
-		// click the report button to open the tab
 		const reportButtonElement = await page.$(
 			`#root > div > div > div:nth-child(1) > div.buttons > button`
 		);
@@ -11,25 +10,22 @@ const saveReport = async (page, directory, delay) => {
 			console.log("No report tab was found");
 			return;
 		}
-		await reportButtonElement.click(); // open the report tab
-
-		// wait for the table content to appear in the dom
+		await reportButtonElement.click();
 		await page.waitForSelector(`#root > div > div > .open > div`);
 	};
 
 	const countTables = async () => {
-		const tableData = await page.$$(
+		const tables = await page.$$(
 			`#root > div > div > .open > div > :nth-child(2) > table`
 		);
-		return tableData.length;
+		return tables.length;
 	};
 
 	const getTableName = async (tableRef) => {
-		const tableName = await page.$eval(
+		return await page.$eval(
 			`${tableRef} > caption`,
-			(el) => el.innerText
+			(tableCaption) => tableCaption.innerText
 		);
-		return tableName;
 	};
 
 	const getTableHeaderData = async (tableRef) => {
@@ -37,7 +33,7 @@ const saveReport = async (page, directory, delay) => {
 			const thead = Array.from(
 				document.querySelectorAll(`${tableRef} > thead > tr > th`)
 			);
-			return thead.map((th) => th.innerText);
+			return thead.map((thead) => thead.innerText);
 		}, tableRef);
 
 		// an alternative method to above (more abstract and less clear what it does)
@@ -47,50 +43,45 @@ const saveReport = async (page, directory, delay) => {
 		// );
 
 		// format header data to be inserted into csv
-		const csvHeaders = tableHeadData.map((header, index) => {
+		return tableHeadData.map((header, index) => {
 			return {
 				id: index,
 				title: header,
 			};
 		});
-		return csvHeaders;
 	};
 
 	const getTableRowData = async (tableRef) => {
-		const tableRowData = await page.evaluate((tableRef) => {
+		// returns a tab separated string of row data
+		return await page.evaluate((tableRef) => {
 			const tbody = Array.from(
 				document.querySelectorAll(`${tableRef} > tbody > tr`)
 			);
 			return tbody.map((tr) => tr.innerText);
 		}, tableRef);
-
-		return tableRowData;
 	};
 
 	const parseTableData = async (tableRowData) => {
-		// extract content from each table row
+		// convert each table row data into an array of rowItems
 		const parsedRowData = [];
-		tableRowData.forEach((element) => {
-			const splitRow = element.split("\t");
+		tableRowData.forEach((row) => {
+			const splitRow = row.split("\t");
 			parsedRowData.push(splitRow);
 		});
-		// console.log(parsedRowData);
 
 		const csvRows = [];
 		parsedRowData.map((row) => {
-			let obj = {};
-			row.map((inner, index) => {
-				obj[index] = inner;
+			let rowObject = {};
+			row.map((rowItem, index) => {
+				rowObject[index] = rowItem; // index will be the csv tab number where the rowItem exists
 			});
-			csvRows.push(obj);
+			csvRows.push(rowObject);
 		});
 
-		// console.log(csvRows);
 		return csvRows;
 	};
 
-	const saveTableToCsv = async (tableName, csvHeaders, csvRows, index) => {
-		// write data to csv
+	const writeTableToCsv = async (tableName, csvHeaders, csvRows) => {
 		const csvWriter = createCsvWriter({
 			path: `${directory}/${tableName}.csv`,
 			header: csvHeaders,
@@ -99,11 +90,11 @@ const saveReport = async (page, directory, delay) => {
 		await csvWriter
 			.writeRecords(csvRows)
 			.then(() => {
-				console.log(`saved ${tableName} as .csv`);
+				console.log(`saved '${tableName}' as .csv`);
 			})
 			.catch((error) => {
 				console.log(
-					`something went wrong trying to write report_${index} as .csv`
+					`something went wrong trying to write report '${tableName}' as .csv`
 				);
 				console.error(error);
 			});
@@ -121,7 +112,7 @@ const saveReport = async (page, directory, delay) => {
 		const csvHeaders = await getTableHeaderData(tableRef);
 		const tableRowData = await getTableRowData(tableRef);
 		const csvRows = await parseTableData(tableRowData);
-		await saveTableToCsv(tableName, csvHeaders, csvRows, i);
+		await writeTableToCsv(tableName, csvHeaders, csvRows);
 	}
 };
 
diff --git a/src/screenshot.js b/src/screenshot.js
index 6c35f61..0a54ff5 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -40,7 +40,7 @@ const takeScreenshots = async (page, directory) => {
 				// get the button to open the tab again (else the node detaches on the second attempt)
 				const element = await page.$(tabButtonRef);
 
-				await element.click(); // open the chart tab
+				await element.click();
 				console.log(`chart tab button ${chartTabNameText} clicked`);
 
 				try {
@@ -114,7 +114,6 @@ const takeScreenshots = async (page, directory) => {
 		}
 	};
 
-	// screenshot each tab containing charts
 	const takeTabScreenshots = async (
 		chartTabGroupElements,
 		chartTabGroupNames,

From e0d9c145a1fbd5343b38104ab76f1a5b9029ad49 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 26 Nov 2020 11:33:29 +0000
Subject: [PATCH 12/20] experimenting with jimp to take full page screenshot

---
 app.js            |    7 +-
 package-lock.json | 1312 +++++++++++++++++++++++++++++++++++++++++++++
 package.json      |    2 +
 src/screenshot.js |   74 ++-
 4 files changed, 1379 insertions(+), 16 deletions(-)

diff --git a/app.js b/app.js
index 905f623..83b9f94 100644
--- a/app.js
+++ b/app.js
@@ -14,12 +14,13 @@ if (!url) {
 
 const puppeteerConnect = async (url) => {
 	const browser = await puppeteer.launch({
-		// headless: false,
+		headless: false,
 		// slowMo: 250,
 	});
 
 	const page = await browser.newPage();
-	await page.setViewport({ width: 1980, height: 50000 }); // setting large height to account for case where there are many charts
+	// await page.setViewport({ width: 1980, height: 50000 }); // setting large height to account for case where there are many charts
+	// await page.setViewport({ width: 1980, height: 50000, deviceScaleFactor: 1 });
 	page.setDefaultTimeout(10000);
 
 	// minimise the window if running with headless mode as false
@@ -77,7 +78,7 @@ const puppeteerConnect = async (url) => {
 
 	// save the information in the 'reports' tab as a .csv
 	try {
-		await saveReport.saveReport(page, directory, delay);
+		// await saveReport.saveReport(page, directory, delay);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
diff --git a/package-lock.json b/package-lock.json
index 798da92..01d6df6 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -4,6 +4,335 @@
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
+    "@babel/runtime": {
+      "version": "7.12.5",
+      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.12.5.tgz",
+      "integrity": "sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg==",
+      "requires": {
+        "regenerator-runtime": "^0.13.4"
+      }
+    },
+    "@jimp/bmp": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.16.1.tgz",
+      "integrity": "sha512-iwyNYQeBawrdg/f24x3pQ5rEx+/GwjZcCXd3Kgc+ZUd+Ivia7sIqBsOnDaMZdKCBPlfW364ekexnlOqyVa0NWg==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1",
+        "bmp-js": "^0.1.0"
+      }
+    },
+    "@jimp/core": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.16.1.tgz",
+      "integrity": "sha512-la7kQia31V6kQ4q1kI/uLimu8FXx7imWVajDGtwUG8fzePLWDFJyZl0fdIXVCL1JW2nBcRHidUot6jvlRDi2+g==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1",
+        "any-base": "^1.1.0",
+        "buffer": "^5.2.0",
+        "exif-parser": "^0.1.12",
+        "file-type": "^9.0.0",
+        "load-bmfont": "^1.3.1",
+        "mkdirp": "^0.5.1",
+        "phin": "^2.9.1",
+        "pixelmatch": "^4.0.2",
+        "tinycolor2": "^1.4.1"
+      }
+    },
+    "@jimp/custom": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.16.1.tgz",
+      "integrity": "sha512-DNUAHNSiUI/j9hmbatD6WN/EBIyeq4AO0frl5ETtt51VN1SvE4t4v83ZA/V6ikxEf3hxLju4tQ5Pc3zmZkN/3A==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/core": "^0.16.1"
+      }
+    },
+    "@jimp/gif": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.16.1.tgz",
+      "integrity": "sha512-r/1+GzIW1D5zrP4tNrfW+3y4vqD935WBXSc8X/wm23QTY9aJO9Lw6PEdzpYCEY+SOklIFKaJYUAq/Nvgm/9ryw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1",
+        "gifwrap": "^0.9.2",
+        "omggif": "^1.0.9"
+      }
+    },
+    "@jimp/jpeg": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.16.1.tgz",
+      "integrity": "sha512-8352zrdlCCLFdZ/J+JjBslDvml+fS3Z8gttdml0We759PnnZGqrnPRhkOEOJbNUlE+dD4ckLeIe6NPxlS/7U+w==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1",
+        "jpeg-js": "0.4.2"
+      }
+    },
+    "@jimp/plugin-blit": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.16.1.tgz",
+      "integrity": "sha512-fKFNARm32RoLSokJ8WZXHHH2CGzz6ire2n1Jh6u+XQLhk9TweT1DcLHIXwQMh8oR12KgjbgsMGvrMVlVknmOAg==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-blur": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.16.1.tgz",
+      "integrity": "sha512-1WhuLGGj9MypFKRcPvmW45ht7nXkOKu+lg3n2VBzIB7r4kKNVchuI59bXaCYQumOLEqVK7JdB4glaDAbCQCLyw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-circle": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.16.1.tgz",
+      "integrity": "sha512-JK7yi1CIU7/XL8hdahjcbGA3V7c+F+Iw+mhMQhLEi7Q0tCnZ69YJBTamMiNg3fWPVfMuvWJJKOBRVpwNTuaZRg==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-color": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.16.1.tgz",
+      "integrity": "sha512-9yQttBAO5SEFj7S6nJK54f+1BnuBG4c28q+iyzm1JjtnehjqMg6Ljw4gCSDCvoCQ3jBSYHN66pmwTV74SU1B7A==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1",
+        "tinycolor2": "^1.4.1"
+      }
+    },
+    "@jimp/plugin-contain": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.16.1.tgz",
+      "integrity": "sha512-44F3dUIjBDHN+Ym/vEfg+jtjMjAqd2uw9nssN67/n4FdpuZUVs7E7wadKY1RRNuJO+WgcD5aDQcsvurXMETQTg==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-cover": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.16.1.tgz",
+      "integrity": "sha512-YztWCIldBAVo0zxcQXR+a/uk3/TtYnpKU2CanOPJ7baIuDlWPsG+YE4xTsswZZc12H9Kl7CiziEbDtvF9kwA/Q==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-crop": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.16.1.tgz",
+      "integrity": "sha512-UQdva9oQzCVadkyo3T5Tv2CUZbf0klm2cD4cWMlASuTOYgaGaFHhT9st+kmfvXjKL8q3STkBu/zUPV6PbuV3ew==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-displace": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.16.1.tgz",
+      "integrity": "sha512-iVAWuz2+G6Heu8gVZksUz+4hQYpR4R0R/RtBzpWEl8ItBe7O6QjORAkhxzg+WdYLL2A/Yd4ekTpvK0/qW8hTVw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-dither": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.16.1.tgz",
+      "integrity": "sha512-tADKVd+HDC9EhJRUDwMvzBXPz4GLoU6s5P7xkVq46tskExYSptgj5713J5Thj3NMgH9Rsqu22jNg1H/7tr3V9Q==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-fisheye": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.16.1.tgz",
+      "integrity": "sha512-BWHnc5hVobviTyIRHhIy9VxI1ACf4CeSuCfURB6JZm87YuyvgQh5aX5UDKtOz/3haMHXBLP61ZBxlNpMD8CG4A==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-flip": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.16.1.tgz",
+      "integrity": "sha512-KdxTf0zErfZ8DyHkImDTnQBuHby+a5YFdoKI/G3GpBl3qxLBvC+PWkS2F/iN3H7wszP7/TKxTEvWL927pypT0w==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-gaussian": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.16.1.tgz",
+      "integrity": "sha512-u9n4wjskh3N1mSqketbL6tVcLU2S5TEaFPR40K6TDv4phPLZALi1Of7reUmYpVm8mBDHt1I6kGhuCJiWvzfGyg==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-invert": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.16.1.tgz",
+      "integrity": "sha512-2DKuyVXANH8WDpW9NG+PYFbehzJfweZszFYyxcaewaPLN0GxvxVLOGOPP1NuUTcHkOdMFbE0nHDuB7f+sYF/2w==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-mask": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.16.1.tgz",
+      "integrity": "sha512-snfiqHlVuj4bSFS0v96vo2PpqCDMe4JB+O++sMo5jF5mvGcGL6AIeLo8cYqPNpdO6BZpBJ8MY5El0Veckhr39Q==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-normalize": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.16.1.tgz",
+      "integrity": "sha512-dOQfIOvGLKDKXPU8xXWzaUeB0nvkosHw6Xg1WhS1Z5Q0PazByhaxOQkSKgUryNN/H+X7UdbDvlyh/yHf3ITRaw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-print": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.16.1.tgz",
+      "integrity": "sha512-ceWgYN40jbN4cWRxixym+csyVymvrryuKBQ+zoIvN5iE6OyS+2d7Mn4zlNgumSczb9GGyZZESIgVcBDA1ezq0Q==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1",
+        "load-bmfont": "^1.4.0"
+      }
+    },
+    "@jimp/plugin-resize": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.16.1.tgz",
+      "integrity": "sha512-u4JBLdRI7dargC04p2Ha24kofQBk3vhaf0q8FwSYgnCRwxfvh2RxvhJZk9H7Q91JZp6wgjz/SjvEAYjGCEgAwQ==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-rotate": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.16.1.tgz",
+      "integrity": "sha512-ZUU415gDQ0VjYutmVgAYYxC9Og9ixu2jAGMCU54mSMfuIlmohYfwARQmI7h4QB84M76c9hVLdONWjuo+rip/zg==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-scale": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.16.1.tgz",
+      "integrity": "sha512-jM2QlgThIDIc4rcyughD5O7sOYezxdafg/2Xtd1csfK3z6fba3asxDwthqPZAgitrLgiKBDp6XfzC07Y/CefUw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-shadow": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.16.1.tgz",
+      "integrity": "sha512-MeD2Is17oKzXLnsphAa1sDstTu6nxscugxAEk3ji0GV1FohCvpHBcec0nAq6/czg4WzqfDts+fcPfC79qWmqrA==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugin-threshold": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.16.1.tgz",
+      "integrity": "sha512-iGW8U/wiCSR0+6syrPioVGoSzQFt4Z91SsCRbgNKTAk7D+XQv6OI78jvvYg4o0c2FOlwGhqz147HZV5utoSLxA==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1"
+      }
+    },
+    "@jimp/plugins": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.16.1.tgz",
+      "integrity": "sha512-c+lCqa25b+4q6mJZSetlxhMoYuiltyS+ValLzdwK/47+aYsq+kcJNl+TuxIEKf59yr9+5rkbpsPkZHLF/V7FFA==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/plugin-blit": "^0.16.1",
+        "@jimp/plugin-blur": "^0.16.1",
+        "@jimp/plugin-circle": "^0.16.1",
+        "@jimp/plugin-color": "^0.16.1",
+        "@jimp/plugin-contain": "^0.16.1",
+        "@jimp/plugin-cover": "^0.16.1",
+        "@jimp/plugin-crop": "^0.16.1",
+        "@jimp/plugin-displace": "^0.16.1",
+        "@jimp/plugin-dither": "^0.16.1",
+        "@jimp/plugin-fisheye": "^0.16.1",
+        "@jimp/plugin-flip": "^0.16.1",
+        "@jimp/plugin-gaussian": "^0.16.1",
+        "@jimp/plugin-invert": "^0.16.1",
+        "@jimp/plugin-mask": "^0.16.1",
+        "@jimp/plugin-normalize": "^0.16.1",
+        "@jimp/plugin-print": "^0.16.1",
+        "@jimp/plugin-resize": "^0.16.1",
+        "@jimp/plugin-rotate": "^0.16.1",
+        "@jimp/plugin-scale": "^0.16.1",
+        "@jimp/plugin-shadow": "^0.16.1",
+        "@jimp/plugin-threshold": "^0.16.1",
+        "timm": "^1.6.1"
+      }
+    },
+    "@jimp/png": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.16.1.tgz",
+      "integrity": "sha512-iyWoCxEBTW0OUWWn6SveD4LePW89kO7ZOy5sCfYeDM/oTPLpR8iMIGvZpZUz1b8kvzFr27vPst4E5rJhGjwsdw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/utils": "^0.16.1",
+        "pngjs": "^3.3.3"
+      }
+    },
+    "@jimp/tiff": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.16.1.tgz",
+      "integrity": "sha512-3K3+xpJS79RmSkAvFMgqY5dhSB+/sxhwTFA9f4AVHUK0oKW+u6r52Z1L0tMXHnpbAdR9EJ+xaAl2D4x19XShkQ==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "utif": "^2.0.1"
+      }
+    },
+    "@jimp/types": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.16.1.tgz",
+      "integrity": "sha512-g1w/+NfWqiVW4CaXSJyD28JQqZtm2eyKMWPhBBDCJN9nLCN12/Az0WFF3JUAktzdsEC2KRN2AqB1a2oMZBNgSQ==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/bmp": "^0.16.1",
+        "@jimp/gif": "^0.16.1",
+        "@jimp/jpeg": "^0.16.1",
+        "@jimp/png": "^0.16.1",
+        "@jimp/tiff": "^0.16.1",
+        "timm": "^1.6.1"
+      }
+    },
+    "@jimp/utils": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.16.1.tgz",
+      "integrity": "sha512-8fULQjB0x4LzUSiSYG6ZtQl355sZjxbv8r9PPAuYHzS9sGiSHJQavNqK/nKnpDsVkU88/vRGcE7t3nMU0dEnVw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "regenerator-runtime": "^0.13.3"
+      }
+    },
     "@types/node": {
       "version": "14.11.2",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz",
@@ -24,6 +353,50 @@
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz",
       "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g=="
     },
+    "ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "requires": {
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      }
+    },
+    "any-base": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz",
+      "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg=="
+    },
+    "asn1": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
+      "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==",
+      "requires": {
+        "safer-buffer": "~2.1.0"
+      }
+    },
+    "assert-plus": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
+      "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+    },
+    "asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
+    },
+    "aws-sign2": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
+      "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
+    },
+    "aws4": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz",
+      "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA=="
+    },
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
@@ -34,6 +407,19 @@
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
       "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
     },
+    "bcrypt-pbkdf": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
+      "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
+      "requires": {
+        "tweetnacl": "^0.14.3"
+      }
+    },
+    "bignumber.js": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-2.4.0.tgz",
+      "integrity": "sha1-g4qZLan51zfg9LLbC+YrsJ3Qxeg="
+    },
     "bl": {
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
@@ -44,6 +430,11 @@
         "readable-stream": "^3.4.0"
       }
     },
+    "bmp-js": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
+      "integrity": "sha1-4Fpj95amwf8l9Hcex62twUjAcjM="
+    },
     "brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -67,21 +458,57 @@
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
       "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
     },
+    "buffer-equal": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz",
+      "integrity": "sha1-kbx0sR6kBbyRa8aqkI+q+ltKrEs="
+    },
+    "caseless": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
+      "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
+    },
     "chownr": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
       "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
     },
+    "combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "requires": {
+        "delayed-stream": "~1.0.0"
+      }
+    },
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
     },
+    "core-js": {
+      "version": "2.6.12",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
+      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ=="
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+    },
     "csv-writer": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz",
       "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g=="
     },
+    "dashdash": {
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
+      "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
+      "requires": {
+        "assert-plus": "^1.0.0"
+      }
+    },
     "debug": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
@@ -90,11 +517,30 @@
         "ms": "2.1.2"
       }
     },
+    "delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
+    },
     "devtools-protocol": {
       "version": "0.0.799653",
       "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz",
       "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg=="
     },
+    "dom-walk": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz",
+      "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w=="
+    },
+    "ecc-jsbn": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
+      "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=",
+      "requires": {
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.1.0"
+      }
+    },
     "end-of-stream": {
       "version": "1.4.4",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
@@ -103,6 +549,21 @@
         "once": "^1.4.0"
       }
     },
+    "es6-promise": {
+      "version": "3.3.1",
+      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz",
+      "integrity": "sha1-oIzd6EzNvzTQJ6FFG8kdS80ophM="
+    },
+    "exif-parser": {
+      "version": "0.1.12",
+      "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz",
+      "integrity": "sha1-WKnS1ywCwfbwKg70qRZicrd2CSI="
+    },
+    "extend": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+      "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
+    },
     "extract-zip": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
@@ -114,6 +575,21 @@
         "yauzl": "^2.10.0"
       }
     },
+    "extsprintf": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
+      "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
+    },
+    "fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
+    },
+    "fast-json-stable-stringify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="
+    },
     "fd-slicer": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
@@ -122,6 +598,11 @@
         "pend": "~1.2.0"
       }
     },
+    "file-type": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/file-type/-/file-type-9.0.0.tgz",
+      "integrity": "sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw=="
+    },
     "find-up": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
@@ -131,6 +612,21 @@
         "path-exists": "^4.0.0"
       }
     },
+    "forever-agent": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
+      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
+    },
+    "form-data": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz",
+      "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==",
+      "requires": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.6",
+        "mime-types": "^2.1.12"
+      }
+    },
     "fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
@@ -149,6 +645,23 @@
         "pump": "^3.0.0"
       }
     },
+    "getpass": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
+      "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
+      "requires": {
+        "assert-plus": "^1.0.0"
+      }
+    },
+    "gifwrap": {
+      "version": "0.9.2",
+      "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.9.2.tgz",
+      "integrity": "sha512-fcIswrPaiCDAyO8xnWvHSZdWChjKXUanKKpAiWWJ/UTkEi/aYKn5+90e7DE820zbEaVR9CE2y4z9bzhQijZ0BA==",
+      "requires": {
+        "image-q": "^1.1.1",
+        "omggif": "^1.0.10"
+      }
+    },
     "glob": {
       "version": "7.1.6",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
@@ -162,6 +675,39 @@
         "path-is-absolute": "^1.0.0"
       }
     },
+    "global": {
+      "version": "4.3.2",
+      "resolved": "https://registry.npmjs.org/global/-/global-4.3.2.tgz",
+      "integrity": "sha1-52mJJopsdMOJCLEwWxD8DjlOnQ8=",
+      "requires": {
+        "min-document": "^2.19.0",
+        "process": "~0.5.1"
+      }
+    },
+    "har-schema": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
+      "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
+    },
+    "har-validator": {
+      "version": "5.1.5",
+      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
+      "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
+      "requires": {
+        "ajv": "^6.12.3",
+        "har-schema": "^2.0.0"
+      }
+    },
+    "http-signature": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
+      "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
+      "requires": {
+        "assert-plus": "^1.0.0",
+        "jsprim": "^1.2.2",
+        "sshpk": "^1.7.0"
+      }
+    },
     "https-proxy-agent": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
@@ -176,6 +722,11 @@
       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
       "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
     },
+    "image-q": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/image-q/-/image-q-1.1.1.tgz",
+      "integrity": "sha1-/IQJlmRGC5DKhi2TALa/u7+/gFY="
+    },
     "inflight": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -190,6 +741,94 @@
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
+    "ip-regex": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-1.0.3.tgz",
+      "integrity": "sha1-3FiQdvZZ9BnCIgOaMzFvHHOH7/0="
+    },
+    "is-function": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.2.tgz",
+      "integrity": "sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ=="
+    },
+    "is-plain-obj": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
+      "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4="
+    },
+    "is-typedarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
+      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
+    },
+    "isstream": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
+      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
+    },
+    "jimp": {
+      "version": "0.16.1",
+      "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.16.1.tgz",
+      "integrity": "sha512-+EKVxbR36Td7Hfd23wKGIeEyHbxShZDX6L8uJkgVW3ESA9GiTEPK08tG1XI2r/0w5Ch0HyJF5kPqF9K7EmGjaw==",
+      "requires": {
+        "@babel/runtime": "^7.7.2",
+        "@jimp/custom": "^0.16.1",
+        "@jimp/plugins": "^0.16.1",
+        "@jimp/types": "^0.16.1",
+        "regenerator-runtime": "^0.13.3"
+      }
+    },
+    "jpeg-js": {
+      "version": "0.4.2",
+      "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.2.tgz",
+      "integrity": "sha512-+az2gi/hvex7eLTMTlbRLOhH6P6WFdk2ITI8HJsaH2VqYO0I594zXSYEP+tf4FW+8Cy68ScDXoAsQdyQanv3sw=="
+    },
+    "jsbn": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
+      "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
+    },
+    "json-schema": {
+      "version": "0.2.3",
+      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
+      "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
+    },
+    "json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
+    },
+    "json-stringify-safe": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
+      "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
+    },
+    "jsprim": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
+      "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
+      "requires": {
+        "assert-plus": "1.0.0",
+        "extsprintf": "1.3.0",
+        "json-schema": "0.2.3",
+        "verror": "1.10.0"
+      }
+    },
+    "load-bmfont": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz",
+      "integrity": "sha512-8UyQoYmdRDy81Brz6aLAUhfZLwr5zV0L3taTQ4hju7m6biuwiWiJXjPhBJxbUQJA8PrkvJ/7Enqmwk2sM14soA==",
+      "requires": {
+        "buffer-equal": "0.0.1",
+        "mime": "^1.3.4",
+        "parse-bmfont-ascii": "^1.0.3",
+        "parse-bmfont-binary": "^1.0.5",
+        "parse-bmfont-xml": "^1.1.4",
+        "phin": "^2.9.1",
+        "xhr": "^2.0.1",
+        "xtend": "^4.0.0"
+      }
+    },
     "locate-path": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
@@ -198,6 +837,80 @@
         "p-locate": "^4.1.0"
       }
     },
+    "merge-img": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/merge-img/-/merge-img-2.1.3.tgz",
+      "integrity": "sha512-PA8caQPleTulKV7s2CXGvzTPxVoT5wkFCB0jf0RVzIYG+job08wibDA9lMYlpcHjCOvup3p1UPk1JgTwgJnu6g==",
+      "requires": {
+        "is-plain-obj": "^1.1.0",
+        "jimp": "0.2.27"
+      },
+      "dependencies": {
+        "bmp-js": {
+          "version": "0.0.1",
+          "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.0.1.tgz",
+          "integrity": "sha1-WtAUcJnROp84qnuZrx1ueGZu038="
+        },
+        "file-type": {
+          "version": "3.9.0",
+          "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz",
+          "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek="
+        },
+        "jimp": {
+          "version": "0.2.27",
+          "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.2.27.tgz",
+          "integrity": "sha1-Qe9Qgti2MgHVR0fgT+i8rLryVHQ=",
+          "requires": {
+            "bignumber.js": "^2.1.0",
+            "bmp-js": "0.0.1",
+            "es6-promise": "^3.0.2",
+            "exif-parser": "^0.1.9",
+            "file-type": "^3.1.0",
+            "jpeg-js": "^0.2.0",
+            "load-bmfont": "^1.2.3",
+            "mime": "^1.3.4",
+            "pixelmatch": "^4.0.0",
+            "pngjs": "^3.0.0",
+            "read-chunk": "^1.0.1",
+            "request": "^2.65.0",
+            "stream-to-buffer": "^0.1.0",
+            "tinycolor2": "^1.1.2",
+            "url-regex": "^3.0.0"
+          }
+        },
+        "jpeg-js": {
+          "version": "0.2.0",
+          "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.2.0.tgz",
+          "integrity": "sha1-U+RI7J0mPmgyZkZ+lELSxaLvVII="
+        }
+      }
+    },
+    "mime": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
+    },
+    "mime-db": {
+      "version": "1.44.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
+      "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg=="
+    },
+    "mime-types": {
+      "version": "2.1.27",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz",
+      "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==",
+      "requires": {
+        "mime-db": "1.44.0"
+      }
+    },
+    "min-document": {
+      "version": "2.19.0",
+      "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz",
+      "integrity": "sha1-e9KC4/WELtKVu3SM3Z8f+iyCRoU=",
+      "requires": {
+        "dom-walk": "^0.1.0"
+      }
+    },
     "minimatch": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
@@ -206,6 +919,19 @@
         "brace-expansion": "^1.1.7"
       }
     },
+    "minimist": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+    },
+    "mkdirp": {
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+      "requires": {
+        "minimist": "^1.2.5"
+      }
+    },
     "mkdirp-classic": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
@@ -216,6 +942,16 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
+    "oauth-sign": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
+      "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
+    },
+    "omggif": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz",
+      "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw=="
+    },
     "once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@@ -245,6 +981,35 @@
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
       "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
     },
+    "pako": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
+    },
+    "parse-bmfont-ascii": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz",
+      "integrity": "sha1-Eaw8P/WPfCAgqyJ2kHkQjU36AoU="
+    },
+    "parse-bmfont-binary": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz",
+      "integrity": "sha1-0Di0dtPp3Z2x4RoLDlOiJ5K2kAY="
+    },
+    "parse-bmfont-xml": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/parse-bmfont-xml/-/parse-bmfont-xml-1.1.4.tgz",
+      "integrity": "sha512-bjnliEOmGv3y1aMEfREMBJ9tfL3WR0i0CKPj61DnSLaoxWR3nLrsQrEbCId/8rF4NyRF0cCqisSVXyQYWM+mCQ==",
+      "requires": {
+        "xml-parse-from-string": "^1.0.0",
+        "xml2js": "^0.4.5"
+      }
+    },
+    "parse-headers": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.3.tgz",
+      "integrity": "sha512-QhhZ+DCCit2Coi2vmAKbq5RGTRcQUOE2+REgv8vdyu7MnYx2eZztegqtTx99TZ86GTIwqiy3+4nQTWZ2tgmdCA=="
+    },
     "path-exists": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -260,6 +1025,24 @@
       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
       "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
     },
+    "performance-now": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
+      "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
+    },
+    "phin": {
+      "version": "2.9.3",
+      "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz",
+      "integrity": "sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA=="
+    },
+    "pixelmatch": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz",
+      "integrity": "sha1-j0fc7FARtHe2fbA8JDvB8wheiFQ=",
+      "requires": {
+        "pngjs": "^3.0.0"
+      }
+    },
     "pkg-dir": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
@@ -268,6 +1051,16 @@
         "find-up": "^4.0.0"
       }
     },
+    "pngjs": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz",
+      "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w=="
+    },
+    "process": {
+      "version": "0.5.2",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.5.2.tgz",
+      "integrity": "sha1-FjjYqONML0QKkduVq5rrZ3/Bhc8="
+    },
     "progress": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
@@ -278,6 +1071,11 @@
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
+    "psl": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
+      "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
+    },
     "pump": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -287,6 +1085,11 @@
         "once": "^1.3.1"
       }
     },
+    "punycode": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
+    },
     "puppeteer": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz",
@@ -305,6 +1108,336 @@
         "ws": "^7.2.3"
       }
     },
+    "puppeteer-full-page-screenshot": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/puppeteer-full-page-screenshot/-/puppeteer-full-page-screenshot-1.0.5.tgz",
+      "integrity": "sha512-OHpPb8spKfCNNnKbxX91fLTk0R7XXv9L47/q+R6vpLUwefYdyY823BfWoxeNxAIFZpk6r3XPlkaI1hfsCmelvQ==",
+      "requires": {
+        "jimp": "^0.6.4",
+        "merge-img": "^2.1.3"
+      },
+      "dependencies": {
+        "@jimp/bmp": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.6.8.tgz",
+          "integrity": "sha512-uxVgSkI62uAzk5ZazYHEHBehow590WAkLKmDXLzkr/XP/Hv2Fx1T4DKwJ/15IY5ktq5VAhAUWGXTyd8KWFsx7w==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "bmp-js": "^0.1.0",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/core": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.6.8.tgz",
+          "integrity": "sha512-JOFqBBcSNiDiMZJFr6OJqC6viXj5NVBQISua0eacoYvo4YJtTajOIxC4MqWyUmGrDpRMZBR8QhSsIOwsFrdROA==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "any-base": "^1.1.0",
+            "buffer": "^5.2.0",
+            "core-js": "^2.5.7",
+            "exif-parser": "^0.1.12",
+            "file-type": "^9.0.0",
+            "load-bmfont": "^1.3.1",
+            "mkdirp": "0.5.1",
+            "phin": "^2.9.1",
+            "pixelmatch": "^4.0.2",
+            "tinycolor2": "^1.4.1"
+          }
+        },
+        "@jimp/custom": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.6.8.tgz",
+          "integrity": "sha512-FrYlzZRVXP2vuVwd7Nc2dlK+iZk4g6IaT1Ib8Z6vU5Kkwlt83FJIPJ2UUFABf3bF5big0wkk8ZUihWxE4Nzdng==",
+          "requires": {
+            "@jimp/core": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/gif": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.6.8.tgz",
+          "integrity": "sha512-yyOlujjQcgz9zkjM5ihZDEppn9d1brJ7jQHP5rAKmqep0G7FU1D0AKcV+Ql18RhuI/CgWs10wAVcrQpmLnu4Yw==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7",
+            "omggif": "^1.0.9"
+          }
+        },
+        "@jimp/jpeg": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.6.8.tgz",
+          "integrity": "sha512-rGtXbYpFXAn471qLpTGvhbBMNHJo5KiufN+vC5AWyufntmkt5f0Ox2Cx4ijuBMDtirZchxbMLtrfGjznS4L/ew==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7",
+            "jpeg-js": "^0.3.4"
+          }
+        },
+        "@jimp/plugin-blit": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.6.8.tgz",
+          "integrity": "sha512-7Tl6YpKTSpvwQbnGNhsfX2zyl3jRVVopd276Y2hF2zpDz9Bycow7NdfNU/4Nx1jaf96X6uWOtSVINcQ7rGd47w==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-blur": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.6.8.tgz",
+          "integrity": "sha512-NpZCMKxXHLDQsX9zPlWtpMA660DQStY6/z8ZetyxCDbqrLe9YCXpeR4MNhdJdABIiwTm1W5FyFF4kp81PHJx3Q==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-color": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.6.8.tgz",
+          "integrity": "sha512-jjFyU0zNmGOH2rjzHuOMU4kaia0oo82s/7UYfn5h7OUkmUZTd6Do3ZSK1PiXA7KR+s4B76/Omm6Doh/0SGb7BQ==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7",
+            "tinycolor2": "^1.4.1"
+          }
+        },
+        "@jimp/plugin-contain": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.6.8.tgz",
+          "integrity": "sha512-p/P2wCXhAzbmEgXvGsvmxLmbz45feF6VpR4m9suPSOr8PC/i/XvTklTqYEUidYYAft4vHgsYJdS74HKSMnH8lw==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-cover": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.6.8.tgz",
+          "integrity": "sha512-2PvWgk+PJfRsfWDI1G8Fpjrsu0ZlpNyZxO2+fqWlVo6y/y2gP4v08FqvbkcqSjNlOu2IDWIFXpgyU0sTINWZLg==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-crop": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.6.8.tgz",
+          "integrity": "sha512-CbrcpWE2xxPK1n/JoTXzhRUhP4mO07mTWaSavenCg664oQl/9XCtL+A0FekuNHzIvn4myEqvkiTwN7FsbunS/Q==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-displace": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.6.8.tgz",
+          "integrity": "sha512-RmV2bPxoPE6mrPxtYSPtHxm2cGwBQr5a2p+9gH6SPy+eUMrbGjbvjwKNfXWUYD0leML+Pt5XOmAS9pIROmuruQ==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-dither": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.6.8.tgz",
+          "integrity": "sha512-x6V/qjxe+xypjpQm7GbiMNqci1EW5UizrcebOhHr8AHijOEqHd2hjXh5f6QIGfrkTFelc4/jzq1UyCsYntqz9Q==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-flip": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.6.8.tgz",
+          "integrity": "sha512-4il6Da6G39s9MyWBEee4jztEOUGJ40E6OlPjkMrdpDNvge6hYEAB31BczTYBP/CEY74j4LDSoY5LbcU4kv06yA==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-gaussian": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.6.8.tgz",
+          "integrity": "sha512-pVOblmjv7stZjsqloi4YzHVwAPXKGdNaHPhp4KP4vj41qtc6Hxd9z/+VWGYRTunMFac84gUToe0UKIXd6GhoKw==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-invert": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.6.8.tgz",
+          "integrity": "sha512-11zuLiXDHr6tFv4U8aieXqNXQEKbDbSBG/h+X62gGTNFpyn8EVPpncHhOqrAFtZUaPibBqMFlNJ15SzwC7ExsQ==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-mask": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.6.8.tgz",
+          "integrity": "sha512-hZJ0OiKGJyv7hDSATwJDkunB1Ie80xJnONMgpUuUseteK45YeYNBOiZVUe8vum8QI1UwavgBzcvQ9u4fcgXc9g==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-normalize": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.6.8.tgz",
+          "integrity": "sha512-Q4oYhU+sSyTJI7pMZlg9/mYh68ujLfOxXzQGEXuw0sHGoGQs3B0Jw7jmzGa6pIS06Hup5hD2Zuh1ppvMdjJBfQ==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-print": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.6.8.tgz",
+          "integrity": "sha512-2aokejGn4Drv1FesnZGqh5KEq0FQtR0drlmtyZrBH+r9cx7hh0Qgf4D1BOTDEgXkfSSngjGRjKKRW/fwOrVXYw==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7",
+            "load-bmfont": "^1.4.0"
+          }
+        },
+        "@jimp/plugin-resize": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.6.8.tgz",
+          "integrity": "sha512-27nPh8L1YWsxtfmV/+Ub5dOTpXyC0HMF2cu52RQSCYxr+Lm1+23dJF70AF1poUbUe+FWXphwuUxQzjBJza9UoA==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-rotate": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.6.8.tgz",
+          "integrity": "sha512-GbjETvL05BDoLdszNUV4Y0yLkHf177MnqGqilA113LIvx9aD0FtUopGXYfRGVvmtTOTouoaGJUc+K6qngvKxww==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugin-scale": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.6.8.tgz",
+          "integrity": "sha512-GzIYWR/oCUK2jAwku23zt19V1ssaEU4pL0x2XsLNKuuJEU6DvEytJyTMXCE7OLG/MpDBQcQclJKHgiyQm5gIOQ==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7"
+          }
+        },
+        "@jimp/plugins": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.6.8.tgz",
+          "integrity": "sha512-fMcTI72Vn/Lz6JftezTURmyP5ml/xGMe0Ljx2KRJ85IWyP33vDmGIUuutFiBEbh2+y7lRT+aTSmjs0QGa/xTmQ==",
+          "requires": {
+            "@jimp/plugin-blit": "^0.6.8",
+            "@jimp/plugin-blur": "^0.6.8",
+            "@jimp/plugin-color": "^0.6.8",
+            "@jimp/plugin-contain": "^0.6.8",
+            "@jimp/plugin-cover": "^0.6.8",
+            "@jimp/plugin-crop": "^0.6.8",
+            "@jimp/plugin-displace": "^0.6.8",
+            "@jimp/plugin-dither": "^0.6.8",
+            "@jimp/plugin-flip": "^0.6.8",
+            "@jimp/plugin-gaussian": "^0.6.8",
+            "@jimp/plugin-invert": "^0.6.8",
+            "@jimp/plugin-mask": "^0.6.8",
+            "@jimp/plugin-normalize": "^0.6.8",
+            "@jimp/plugin-print": "^0.6.8",
+            "@jimp/plugin-resize": "^0.6.8",
+            "@jimp/plugin-rotate": "^0.6.8",
+            "@jimp/plugin-scale": "^0.6.8",
+            "core-js": "^2.5.7",
+            "timm": "^1.6.1"
+          }
+        },
+        "@jimp/png": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.6.8.tgz",
+          "integrity": "sha512-JHHg/BZ7KDtHQrcG+a7fztw45rdf7okL/YwkN4qU5FH7Fcrp41nX5QnRviDtD9hN+GaNC7kvjvcqRAxW25qjew==",
+          "requires": {
+            "@jimp/utils": "^0.6.8",
+            "core-js": "^2.5.7",
+            "pngjs": "^3.3.3"
+          }
+        },
+        "@jimp/tiff": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.6.8.tgz",
+          "integrity": "sha512-iWHbxd+0IKWdJyJ0HhoJCGYmtjPBOusz1z1HT/DnpePs/Lo3TO4d9ALXqYfUkyG74ZK5jULZ69KLtwuhuJz1bg==",
+          "requires": {
+            "core-js": "^2.5.7",
+            "utif": "^2.0.1"
+          }
+        },
+        "@jimp/types": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.6.8.tgz",
+          "integrity": "sha512-vCZ/Cp2osy69VP21XOBACfHI5HeR60Rfd4Jidj4W73UL+HrFWOtyQiJ7hlToyu1vI5mR/NsUQpzyQvz56ADm5A==",
+          "requires": {
+            "@jimp/bmp": "^0.6.8",
+            "@jimp/gif": "^0.6.8",
+            "@jimp/jpeg": "^0.6.8",
+            "@jimp/png": "^0.6.8",
+            "@jimp/tiff": "^0.6.8",
+            "core-js": "^2.5.7",
+            "timm": "^1.6.1"
+          }
+        },
+        "@jimp/utils": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.6.8.tgz",
+          "integrity": "sha512-7RDfxQ2C/rarNG9iso5vmnKQbcvlQjBIlF/p7/uYj72WeZgVCB+5t1fFBKJSU4WhniHX4jUMijK+wYGE3Y3bGw==",
+          "requires": {
+            "core-js": "^2.5.7"
+          }
+        },
+        "jimp": {
+          "version": "0.6.8",
+          "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.6.8.tgz",
+          "integrity": "sha512-F7emeG7Hp61IM8VFbNvWENLTuHe0ghizWPuP4JS9ujx2r5mCVYEd/zdaz6M2M42ZdN41blxPajLWl9FXo7Mr2Q==",
+          "requires": {
+            "@jimp/custom": "^0.6.8",
+            "@jimp/plugins": "^0.6.8",
+            "@jimp/types": "^0.6.8",
+            "core-js": "^2.5.7",
+            "regenerator-runtime": "^0.13.3"
+          }
+        },
+        "jpeg-js": {
+          "version": "0.3.7",
+          "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.7.tgz",
+          "integrity": "sha512-9IXdWudL61npZjvLuVe/ktHiA41iE8qFyLB+4VDTblEsWBzeg8WQTlktdUK4CdncUqtUgUg0bbOmTE2bKBKaBQ=="
+        },
+        "minimist": {
+          "version": "0.0.8",
+          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
+          "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+        },
+        "mkdirp": {
+          "version": "0.5.1",
+          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+          "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+          "requires": {
+            "minimist": "0.0.8"
+          }
+        }
+      }
+    },
+    "qs": {
+      "version": "6.5.2",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
+      "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
+    },
+    "read-chunk": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-1.0.1.tgz",
+      "integrity": "sha1-X2jKswfmY/GZk1J9m1icrORmEZQ="
+    },
     "readable-stream": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -315,6 +1448,45 @@
         "util-deprecate": "^1.0.1"
       }
     },
+    "regenerator-runtime": {
+      "version": "0.13.7",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz",
+      "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew=="
+    },
+    "request": {
+      "version": "2.88.2",
+      "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
+      "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
+      "requires": {
+        "aws-sign2": "~0.7.0",
+        "aws4": "^1.8.0",
+        "caseless": "~0.12.0",
+        "combined-stream": "~1.0.6",
+        "extend": "~3.0.2",
+        "forever-agent": "~0.6.1",
+        "form-data": "~2.3.2",
+        "har-validator": "~5.1.3",
+        "http-signature": "~1.2.0",
+        "is-typedarray": "~1.0.0",
+        "isstream": "~0.1.2",
+        "json-stringify-safe": "~5.0.1",
+        "mime-types": "~2.1.19",
+        "oauth-sign": "~0.9.0",
+        "performance-now": "^2.1.0",
+        "qs": "~6.5.2",
+        "safe-buffer": "^5.1.2",
+        "tough-cookie": "~2.5.0",
+        "tunnel-agent": "^0.6.0",
+        "uuid": "^3.3.2"
+      },
+      "dependencies": {
+        "uuid": {
+          "version": "3.4.0",
+          "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
+          "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
+        }
+      }
+    },
     "rimraf": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -328,6 +1500,45 @@
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
       "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
     },
+    "safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+    },
+    "sax": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+    },
+    "sshpk": {
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
+      "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==",
+      "requires": {
+        "asn1": "~0.2.3",
+        "assert-plus": "^1.0.0",
+        "bcrypt-pbkdf": "^1.0.0",
+        "dashdash": "^1.12.0",
+        "ecc-jsbn": "~0.1.1",
+        "getpass": "^0.1.1",
+        "jsbn": "~0.1.0",
+        "safer-buffer": "^2.0.2",
+        "tweetnacl": "~0.14.0"
+      }
+    },
+    "stream-to": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/stream-to/-/stream-to-0.2.2.tgz",
+      "integrity": "sha1-hDBgmNhf25kLn6MAsbPM9V6O8B0="
+    },
+    "stream-to-buffer": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/stream-to-buffer/-/stream-to-buffer-0.1.0.tgz",
+      "integrity": "sha1-JnmdkDqyAlyb1VCsRxcbAPjdgKk=",
+      "requires": {
+        "stream-to": "~0.2.0"
+      }
+    },
     "string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -364,6 +1575,38 @@
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
     },
+    "timm": {
+      "version": "1.7.1",
+      "resolved": "https://registry.npmjs.org/timm/-/timm-1.7.1.tgz",
+      "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw=="
+    },
+    "tinycolor2": {
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.4.2.tgz",
+      "integrity": "sha512-vJhccZPs965sV/L2sU4oRQVAos0pQXwsvTLkWYdqJ+a8Q5kPFzJTuOFwy7UniPli44NKQGAglksjvOcpo95aZA=="
+    },
+    "tough-cookie": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
+      "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
+      "requires": {
+        "psl": "^1.1.28",
+        "punycode": "^2.1.1"
+      }
+    },
+    "tunnel-agent": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
+      "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
+      "requires": {
+        "safe-buffer": "^5.0.1"
+      }
+    },
+    "tweetnacl": {
+      "version": "0.14.5",
+      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
+      "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
+    },
     "unbzip2-stream": {
       "version": "1.4.3",
       "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
@@ -373,6 +1616,30 @@
         "through": "^2.3.8"
       }
     },
+    "uri-js": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz",
+      "integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==",
+      "requires": {
+        "punycode": "^2.1.0"
+      }
+    },
+    "url-regex": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/url-regex/-/url-regex-3.2.0.tgz",
+      "integrity": "sha1-260eDJ4p4QXdCx8J9oYvf9tIJyQ=",
+      "requires": {
+        "ip-regex": "^1.0.1"
+      }
+    },
+    "utif": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/utif/-/utif-2.0.1.tgz",
+      "integrity": "sha512-Z/S1fNKCicQTf375lIP9G8Sa1H/phcysstNrrSdZKj1f9g58J4NMgb5IgiEZN9/nLMPDwF0W7hdOe9Qq2IYoLg==",
+      "requires": {
+        "pako": "^1.0.5"
+      }
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -383,6 +1650,16 @@
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz",
       "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg=="
     },
+    "verror": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
+      "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
+      "requires": {
+        "assert-plus": "^1.0.0",
+        "core-util-is": "1.0.2",
+        "extsprintf": "^1.2.0"
+      }
+    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -393,6 +1670,41 @@
       "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz",
       "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA=="
     },
+    "xhr": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.5.0.tgz",
+      "integrity": "sha512-4nlO/14t3BNUZRXIXfXe+3N6w3s1KoxcJUUURctd64BLRe67E4gRwp4PjywtDY72fXpZ1y6Ch0VZQRY/gMPzzQ==",
+      "requires": {
+        "global": "~4.3.0",
+        "is-function": "^1.0.1",
+        "parse-headers": "^2.0.0",
+        "xtend": "^4.0.0"
+      }
+    },
+    "xml-parse-from-string": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz",
+      "integrity": "sha1-qQKekp09vN7RafPG4oI42VpdWig="
+    },
+    "xml2js": {
+      "version": "0.4.23",
+      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
+      "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
+      "requires": {
+        "sax": ">=0.6.0",
+        "xmlbuilder": "~11.0.0"
+      }
+    },
+    "xmlbuilder": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+      "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
+    },
+    "xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
+    },
     "yauzl": {
       "version": "2.10.0",
       "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
diff --git a/package.json b/package.json
index 5d302ad..0e1f7c7 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,9 @@
 	"license": "ISC",
 	"dependencies": {
 		"csv-writer": "^1.6.0",
+		"jimp": "^0.16.1",
 		"puppeteer": "^5.3.1",
+		"puppeteer-full-page-screenshot": "^1.0.5",
 		"uuid": "^8.3.1"
 	},
 	"devDependencies": {}
diff --git a/src/screenshot.js b/src/screenshot.js
index 0a54ff5..a298364 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -1,7 +1,8 @@
 const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
+const jimp = require("jimp");
 
-const takeScreenshots = async (page, directory) => {
+const takeScreenshots = async (page, directory, delay) => {
 	const getElementText = async (elementRef) => {
 		// get the name of the current chart tab
 		const elementTextRef = await page.$(elementRef);
@@ -162,11 +163,11 @@ const takeScreenshots = async (page, directory) => {
 	};
 
 	const takeFullpageScreenshot = async (viewport) => {
-		if (viewport.height > page.viewport().height) {
-			console.log(
-				`Viewport too small to fit images on screen. You need to increase the viewport height in app.js line 20. Please set the height larger than ${height}`
-			);
-		}
+		// if (viewport.height > page.viewport().height) {
+		// 	console.log(
+		// 		`Viewport too small to fit images on screen. You need to increase the viewport height in app.js line 20. Please set the height larger than ${height}`
+		// 	);
+		// }
 
 		// await page.screenshot({
 		// 	path: fs.existsSync(`./${directory}/full_page.png`)
@@ -199,6 +200,50 @@ const takeScreenshots = async (page, directory) => {
 		//await page.setViewport({ width: viewport.width, height: viewport.height });
 	};
 
+	const startScrolling = async (chartTabGroupElements, chartTabGroupNames) => {
+		const images = [];
+		await page.hover("#root > div > div > div");
+		const firstImage = await page.screenshot({ path: `test0.png` });
+		images.push(firstImage);
+
+		for (let i = 0; i < chartTabGroupElements.length; i++) {
+			console.log(`scrolling to: ${chartTabGroupNames[i]}, please wait...`);
+			const chartTab = await page.$(chartTabGroupElements[i]);
+
+			// await page.evaluate((selector) => {
+			// 	const scrollableSection = document.querySelector(selector);
+
+			// 	scrollableSection.scrollTop = scrollableSection.offsetHeight;
+			// }, chartTabGroupElements[i]);
+
+			await page.evaluate((_) => {
+				window.scrollBy(0, window.innerHeight);
+			});
+
+			const image = await page.screenshot({ path: `test${i + 1}.png` });
+			images.push(image);
+		}
+		console.log(images);
+
+		let jimps = [];
+		for (let i = 0; i < images.length; i++) {
+			jimps.push(jimp.read(images[i]));
+		}
+
+		Promise.all(jimps)
+			.then((data) => {
+				return Promise.all(jimps);
+			})
+			.then((data) => {
+				data[0].composite(data[1], 0, 100);
+				data[0].composite(data[2], 0, 200);
+
+				data[0].write("final.png", () => {
+					console.log("done");
+				});
+			});
+	};
+
 	const chartTabGroupElements = []; // stores the chart tab dom elements
 	const chartTabGroupNames = []; // stores the name of each chart tab
 	const failedScreenshots = []; // stores the names of any charts that failed to screenshot
@@ -206,14 +251,17 @@ const takeScreenshots = async (page, directory) => {
 	const domElementCount = await countElements();
 	await openChartTabs(domElementCount);
 	await storeChartElementRefs(domElementCount);
-	await takeTabScreenshots(
-		chartTabGroupElements,
-		chartTabGroupNames,
-		failedScreenshots
-	);
-	await takeHeaderScreenshot();
+
+	await startScrolling(chartTabGroupElements, chartTabGroupNames);
+
+	// await takeTabScreenshots(
+	// 	chartTabGroupElements,
+	// 	chartTabGroupNames,
+	// 	failedScreenshots
+	// );
+	// await takeHeaderScreenshot();
 	const viewport = await getViewport();
-	// await setViewport(viewport);
+	await setViewport(viewport);
 	await takeFullpageScreenshot(viewport);
 };
 

From 3e1f2ca179161b8f61b42852d4e13e3b4dd871a4 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 26 Nov 2020 13:55:50 +0000
Subject: [PATCH 13/20] jimp prototype working

---
 app.js            |  12 +++--
 package.json      |   1 +
 src/screenshot.js | 124 +++++++++++++++++++++++++++++++++++-----------
 3 files changed, 104 insertions(+), 33 deletions(-)

diff --git a/app.js b/app.js
index 83b9f94..d961937 100644
--- a/app.js
+++ b/app.js
@@ -14,13 +14,12 @@ if (!url) {
 
 const puppeteerConnect = async (url) => {
 	const browser = await puppeteer.launch({
-		headless: false,
+		// headless: false,
 		// slowMo: 250,
 	});
 
 	const page = await browser.newPage();
-	// await page.setViewport({ width: 1980, height: 50000 }); // setting large height to account for case where there are many charts
-	// await page.setViewport({ width: 1980, height: 50000, deviceScaleFactor: 1 });
+	await page.setViewport({ width: 1980, height: 1000 }); // setting large height to account for case where there are many charts
 	page.setDefaultTimeout(10000);
 
 	// minimise the window if running with headless mode as false
@@ -65,9 +64,14 @@ const puppeteerConnect = async (url) => {
 		fs.mkdirSync(directory, { recursive: true });
 	}
 
+	const tempFullPageDir = "./outputs/tempFullPage";
+	if (!fs.existsSync(tempFullPageDir)) {
+		fs.mkdirSync(tempFullPageDir, { recursive: true });
+	}
+
 	// take screenshots of charts
 	try {
-		await screenshot.takeScreenshots(page, directory, delay);
+		await screenshot.takeScreenshots(page, directory, delay, tempFullPageDir);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
diff --git a/package.json b/package.json
index 0e1f7c7..2e239c0 100644
--- a/package.json
+++ b/package.json
@@ -13,6 +13,7 @@
 	"dependencies": {
 		"csv-writer": "^1.6.0",
 		"jimp": "^0.16.1",
+		"merge-img": "^2.1.3",
 		"puppeteer": "^5.3.1",
 		"puppeteer-full-page-screenshot": "^1.0.5",
 		"uuid": "^8.3.1"
diff --git a/src/screenshot.js b/src/screenshot.js
index a298364..4265630 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -1,8 +1,9 @@
 const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 const jimp = require("jimp");
+const mergeImg = require("merge-img");
 
-const takeScreenshots = async (page, directory, delay) => {
+const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 	const getElementText = async (elementRef) => {
 		// get the name of the current chart tab
 		const elementTextRef = await page.$(elementRef);
@@ -200,34 +201,95 @@ const takeScreenshots = async (page, directory, delay) => {
 		//await page.setViewport({ width: viewport.width, height: viewport.height });
 	};
 
+	const pageDown = async () => {
+		const isEnd = await page.evaluate(() => {
+			window.scrollBy(0, window.innerHeight);
+			return window.scrollY >= document.body.clientHeight - window.innerHeight;
+		});
+		return isEnd;
+	};
+
 	const startScrolling = async (chartTabGroupElements, chartTabGroupNames) => {
+		const { pagesCount, extraPixels, viewport } = await page.evaluate(() => {
+			window.scrollTo(0, 0);
+			return {
+				pagesCount: Math.ceil(document.body.clientHeight / window.innerHeight),
+				extraPixels: document.body.clientHeight % window.innerHeight,
+				viewport: {
+					height: window.innerHeight,
+					width: window.innerWidth,
+				},
+			};
+		});
 		const images = [];
-		await page.hover("#root > div > div > div");
-		const firstImage = await page.screenshot({ path: `test0.png` });
-		images.push(firstImage);
 
-		for (let i = 0; i < chartTabGroupElements.length; i++) {
-			console.log(`scrolling to: ${chartTabGroupNames[i]}, please wait...`);
-			const chartTab = await page.$(chartTabGroupElements[i]);
+		console.log(pagesCount);
+		for (let index = 0; index < pagesCount; index += 1) {
+			const image = await page.screenshot({
+				path: `./${tempFullPageDir}/test${index}.png`,
+			});
+			await pageDown();
+			images.push(image);
+		}
 
-			// await page.evaluate((selector) => {
-			// 	const scrollableSection = document.querySelector(selector);
+		// if (pagesCount === 1) {
+		// 	const image = await jimp.read(images[0]);
+		// 	if (options.path) image.write(options.path);
+		// 	return image;
+		// } // crop last image extra pixels
+
+		const cropped = await jimp
+			.read(images.pop())
+			.then((image) =>
+				image.crop(
+					0,
+					viewport.height - extraPixels - 15,
+					viewport.width,
+					extraPixels
+				)
+			)
+			.then((image) => image.getBufferAsync(jimp.AUTO));
+		images.push(cropped);
+		const mergedImage = await (0, mergeImg)(images, {
+			direction: true,
+		});
+		mergedImage.write(`./${directory}/full_page_jimp.png`);
 
-			// 	scrollableSection.scrollTop = scrollableSection.offsetHeight;
-			// }, chartTabGroupElements[i]);
+		// if (options.path) mergedImage.write(options.path);
+		//return mergedImage;
 
-			await page.evaluate((_) => {
-				window.scrollBy(0, window.innerHeight);
-			});
+		// const firstImage = await page.screenshot({
+		// 	path: `./${tempFullPageDir}/test0.png`,
+		// });
 
-			const image = await page.screenshot({ path: `test${i + 1}.png` });
-			images.push(image);
-		}
-		console.log(images);
+		// images.push(firstImage);
+
+		// for (let i = 0; i < chartTabGroupElements.length; i++) {
+		// 	console.log(`scrolling to: ${chartTabGroupNames[i]}, please wait...`);
+
+		// 	await page.evaluate((_) => {
+		// 		window.scrollBy(0, window.innerHeight);
+		// 	});
+
+		// 	const image = await page.screenshot({
+		// 		path: `./${tempFullPageDir}/test${i + 1}.png`,
+		// 	});
+
+		// 	images.push(image);
+		// }
+
+		let mainImage = new jimp(
+			1980,
+			1000 * images.length,
+			0x0,
+			function (err, image) {
+				// do stuff with image
+			}
+		);
 
 		let jimps = [];
-		for (let i = 0; i < images.length; i++) {
-			jimps.push(jimp.read(images[i]));
+		for (let k = 0; k < images.length; k++) {
+			jimps.push(jimp.read(images[k]));
 		}
 
 		Promise.all(jimps)
@@ -235,10 +297,14 @@ const takeScreenshots = async (page, directory, delay) => {
 				return Promise.all(jimps);
 			})
 			.then((data) => {
-				data[0].composite(data[1], 0, 100);
-				data[0].composite(data[2], 0, 200);
+				let offset = 1000;
+				mainImage.composite(data[0], 0, 0);
+				for (let j = 1; j < images.length; j++) {
+					mainImage.composite(data[j], 0, offset);
+					offset = offset + 1000;
+				}
 
-				data[0].write("final.png", () => {
+				mainImage.write(`./${directory}/full_page_jimp.png`, () => {
 					console.log("done");
 				});
 			});
@@ -254,12 +320,12 @@ const takeScreenshots = async (page, directory, delay) => {
 
 	await startScrolling(chartTabGroupElements, chartTabGroupNames);
 
-	// await takeTabScreenshots(
-	// 	chartTabGroupElements,
-	// 	chartTabGroupNames,
-	// 	failedScreenshots
-	// );
-	// await takeHeaderScreenshot();
+	await takeTabScreenshots(
+		chartTabGroupElements,
+		chartTabGroupNames,
+		failedScreenshots
+	);
+	await takeHeaderScreenshot();
 	const viewport = await getViewport();
 	await setViewport(viewport);
 	await takeFullpageScreenshot(viewport);

From baf53c0b5c9118526998fb1d4045e0cd36e4da3b Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 26 Nov 2020 17:25:37 +0000
Subject: [PATCH 14/20] tidying up jimp code

---
 app.js            |   2 +-
 src/screenshot.js | 151 ++++++++++++++++------------------------------
 2 files changed, 54 insertions(+), 99 deletions(-)

diff --git a/app.js b/app.js
index d961937..1d04a72 100644
--- a/app.js
+++ b/app.js
@@ -64,7 +64,7 @@ const puppeteerConnect = async (url) => {
 		fs.mkdirSync(directory, { recursive: true });
 	}
 
-	const tempFullPageDir = "./outputs/tempFullPage";
+	const tempFullPageDir = "./outputs/tempImages";
 	if (!fs.existsSync(tempFullPageDir)) {
 		fs.mkdirSync(tempFullPageDir, { recursive: true });
 	}
diff --git a/src/screenshot.js b/src/screenshot.js
index 4265630..17d1a73 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -163,7 +163,15 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 		});
 	};
 
-	const takeFullpageScreenshot = async (viewport) => {
+	const pageDown = async () => {
+		const isEnd = await page.evaluate(() => {
+			window.scrollBy(0, window.innerHeight);
+			return window.scrollY >= document.body.clientHeight - window.innerHeight;
+		});
+		return isEnd;
+	};
+
+	const takeFullpageScreenshot = async () => {
 		// if (viewport.height > page.viewport().height) {
 		// 	console.log(
 		// 		`Viewport too small to fit images on screen. You need to increase the viewport height in app.js line 20. Please set the height larger than ${height}`
@@ -178,39 +186,21 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 		// });
 
 		// screenshot the entire page (append unique id to name if it already exists in the directory)
-		const rootElement = await page.$("#root");
-		await rootElement.screenshot({
-			path: fs.existsSync(`./${directory}/full_page.png`)
-				? `${directory}/full_page_${uuidv4()}.png`
-				: `${directory}/full_page.png`,
-			omitBackground: true,
-		});
-	};
-
-	const getViewport = async () => {
-		// detect the size of the page when all chart tabs are open so we can take a full page screenshot
-		const rootElement = await page.$("#root");
-		const boundingBox = await rootElement.boundingBox();
-		const { width, height } = boundingBox;
-		return boundingBox;
-	};
-
-	const setViewport = async (viewport) => {
-		console.log(viewport);
-		// commented out setViewport as the chart tabs get closed when changing the viewport size (this is an issue with RAMPART not this app)
-		//await page.setViewport({ width: viewport.width, height: viewport.height });
-	};
-
-	const pageDown = async () => {
-		const isEnd = await page.evaluate(() => {
-			window.scrollBy(0, window.innerHeight);
-			return window.scrollY >= document.body.clientHeight - window.innerHeight;
-		});
-		return isEnd;
-	};
+		// const rootElement = await page.$("#root");
+		// await rootElement.screenshot({
+		// 	path: fs.existsSync(`./${directory}/full_page.png`)
+		// 		? `${directory}/full_page_${uuidv4()}.png`
+		// 		: `${directory}/full_page.png`,
+		// 	omitBackground: true,
+		// });
 
-	const startScrolling = async (chartTabGroupElements, chartTabGroupNames) => {
-		const { pagesCount, extraPixels, viewport } = await page.evaluate(() => {
+		const {
+			pagesCount,
+			extraPixels,
+			viewport,
+			clientHeight,
+			innerHeight,
+		} = await page.evaluate(() => {
 			window.scrollTo(0, 0);
 			return {
 				pagesCount: Math.ceil(document.body.clientHeight / window.innerHeight),
@@ -219,31 +209,38 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 					height: window.innerHeight,
 					width: window.innerWidth,
 				},
+				clientHeight: document.body.clientHeight,
+				innerHeight: window.innerHeight,
 			};
 		});
-		const images = [];
+		console.log(
+			`pages count: ${pagesCount}, \n extra pixels: ${extraPixels}, \n viewport width: ${viewport.width}, viewport height: ${viewport.height}, \n client height: ${clientHeight}, \n inner height: ${innerHeight} `
+		);
 
-		console.log(pagesCount);
-		for (let index = 0; index < pagesCount; index += 1) {
+		const images = [];
+		for (let i = 0; i < pagesCount; i += 1) {
 			const image = await page.screenshot({
-				path: `./${tempFullPageDir}/test${index}.png`,
+				path: `./${tempFullPageDir}/test${i}.png`,
 			});
 			await pageDown();
 			images.push(image);
 		}
 
-		// if (pagesCount === 1) {
-		// 	const image = await jimp.read(images[0]);
-		// 	if (options.path) image.write(options.path);
-		// 	return image;
-		// } // crop last image extra pixels
+		if (pagesCount === 1) {
+			const image = await jimp.read(images[0]);
+			image.write(`./${directory}/full_page_jimp.png`, () => {
+				console.log("done");
+			});
+			return;
+		}
 
+		// crop the last image so that we can append it to the end of the full page png
 		const cropped = await jimp
 			.read(images.pop())
 			.then((image) =>
 				image.crop(
 					0,
-					viewport.height - extraPixels - 15,
+					viewport.height - extraPixels - 16,
 					viewport.width,
 					extraPixels
 				)
@@ -254,60 +251,20 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 			direction: true,
 		});
 		mergedImage.write(`./${directory}/full_page_jimp.png`);
+	};
 
-		// if (options.path) mergedImage.write(options.path);
-		//return mergedImage;
-
-		// const firstImage = await page.screenshot({
-		// 	path: `./${tempFullPageDir}/test0.png`,
-		// });
-
-		// images.push(firstImage);
-
-		// for (let i = 0; i < chartTabGroupElements.length; i++) {
-		// 	console.log(`scrolling to: ${chartTabGroupNames[i]}, please wait...`);
-
-		// 	await page.evaluate((_) => {
-		// 		window.scrollBy(0, window.innerHeight);
-		// 	});
-
-		// 	const image = await page.screenshot({
-		// 		path: `./${tempFullPageDir}/test${i + 1}.png`,
-		// 	});
-
-		// 	images.push(image);
-		// }
-
-		let mainImage = new jimp(
-			1980,
-			1000 * images.length,
-			0x0,
-			function (err, image) {
-				// do stuff with image
-			}
-		);
-
-		let jimps = [];
-		for (let k = 0; k < images.length; k++) {
-			jimps.push(jimp.read(images[k]));
-		}
-
-		Promise.all(jimps)
-			.then((data) => {
-				return Promise.all(jimps);
-			})
-			.then((data) => {
-				let offset = 1000;
-				mainImage.composite(data[0], 0, 0);
-				for (let j = 1; j < images.length; j++) {
-					mainImage.composite(data[j], 0, offset);
-					offset = offset + 1000;
-				}
+	const getViewport = async () => {
+		// detect the size of the page when all chart tabs are open so we can take a full page screenshot
+		const rootElement = await page.$("#root");
+		const boundingBox = await rootElement.boundingBox();
+		const { width, height } = boundingBox;
+		return boundingBox;
+	};
 
-				mainImage.write(`./${directory}/full_page_jimp.png`, () => {
-					console.log("done");
-				});
-			});
+	const setViewport = async (viewport) => {
+		console.log(viewport);
+		// commented out setViewport as the chart tabs get closed when changing the viewport size (this is an issue with RAMPART not this app)
+		//await page.setViewport({ width: viewport.width, height: viewport.height });
 	};
 
 	const chartTabGroupElements = []; // stores the chart tab dom elements
@@ -318,8 +275,6 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 	await openChartTabs(domElementCount);
 	await storeChartElementRefs(domElementCount);
 
-	await startScrolling(chartTabGroupElements, chartTabGroupNames);
-
 	await takeTabScreenshots(
 		chartTabGroupElements,
 		chartTabGroupNames,
@@ -328,7 +283,7 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 	await takeHeaderScreenshot();
 	const viewport = await getViewport();
 	await setViewport(viewport);
-	await takeFullpageScreenshot(viewport);
+	await takeFullpageScreenshot();
 };
 
 exports.takeScreenshots = takeScreenshots;

From 0c521e665cc269b78978a9c2a2a08eb60d90d6ba Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Fri, 27 Nov 2020 10:04:57 +0000
Subject: [PATCH 15/20] added try catch block to full page screenshot

---
 app.js            |   2 +-
 src/screenshot.js | 142 +++++++++++++++++++++-------------------------
 2 files changed, 66 insertions(+), 78 deletions(-)

diff --git a/app.js b/app.js
index 1d04a72..edfa033 100644
--- a/app.js
+++ b/app.js
@@ -82,7 +82,7 @@ const puppeteerConnect = async (url) => {
 
 	// save the information in the 'reports' tab as a .csv
 	try {
-		// await saveReport.saveReport(page, directory, delay);
+		await saveReport.saveReport(page, directory, delay);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
diff --git a/src/screenshot.js b/src/screenshot.js
index 17d1a73..fbbf288 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -1,6 +1,6 @@
 const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
-const jimp = require("jimp");
+const Jimp = require("jimp");
 const mergeImg = require("merge-img");
 
 const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
@@ -163,7 +163,7 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 		});
 	};
 
-	const pageDown = async () => {
+	const scrollDown = async () => {
 		const isEnd = await page.evaluate(() => {
 			window.scrollBy(0, window.innerHeight);
 			return window.scrollY >= document.body.clientHeight - window.innerHeight;
@@ -172,85 +172,73 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 	};
 
 	const takeFullpageScreenshot = async () => {
-		// if (viewport.height > page.viewport().height) {
-		// 	console.log(
-		// 		`Viewport too small to fit images on screen. You need to increase the viewport height in app.js line 20. Please set the height larger than ${height}`
-		// 	);
-		// }
-
-		// await page.screenshot({
-		// 	path: fs.existsSync(`./${directory}/full_page.png`)
-		// 		? `${directory}/full_page${uuidv4()}.png`
-		// 		: `${directory}/full_page.png`,
-		// 	fullPage: true,
-		// });
-
-		// screenshot the entire page (append unique id to name if it already exists in the directory)
-		// const rootElement = await page.$("#root");
-		// await rootElement.screenshot({
-		// 	path: fs.existsSync(`./${directory}/full_page.png`)
-		// 		? `${directory}/full_page_${uuidv4()}.png`
-		// 		: `${directory}/full_page.png`,
-		// 	omitBackground: true,
-		// });
-
-		const {
-			pagesCount,
-			extraPixels,
-			viewport,
-			clientHeight,
-			innerHeight,
-		} = await page.evaluate(() => {
-			window.scrollTo(0, 0);
-			return {
-				pagesCount: Math.ceil(document.body.clientHeight / window.innerHeight),
-				extraPixels: document.body.clientHeight % window.innerHeight,
-				viewport: {
-					height: window.innerHeight,
-					width: window.innerWidth,
-				},
-				clientHeight: document.body.clientHeight,
-				innerHeight: window.innerHeight,
-			};
-		});
-		console.log(
-			`pages count: ${pagesCount}, \n extra pixels: ${extraPixels}, \n viewport width: ${viewport.width}, viewport height: ${viewport.height}, \n client height: ${clientHeight}, \n inner height: ${innerHeight} `
-		);
-
-		const images = [];
-		for (let i = 0; i < pagesCount; i += 1) {
-			const image = await page.screenshot({
-				path: `./${tempFullPageDir}/test${i}.png`,
+		console.log("\n Taking full page screenshot, please wait...");
+		try {
+			const {
+				pagesCount,
+				extraPixels,
+				viewport,
+				clientHeight,
+				innerHeight,
+			} = await page.evaluate(() => {
+				window.scrollTo(0, 0);
+				return {
+					pagesCount: Math.ceil(
+						document.body.clientHeight / window.innerHeight
+					),
+					extraPixels: document.body.clientHeight % window.innerHeight,
+					viewport: {
+						height: window.innerHeight,
+						width: window.innerWidth,
+					},
+					clientHeight: document.body.clientHeight,
+					innerHeight: window.innerHeight,
+				};
 			});
-			await pageDown();
-			images.push(image);
-		}
+			// console.log(
+			// 	`pages count: ${pagesCount}, \n extra pixels: ${extraPixels}, \n viewport width: ${viewport.width}, viewport height: ${viewport.height}, \n client height: ${clientHeight}, \n inner height: ${innerHeight} `
+			// );
+
+			const images = [];
+			for (let i = 0; i < pagesCount; i += 1) {
+				const image = await page.screenshot({
+					path: `./${tempFullPageDir}/temp_${i}.png`,
+				});
+				await scrollDown();
+				images.push(image);
+			}
 
-		if (pagesCount === 1) {
-			const image = await jimp.read(images[0]);
-			image.write(`./${directory}/full_page_jimp.png`, () => {
-				console.log("done");
-			});
-			return;
-		}
+			if (pagesCount === 1) {
+				const image = await Jimp.read(images[0]);
+				image.write(`./${directory}/full_page_screenshot.png`, () => {
+					console.log("successfully took full page screenshot");
+				});
+				return;
+			}
 
-		// crop the last image so that we can append it to the end of the full page png
-		const cropped = await jimp
-			.read(images.pop())
-			.then((image) =>
-				image.crop(
-					0,
-					viewport.height - extraPixels - 16,
-					viewport.width,
-					extraPixels
+			// crop the last image so that we can append it to the end of the full page png
+			const cropped = await Jimp.read(images.pop())
+				.then((image) =>
+					image.crop(
+						0,
+						viewport.height - extraPixels - 16,
+						viewport.width,
+						extraPixels
+					)
 				)
-			)
-			.then((image) => image.getBufferAsync(jimp.AUTO));
-		images.push(cropped);
-		const mergedImage = await (0, mergeImg)(images, {
-			direction: true,
-		});
-		mergedImage.write(`./${directory}/full_page_jimp.png`);
+				.then((image) => image.getBufferAsync(Jimp.AUTO));
+			images.push(cropped);
+			const mergedImage = await (0, mergeImg)(images, {
+				direction: true,
+			});
+			await mergedImage.write(`./${directory}/full_page_jimp.png`);
+			console.log("successfully took full page screenshot");
+		} catch (err) {
+			console.log(
+				"something went wrong taking the full page screenshot, printing error... \n " +
+					err
+			);
+		}
 	};
 
 	const getViewport = async () => {

From dedaf06ef932ba6a25800d0a0e4997a4b7043a79 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Fri, 27 Nov 2020 12:43:44 +0000
Subject: [PATCH 16/20] tidying things up (removing comments etc.)

---
 app.js             |  21 +--
 package-lock.json  | 325 ---------------------------------------------
 package.json       |   4 +-
 src/save_report.js |  14 +-
 src/screenshot.js  |  55 +++-----
 5 files changed, 37 insertions(+), 382 deletions(-)

diff --git a/app.js b/app.js
index edfa033..4b6e0f5 100644
--- a/app.js
+++ b/app.js
@@ -19,17 +19,9 @@ const puppeteerConnect = async (url) => {
 	});
 
 	const page = await browser.newPage();
-	await page.setViewport({ width: 1980, height: 1000 }); // setting large height to account for case where there are many charts
+	await page.setViewport({ width: 1980, height: 1000 });
 	page.setDefaultTimeout(10000);
 
-	// minimise the window if running with headless mode as false
-	// const session = await page.target().createCDPSession();
-	// const { windowId } = await session.send("Browser.getWindowForTarget");
-	// await session.send("Browser.setWindowBounds", {
-	// 	windowId,
-	// 	bounds: { windowState: "minimized" },
-	// });
-
 	try {
 		await page.goto(url, {
 			waitUntil: "networkidle2",
@@ -64,14 +56,14 @@ const puppeteerConnect = async (url) => {
 		fs.mkdirSync(directory, { recursive: true });
 	}
 
-	const tempFullPageDir = "./outputs/tempImages";
-	if (!fs.existsSync(tempFullPageDir)) {
-		fs.mkdirSync(tempFullPageDir, { recursive: true });
+	const tempImagesDir = "./outputs/tempImages";
+	if (!fs.existsSync(tempImagesDir)) {
+		fs.mkdirSync(tempImagesDir, { recursive: true });
 	}
 
 	// take screenshots of charts
 	try {
-		await screenshot.takeScreenshots(page, directory, delay, tempFullPageDir);
+		await screenshot.takeScreenshots(page, directory, tempImagesDir);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
@@ -82,7 +74,7 @@ const puppeteerConnect = async (url) => {
 
 	// save the information in the 'reports' tab as a .csv
 	try {
-		await saveReport.saveReport(page, directory, delay);
+		await saveReport.saveReport(page, directory);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
@@ -91,6 +83,7 @@ const puppeteerConnect = async (url) => {
 		console.log(err);
 	}
 
+	console.log("\x1b[36m%s\x1b[0m", "all operations completed");
 	browser.close();
 };
 
diff --git a/package-lock.json b/package-lock.json
index 01d6df6..ceea81b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -486,11 +486,6 @@
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
     },
-    "core-js": {
-      "version": "2.6.12",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz",
-      "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ=="
-    },
     "core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@@ -1108,326 +1103,6 @@
         "ws": "^7.2.3"
       }
     },
-    "puppeteer-full-page-screenshot": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/puppeteer-full-page-screenshot/-/puppeteer-full-page-screenshot-1.0.5.tgz",
-      "integrity": "sha512-OHpPb8spKfCNNnKbxX91fLTk0R7XXv9L47/q+R6vpLUwefYdyY823BfWoxeNxAIFZpk6r3XPlkaI1hfsCmelvQ==",
-      "requires": {
-        "jimp": "^0.6.4",
-        "merge-img": "^2.1.3"
-      },
-      "dependencies": {
-        "@jimp/bmp": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.6.8.tgz",
-          "integrity": "sha512-uxVgSkI62uAzk5ZazYHEHBehow590WAkLKmDXLzkr/XP/Hv2Fx1T4DKwJ/15IY5ktq5VAhAUWGXTyd8KWFsx7w==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "bmp-js": "^0.1.0",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/core": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.6.8.tgz",
-          "integrity": "sha512-JOFqBBcSNiDiMZJFr6OJqC6viXj5NVBQISua0eacoYvo4YJtTajOIxC4MqWyUmGrDpRMZBR8QhSsIOwsFrdROA==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "any-base": "^1.1.0",
-            "buffer": "^5.2.0",
-            "core-js": "^2.5.7",
-            "exif-parser": "^0.1.12",
-            "file-type": "^9.0.0",
-            "load-bmfont": "^1.3.1",
-            "mkdirp": "0.5.1",
-            "phin": "^2.9.1",
-            "pixelmatch": "^4.0.2",
-            "tinycolor2": "^1.4.1"
-          }
-        },
-        "@jimp/custom": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.6.8.tgz",
-          "integrity": "sha512-FrYlzZRVXP2vuVwd7Nc2dlK+iZk4g6IaT1Ib8Z6vU5Kkwlt83FJIPJ2UUFABf3bF5big0wkk8ZUihWxE4Nzdng==",
-          "requires": {
-            "@jimp/core": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/gif": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.6.8.tgz",
-          "integrity": "sha512-yyOlujjQcgz9zkjM5ihZDEppn9d1brJ7jQHP5rAKmqep0G7FU1D0AKcV+Ql18RhuI/CgWs10wAVcrQpmLnu4Yw==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7",
-            "omggif": "^1.0.9"
-          }
-        },
-        "@jimp/jpeg": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.6.8.tgz",
-          "integrity": "sha512-rGtXbYpFXAn471qLpTGvhbBMNHJo5KiufN+vC5AWyufntmkt5f0Ox2Cx4ijuBMDtirZchxbMLtrfGjznS4L/ew==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7",
-            "jpeg-js": "^0.3.4"
-          }
-        },
-        "@jimp/plugin-blit": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.6.8.tgz",
-          "integrity": "sha512-7Tl6YpKTSpvwQbnGNhsfX2zyl3jRVVopd276Y2hF2zpDz9Bycow7NdfNU/4Nx1jaf96X6uWOtSVINcQ7rGd47w==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-blur": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.6.8.tgz",
-          "integrity": "sha512-NpZCMKxXHLDQsX9zPlWtpMA660DQStY6/z8ZetyxCDbqrLe9YCXpeR4MNhdJdABIiwTm1W5FyFF4kp81PHJx3Q==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-color": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.6.8.tgz",
-          "integrity": "sha512-jjFyU0zNmGOH2rjzHuOMU4kaia0oo82s/7UYfn5h7OUkmUZTd6Do3ZSK1PiXA7KR+s4B76/Omm6Doh/0SGb7BQ==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7",
-            "tinycolor2": "^1.4.1"
-          }
-        },
-        "@jimp/plugin-contain": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.6.8.tgz",
-          "integrity": "sha512-p/P2wCXhAzbmEgXvGsvmxLmbz45feF6VpR4m9suPSOr8PC/i/XvTklTqYEUidYYAft4vHgsYJdS74HKSMnH8lw==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-cover": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.6.8.tgz",
-          "integrity": "sha512-2PvWgk+PJfRsfWDI1G8Fpjrsu0ZlpNyZxO2+fqWlVo6y/y2gP4v08FqvbkcqSjNlOu2IDWIFXpgyU0sTINWZLg==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-crop": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.6.8.tgz",
-          "integrity": "sha512-CbrcpWE2xxPK1n/JoTXzhRUhP4mO07mTWaSavenCg664oQl/9XCtL+A0FekuNHzIvn4myEqvkiTwN7FsbunS/Q==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-displace": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.6.8.tgz",
-          "integrity": "sha512-RmV2bPxoPE6mrPxtYSPtHxm2cGwBQr5a2p+9gH6SPy+eUMrbGjbvjwKNfXWUYD0leML+Pt5XOmAS9pIROmuruQ==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-dither": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.6.8.tgz",
-          "integrity": "sha512-x6V/qjxe+xypjpQm7GbiMNqci1EW5UizrcebOhHr8AHijOEqHd2hjXh5f6QIGfrkTFelc4/jzq1UyCsYntqz9Q==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-flip": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.6.8.tgz",
-          "integrity": "sha512-4il6Da6G39s9MyWBEee4jztEOUGJ40E6OlPjkMrdpDNvge6hYEAB31BczTYBP/CEY74j4LDSoY5LbcU4kv06yA==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-gaussian": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.6.8.tgz",
-          "integrity": "sha512-pVOblmjv7stZjsqloi4YzHVwAPXKGdNaHPhp4KP4vj41qtc6Hxd9z/+VWGYRTunMFac84gUToe0UKIXd6GhoKw==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-invert": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.6.8.tgz",
-          "integrity": "sha512-11zuLiXDHr6tFv4U8aieXqNXQEKbDbSBG/h+X62gGTNFpyn8EVPpncHhOqrAFtZUaPibBqMFlNJ15SzwC7ExsQ==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-mask": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.6.8.tgz",
-          "integrity": "sha512-hZJ0OiKGJyv7hDSATwJDkunB1Ie80xJnONMgpUuUseteK45YeYNBOiZVUe8vum8QI1UwavgBzcvQ9u4fcgXc9g==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-normalize": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.6.8.tgz",
-          "integrity": "sha512-Q4oYhU+sSyTJI7pMZlg9/mYh68ujLfOxXzQGEXuw0sHGoGQs3B0Jw7jmzGa6pIS06Hup5hD2Zuh1ppvMdjJBfQ==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-print": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.6.8.tgz",
-          "integrity": "sha512-2aokejGn4Drv1FesnZGqh5KEq0FQtR0drlmtyZrBH+r9cx7hh0Qgf4D1BOTDEgXkfSSngjGRjKKRW/fwOrVXYw==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7",
-            "load-bmfont": "^1.4.0"
-          }
-        },
-        "@jimp/plugin-resize": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.6.8.tgz",
-          "integrity": "sha512-27nPh8L1YWsxtfmV/+Ub5dOTpXyC0HMF2cu52RQSCYxr+Lm1+23dJF70AF1poUbUe+FWXphwuUxQzjBJza9UoA==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-rotate": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.6.8.tgz",
-          "integrity": "sha512-GbjETvL05BDoLdszNUV4Y0yLkHf177MnqGqilA113LIvx9aD0FtUopGXYfRGVvmtTOTouoaGJUc+K6qngvKxww==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugin-scale": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.6.8.tgz",
-          "integrity": "sha512-GzIYWR/oCUK2jAwku23zt19V1ssaEU4pL0x2XsLNKuuJEU6DvEytJyTMXCE7OLG/MpDBQcQclJKHgiyQm5gIOQ==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7"
-          }
-        },
-        "@jimp/plugins": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.6.8.tgz",
-          "integrity": "sha512-fMcTI72Vn/Lz6JftezTURmyP5ml/xGMe0Ljx2KRJ85IWyP33vDmGIUuutFiBEbh2+y7lRT+aTSmjs0QGa/xTmQ==",
-          "requires": {
-            "@jimp/plugin-blit": "^0.6.8",
-            "@jimp/plugin-blur": "^0.6.8",
-            "@jimp/plugin-color": "^0.6.8",
-            "@jimp/plugin-contain": "^0.6.8",
-            "@jimp/plugin-cover": "^0.6.8",
-            "@jimp/plugin-crop": "^0.6.8",
-            "@jimp/plugin-displace": "^0.6.8",
-            "@jimp/plugin-dither": "^0.6.8",
-            "@jimp/plugin-flip": "^0.6.8",
-            "@jimp/plugin-gaussian": "^0.6.8",
-            "@jimp/plugin-invert": "^0.6.8",
-            "@jimp/plugin-mask": "^0.6.8",
-            "@jimp/plugin-normalize": "^0.6.8",
-            "@jimp/plugin-print": "^0.6.8",
-            "@jimp/plugin-resize": "^0.6.8",
-            "@jimp/plugin-rotate": "^0.6.8",
-            "@jimp/plugin-scale": "^0.6.8",
-            "core-js": "^2.5.7",
-            "timm": "^1.6.1"
-          }
-        },
-        "@jimp/png": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.6.8.tgz",
-          "integrity": "sha512-JHHg/BZ7KDtHQrcG+a7fztw45rdf7okL/YwkN4qU5FH7Fcrp41nX5QnRviDtD9hN+GaNC7kvjvcqRAxW25qjew==",
-          "requires": {
-            "@jimp/utils": "^0.6.8",
-            "core-js": "^2.5.7",
-            "pngjs": "^3.3.3"
-          }
-        },
-        "@jimp/tiff": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.6.8.tgz",
-          "integrity": "sha512-iWHbxd+0IKWdJyJ0HhoJCGYmtjPBOusz1z1HT/DnpePs/Lo3TO4d9ALXqYfUkyG74ZK5jULZ69KLtwuhuJz1bg==",
-          "requires": {
-            "core-js": "^2.5.7",
-            "utif": "^2.0.1"
-          }
-        },
-        "@jimp/types": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.6.8.tgz",
-          "integrity": "sha512-vCZ/Cp2osy69VP21XOBACfHI5HeR60Rfd4Jidj4W73UL+HrFWOtyQiJ7hlToyu1vI5mR/NsUQpzyQvz56ADm5A==",
-          "requires": {
-            "@jimp/bmp": "^0.6.8",
-            "@jimp/gif": "^0.6.8",
-            "@jimp/jpeg": "^0.6.8",
-            "@jimp/png": "^0.6.8",
-            "@jimp/tiff": "^0.6.8",
-            "core-js": "^2.5.7",
-            "timm": "^1.6.1"
-          }
-        },
-        "@jimp/utils": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.6.8.tgz",
-          "integrity": "sha512-7RDfxQ2C/rarNG9iso5vmnKQbcvlQjBIlF/p7/uYj72WeZgVCB+5t1fFBKJSU4WhniHX4jUMijK+wYGE3Y3bGw==",
-          "requires": {
-            "core-js": "^2.5.7"
-          }
-        },
-        "jimp": {
-          "version": "0.6.8",
-          "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.6.8.tgz",
-          "integrity": "sha512-F7emeG7Hp61IM8VFbNvWENLTuHe0ghizWPuP4JS9ujx2r5mCVYEd/zdaz6M2M42ZdN41blxPajLWl9FXo7Mr2Q==",
-          "requires": {
-            "@jimp/custom": "^0.6.8",
-            "@jimp/plugins": "^0.6.8",
-            "@jimp/types": "^0.6.8",
-            "core-js": "^2.5.7",
-            "regenerator-runtime": "^0.13.3"
-          }
-        },
-        "jpeg-js": {
-          "version": "0.3.7",
-          "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.7.tgz",
-          "integrity": "sha512-9IXdWudL61npZjvLuVe/ktHiA41iE8qFyLB+4VDTblEsWBzeg8WQTlktdUK4CdncUqtUgUg0bbOmTE2bKBKaBQ=="
-        },
-        "minimist": {
-          "version": "0.0.8",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-          "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
-        },
-        "mkdirp": {
-          "version": "0.5.1",
-          "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
-          "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
-          "requires": {
-            "minimist": "0.0.8"
-          }
-        }
-      }
-    },
     "qs": {
       "version": "6.5.2",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
diff --git a/package.json b/package.json
index 2e239c0..e673922 100644
--- a/package.json
+++ b/package.json
@@ -5,8 +5,7 @@
 	"main": "app.js",
 	"scripts": {
 		"test": "echo \"Error: no test specified\" && exit 1",
-		"start": "node app.js",
-		"build": "nexe"
+		"start": "node app.js"
 	},
 	"author": "",
 	"license": "ISC",
@@ -15,7 +14,6 @@
 		"jimp": "^0.16.1",
 		"merge-img": "^2.1.3",
 		"puppeteer": "^5.3.1",
-		"puppeteer-full-page-screenshot": "^1.0.5",
 		"uuid": "^8.3.1"
 	},
 	"devDependencies": {}
diff --git a/src/save_report.js b/src/save_report.js
index d7508cd..4b87d73 100644
--- a/src/save_report.js
+++ b/src/save_report.js
@@ -1,7 +1,8 @@
+const fs = require("fs");
 const { v4: uuidv4 } = require("uuid");
 const createCsvWriter = require("csv-writer").createObjectCsvWriter;
 
-const saveReport = async (page, directory, delay) => {
+const saveReport = async (page, directory) => {
 	const openReportTab = async () => {
 		const reportButtonElement = await page.$(
 			`#root > div > div > div:nth-child(1) > div.buttons > button`
@@ -83,7 +84,9 @@ const saveReport = async (page, directory, delay) => {
 
 	const writeTableToCsv = async (tableName, csvHeaders, csvRows) => {
 		const csvWriter = createCsvWriter({
-			path: `${directory}/${tableName}.csv`,
+			path: fs.existsSync(`./${directory}/report_${tableName}.csv`)
+				? `${directory}/${tableName}_${uuidv4()}.csv`
+				: `${directory}/${tableName}.csv`,
 			header: csvHeaders,
 		});
 
@@ -94,15 +97,18 @@ const saveReport = async (page, directory, delay) => {
 			})
 			.catch((error) => {
 				console.log(
-					`something went wrong trying to write report '${tableName}' as .csv`
+					`something went wrong trying to write report '${tableName}' as .csv \n${error}`
 				);
-				console.error(error);
 			});
 	};
 
 	// App flow starts here
 	await openReportTab();
 	const tableCount = await countTables();
+	if (tableCount === 0) {
+		console.log("no tables were found");
+		return;
+	}
 
 	// loop through each table and save the data as csv
 	for (let i = 1; i < tableCount + 1; i++) {
diff --git a/src/screenshot.js b/src/screenshot.js
index fbbf288..3269de4 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -3,7 +3,7 @@ const { v4: uuidv4 } = require("uuid");
 const Jimp = require("jimp");
 const mergeImg = require("merge-img");
 
-const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
+const takeScreenshots = async (page, directory, tempImagesDir) => {
 	const getElementText = async (elementRef) => {
 		// get the name of the current chart tab
 		const elementTextRef = await page.$(elementRef);
@@ -11,7 +11,6 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 			(element) => element.innerHTML,
 			elementTextRef
 		);
-
 		return text;
 	};
 
@@ -41,7 +40,6 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 			const openTabAndWait = async () => {
 				// get the button to open the tab again (else the node detaches on the second attempt)
 				const element = await page.$(tabButtonRef);
-
 				await element.click();
 				console.log(`chart tab button ${chartTabNameText} clicked`);
 
@@ -172,15 +170,9 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 	};
 
 	const takeFullpageScreenshot = async () => {
-		console.log("\n Taking full page screenshot, please wait...");
+		console.log("\nTaking full page screenshot, please wait...");
 		try {
-			const {
-				pagesCount,
-				extraPixels,
-				viewport,
-				clientHeight,
-				innerHeight,
-			} = await page.evaluate(() => {
+			const { pagesCount, extraPixels, viewport } = await page.evaluate(() => {
 				window.scrollTo(0, 0);
 				return {
 					pagesCount: Math.ceil(
@@ -195,14 +187,11 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 					innerHeight: window.innerHeight,
 				};
 			});
-			// console.log(
-			// 	`pages count: ${pagesCount}, \n extra pixels: ${extraPixels}, \n viewport width: ${viewport.width}, viewport height: ${viewport.height}, \n client height: ${clientHeight}, \n inner height: ${innerHeight} `
-			// );
 
 			const images = [];
 			for (let i = 0; i < pagesCount; i += 1) {
 				const image = await page.screenshot({
-					path: `./${tempFullPageDir}/temp_${i}.png`,
+					path: `./${tempImagesDir}/temp_${i}.png`,
 				});
 				await scrollDown();
 				images.push(image);
@@ -210,9 +199,14 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 
 			if (pagesCount === 1) {
 				const image = await Jimp.read(images[0]);
-				image.write(`./${directory}/full_page_screenshot.png`, () => {
-					console.log("successfully took full page screenshot");
-				});
+				image.write(
+					fs.existsSync(`./${directory}/full_page_screenshot.png`)
+						? `${directory}/full_page_screenshot_${uuidv4()}.png`
+						: `${directory}/full_page_screenshot.png`,
+					() => {
+						console.log("successfully took full page screenshot \n");
+					}
+				);
 				return;
 			}
 
@@ -231,8 +225,12 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 			const mergedImage = await (0, mergeImg)(images, {
 				direction: true,
 			});
-			await mergedImage.write(`./${directory}/full_page_jimp.png`);
-			console.log("successfully took full page screenshot");
+			await mergedImage.write(
+				fs.existsSync(`./${directory}/full_page_screenshot.png`)
+					? `${directory}/full_page_screenshot_${uuidv4()}.png`
+					: `${directory}/full_page_screenshot.png`
+			);
+			console.log("successfully took full page screenshot \n");
 		} catch (err) {
 			console.log(
 				"something went wrong taking the full page screenshot, printing error... \n " +
@@ -241,20 +239,7 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 		}
 	};
 
-	const getViewport = async () => {
-		// detect the size of the page when all chart tabs are open so we can take a full page screenshot
-		const rootElement = await page.$("#root");
-		const boundingBox = await rootElement.boundingBox();
-		const { width, height } = boundingBox;
-		return boundingBox;
-	};
-
-	const setViewport = async (viewport) => {
-		console.log(viewport);
-		// commented out setViewport as the chart tabs get closed when changing the viewport size (this is an issue with RAMPART not this app)
-		//await page.setViewport({ width: viewport.width, height: viewport.height });
-	};
-
+	// App flow starts here
 	const chartTabGroupElements = []; // stores the chart tab dom elements
 	const chartTabGroupNames = []; // stores the name of each chart tab
 	const failedScreenshots = []; // stores the names of any charts that failed to screenshot
@@ -269,8 +254,6 @@ const takeScreenshots = async (page, directory, delay, tempFullPageDir) => {
 		failedScreenshots
 	);
 	await takeHeaderScreenshot();
-	const viewport = await getViewport();
-	await setViewport(viewport);
 	await takeFullpageScreenshot();
 };
 

From 51703b627552c887f3b7e7cd280b224bcba6e529 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Fri, 27 Nov 2020 13:28:16 +0000
Subject: [PATCH 17/20] application now cleans up temporary files and folders
 automatically

---
 README.md    | 6 +++++-
 app.js       | 7 ++++++-
 package.json | 1 +
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 8937130..4b527ef 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
 This application allows users of the RAMPART project [https://github.com/artic-network/rampart](https://github.com/artic-network/rampart) to take automatic screenshots of RAMPART chart outputs using puppeteer https://github.com/puppeteer/puppeteer](https://github.com/puppeteer/puppeteer).
 
+The application will also save the table data in the RAMPART reports tab into separate .csv files.
+
 ---
 
 ## Requirements
@@ -30,6 +32,8 @@ node app.js
 
 The application will then automatically take screenshots of the charts in your active RAMPART session. Screenshots are saved as .png files in the project root under `/outputs`. Depending on how many charts you have in your project, it may take some time to finish taking all of the screenshots. Any screenshots that fail (see below for possible errors) will have "-failed" appended to the filename.
 
+The .csv table data in the RAMPART reports tab will also be saved into the `/outputs` directory.
+
 ---
 
 ## Specifying a custom port
@@ -54,7 +58,7 @@ When no arguments are passed the application will assume your RAMPART project is
 
 There is currently a known chromium bug that is affecting the behaviour of some fullpage screenshots, see here for more information [https://github.com/puppeteer/puppeteer/issues/1576](https://github.com/puppeteer/puppeteer/issues/1576). This seems to affect pages that have a large viewport height and causes the fullpage screenshot to duplicate content.
 
-If you are running into this issue there are currently two options: 1. set headless mode to false (app.js line 17); 2. take the fullpage screenshot manually.
+This application currently uses a workaround to take the fullpage screenshot until the above bug is fixed. This workaround uses Jimp [https://www.npmjs.com/package/jimp](https://www.npmjs.com/package/jimp) to merge several screenshots into a fullpage image.
 
 ---
 
diff --git a/app.js b/app.js
index 4b6e0f5..fcf0dc2 100644
--- a/app.js
+++ b/app.js
@@ -1,4 +1,5 @@
 const fs = require("fs");
+const rimraf = require("rimraf");
 const puppeteer = require("puppeteer");
 const screenshot = require("./src/screenshot");
 const saveReport = require("./src/save_report");
@@ -83,7 +84,11 @@ const puppeteerConnect = async (url) => {
 		console.log(err);
 	}
 
-	console.log("\x1b[36m%s\x1b[0m", "all operations completed");
+	console.log("cleaning up temporary files and folders");
+	await rimraf("./outputs/tempImages", function () {
+		console.log("\x1b[36m%s\x1b[0m", "all operations completed");
+	});
+
 	browser.close();
 };
 
diff --git a/package.json b/package.json
index e673922..f8b6ec5 100644
--- a/package.json
+++ b/package.json
@@ -14,6 +14,7 @@
 		"jimp": "^0.16.1",
 		"merge-img": "^2.1.3",
 		"puppeteer": "^5.3.1",
+		"rimraf": "^3.0.2",
 		"uuid": "^8.3.1"
 	},
 	"devDependencies": {}

From b12888b824181b9c62ce5868bd0ca84e76dc70d3 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Fri, 27 Nov 2020 14:14:38 +0000
Subject: [PATCH 18/20] now appending '_report' to .csv filenames

---
 src/save_report.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/save_report.js b/src/save_report.js
index 4b87d73..45dc42c 100644
--- a/src/save_report.js
+++ b/src/save_report.js
@@ -85,8 +85,8 @@ const saveReport = async (page, directory) => {
 	const writeTableToCsv = async (tableName, csvHeaders, csvRows) => {
 		const csvWriter = createCsvWriter({
 			path: fs.existsSync(`./${directory}/report_${tableName}.csv`)
-				? `${directory}/${tableName}_${uuidv4()}.csv`
-				: `${directory}/${tableName}.csv`,
+				? `${directory}/report_${tableName}_${uuidv4()}.csv`
+				: `${directory}/report_${tableName}.csv`,
 			header: csvHeaders,
 		});
 

From 395d865777c0be3f5dc0433a9369127e9b6bb831 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Wed, 2 Dec 2020 15:39:21 +0000
Subject: [PATCH 19/20] now saving reports in separate directory to images

---
 app.js            | 24 +++++++++++++++---------
 src/screenshot.js |  4 ++--
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/app.js b/app.js
index fcf0dc2..3ed186a 100644
--- a/app.js
+++ b/app.js
@@ -50,21 +50,27 @@ const puppeteerConnect = async (url) => {
 	let month = dateObject.getMonth() + 1;
 	let year = dateObject.getFullYear();
 
-	const directory = `./outputs/${
+	const imagesDirectory = `./outputs/${
 		year + "-" + month + "-" + date + "-" + timeStamp
 	}`;
-	if (!fs.existsSync(directory)) {
-		fs.mkdirSync(directory, { recursive: true });
+	const reportsDirectory = `${imagesDirectory}/reports`;
+
+	if (!fs.existsSync(reportsDirectory)) {
+		fs.mkdirSync(reportsDirectory, { recursive: true });
 	}
 
-	const tempImagesDir = "./outputs/tempImages";
-	if (!fs.existsSync(tempImagesDir)) {
-		fs.mkdirSync(tempImagesDir, { recursive: true });
+	const tempImagesDirectory = "./outputs/tempImages";
+	if (!fs.existsSync(tempImagesDirectory)) {
+		fs.mkdirSync(tempImagesDirectory, { recursive: true });
 	}
 
 	// take screenshots of charts
 	try {
-		await screenshot.takeScreenshots(page, directory, tempImagesDir);
+		await screenshot.takeScreenshots(
+			page,
+			imagesDirectory,
+			tempImagesDirectory
+		);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
@@ -75,7 +81,7 @@ const puppeteerConnect = async (url) => {
 
 	// save the information in the 'reports' tab as a .csv
 	try {
-		await saveReport.saveReport(page, directory);
+		await saveReport.saveReport(page, reportsDirectory);
 	} catch (err) {
 		console.log(
 			"\x1b[36m%s\x1b[0m",
@@ -85,7 +91,7 @@ const puppeteerConnect = async (url) => {
 	}
 
 	console.log("cleaning up temporary files and folders");
-	await rimraf("./outputs/tempImages", function () {
+	await rimraf(tempImagesDirectory, function () {
 		console.log("\x1b[36m%s\x1b[0m", "all operations completed");
 	});
 
diff --git a/src/screenshot.js b/src/screenshot.js
index 3269de4..1fef77c 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -3,7 +3,7 @@ const { v4: uuidv4 } = require("uuid");
 const Jimp = require("jimp");
 const mergeImg = require("merge-img");
 
-const takeScreenshots = async (page, directory, tempImagesDir) => {
+const takeScreenshots = async (page, directory, tempImagesDirectory) => {
 	const getElementText = async (elementRef) => {
 		// get the name of the current chart tab
 		const elementTextRef = await page.$(elementRef);
@@ -191,7 +191,7 @@ const takeScreenshots = async (page, directory, tempImagesDir) => {
 			const images = [];
 			for (let i = 0; i < pagesCount; i += 1) {
 				const image = await page.screenshot({
-					path: `./${tempImagesDir}/temp_${i}.png`,
+					path: `./${tempImagesDirectory}/temp_${i}.png`,
 				});
 				await scrollDown();
 				images.push(image);

From 48f27719b55322c32583b23e03f36a70936d2d66 Mon Sep 17 00:00:00 2001
From: James Hughes <james.hughes@ucl.ac.uk>
Date: Thu, 3 Dec 2020 09:30:35 +0000
Subject: [PATCH 20/20] extracted more functions into separate modules

---
 app.js                    | 63 +++++++--------------------------------
 src/create_directories.js | 32 ++++++++++++++++++++
 src/puppeteer_connect.js  | 29 ++++++++++++++++++
 src/save_report.js        |  2 +-
 src/screenshot.js         |  6 ++--
 5 files changed, 75 insertions(+), 57 deletions(-)
 create mode 100644 src/create_directories.js
 create mode 100644 src/puppeteer_connect.js

diff --git a/app.js b/app.js
index 3ed186a..8d6ca53 100644
--- a/app.js
+++ b/app.js
@@ -1,6 +1,7 @@
-const fs = require("fs");
 const rimraf = require("rimraf");
-const puppeteer = require("puppeteer");
+
+const puppeteerConnect = require("./src/puppeteer_connect");
+const createDirectories = require("./src/create_directories");
 const screenshot = require("./src/screenshot");
 const saveReport = require("./src/save_report");
 
@@ -13,56 +14,14 @@ if (!url) {
 	url = "http://localhost:3000";
 }
 
-const puppeteerConnect = async (url) => {
-	const browser = await puppeteer.launch({
-		// headless: false,
-		// slowMo: 250,
-	});
-
-	const page = await browser.newPage();
-	await page.setViewport({ width: 1980, height: 1000 });
-	page.setDefaultTimeout(10000);
-
-	try {
-		await page.goto(url, {
-			waitUntil: "networkidle2",
-		});
-	} catch (err) {
-		console.log(
-			"Having trouple establishing a connection. If you are attempting to connect to localhost try entering the url as your local ip address"
-		);
-		console.log(err);
-		browser.close();
-		return;
-	}
-
-	// helper function to wait for a specified period of time
-	const delay = (time) => {
-		return new Promise(function (resolve) {
-			setTimeout(resolve, time);
-		});
-	};
-
-	// get the current date in YYYY-MM-DD format (for creating directories)
-	let timeStamp = Date.now();
-	let dateObject = new Date(timeStamp);
-	let date = dateObject.getDate();
-	let month = dateObject.getMonth() + 1;
-	let year = dateObject.getFullYear();
+const runApp = async (url) => {
+	const { browser, page } = await puppeteerConnect.puppeteerConnect(url);
 
-	const imagesDirectory = `./outputs/${
-		year + "-" + month + "-" + date + "-" + timeStamp
-	}`;
-	const reportsDirectory = `${imagesDirectory}/reports`;
-
-	if (!fs.existsSync(reportsDirectory)) {
-		fs.mkdirSync(reportsDirectory, { recursive: true });
-	}
-
-	const tempImagesDirectory = "./outputs/tempImages";
-	if (!fs.existsSync(tempImagesDirectory)) {
-		fs.mkdirSync(tempImagesDirectory, { recursive: true });
-	}
+	const {
+		imagesDirectory,
+		reportsDirectory,
+		tempImagesDirectory,
+	} = await createDirectories.createDirectories();
 
 	// take screenshots of charts
 	try {
@@ -98,4 +57,4 @@ const puppeteerConnect = async (url) => {
 	browser.close();
 };
 
-puppeteerConnect(url);
+runApp(url);
diff --git a/src/create_directories.js b/src/create_directories.js
new file mode 100644
index 0000000..ba7d439
--- /dev/null
+++ b/src/create_directories.js
@@ -0,0 +1,32 @@
+const fs = require("fs");
+
+const createDirectories = async () => {
+	// get the current date in YYYY-MM-DD format (for creating directories)
+	let timeStamp = Date.now();
+	let dateObject = new Date(timeStamp);
+	let date = dateObject.getDate();
+	let month = dateObject.getMonth() + 1;
+	let year = dateObject.getFullYear();
+
+	const imagesDirectory = `./outputs/${
+		year + "-" + month + "-" + date + "-" + timeStamp
+	}`;
+	const reportsDirectory = `${imagesDirectory}/reports`;
+
+	if (!fs.existsSync(reportsDirectory)) {
+		fs.mkdirSync(reportsDirectory, { recursive: true });
+	}
+
+	const tempImagesDirectory = "./outputs/tempImages";
+	if (!fs.existsSync(tempImagesDirectory)) {
+		fs.mkdirSync(tempImagesDirectory, { recursive: true });
+	}
+
+	return {
+		imagesDirectory: imagesDirectory,
+		reportsDirectory: reportsDirectory,
+		tempImagesDirectory: tempImagesDirectory,
+	};
+};
+
+exports.createDirectories = createDirectories;
diff --git a/src/puppeteer_connect.js b/src/puppeteer_connect.js
new file mode 100644
index 0000000..a34bad8
--- /dev/null
+++ b/src/puppeteer_connect.js
@@ -0,0 +1,29 @@
+const puppeteer = require("puppeteer");
+
+const puppeteerConnect = async (url) => {
+	const browser = await puppeteer.launch({
+		// headless: false,
+		// slowMo: 250,
+	});
+
+	const page = await browser.newPage();
+	await page.setViewport({ width: 1980, height: 1000 });
+	page.setDefaultTimeout(10000);
+
+	try {
+		await page.goto(url, {
+			waitUntil: "networkidle2",
+		});
+	} catch (err) {
+		console.log(
+			"Having trouple establishing a connection. If you are attempting to connect to localhost try entering the url as your local ip address"
+		);
+		console.log(err);
+		browser.close();
+		return;
+	}
+
+	return { browser: browser, page: page };
+};
+
+exports.puppeteerConnect = puppeteerConnect;
diff --git a/src/save_report.js b/src/save_report.js
index 45dc42c..69ea365 100644
--- a/src/save_report.js
+++ b/src/save_report.js
@@ -102,7 +102,7 @@ const saveReport = async (page, directory) => {
 			});
 	};
 
-	// App flow starts here
+	// module flow starts here
 	await openReportTab();
 	const tableCount = await countTables();
 	if (tableCount === 0) {
diff --git a/src/screenshot.js b/src/screenshot.js
index 1fef77c..5295d4c 100644
--- a/src/screenshot.js
+++ b/src/screenshot.js
@@ -78,9 +78,7 @@ const takeScreenshots = async (page, directory, tempImagesDirectory) => {
 			);
 
 			const tabOpened = await openTabAndWait();
-			if (tabOpened === false) {
-				continue;
-			}
+			if (tabOpened === false) continue;
 		}
 	};
 
@@ -239,7 +237,7 @@ const takeScreenshots = async (page, directory, tempImagesDirectory) => {
 		}
 	};
 
-	// App flow starts here
+	// module flow starts here
 	const chartTabGroupElements = []; // stores the chart tab dom elements
 	const chartTabGroupNames = []; // stores the name of each chart tab
 	const failedScreenshots = []; // stores the names of any charts that failed to screenshot