Show progress bars when downloading assets, update README (#257)

* Display download progress for remotely hosted assets 
* Fix race condition between inputs and available options, avoiding duplicate downloads of the same asset
* Show logs for download progress
* Update README
* Bump app version

Author: jkanche

README.md

@@ -70,6 +70,23 @@ and diagnostic plots from the individual analysis steps.
 - Clicking on "What's happening" will show logs describing how long each step of the analysis took (and any errors during the analysis).
 - Clicking Export will save the analysis either to the browser or download the analysis as a .kana file. Loading these files will restore the state of the application
 
+If you use **Kana** for analysis or exploration, consider citing our JOSS publication -
+
+```bibtex
+@article{Kana2023, 
+  doi = {10.21105/joss.05603}, 
+  url = {https://doi.org/10.21105/joss.05603}, 
+  year = {2023}, 
+  publisher = {The Open Journal}, 
+  volume = {8}, 
+  number = {89}, 
+  pages = {5603}, 
+  author = {Aaron Tin Long Lun and Jayaram Kancherla}, 
+  title = {Powering single-cell analyses in the browser with WebAssembly}, 
+  journal = {Journal of Open Source Software} 
+}
+```
+
 ## For developers
 
 ***Check out [Contributing](./CONTRIBUTING.md) for guidelines on opening issues and pull requests.***

package.json

@@ -1,7 +1,7 @@
 {
   "name": "kana",
   "description": "Single-cell data analysis in the browser",
-  "version": "3.0.22",
+  "version": "3.0.23",
   "author": {
     "name": "Jayaram Kancherla",
     "email": "[email protected]",

src/DownloadToaster.js

@@ -0,0 +1,60 @@
+import {
+  OverlayToaster,
+  Position,
+  ProgressBar,
+  Classes,
+} from "@blueprintjs/core";
+
+import { Tooltip2 } from "@blueprintjs/popover2";
+
+import classNames from "classnames";
+
+export const DownloadToaster = OverlayToaster.create({
+  className: "recipe-toaster",
+  position: Position.TOP_RIGHT,
+});
+
+let download_toasters = {};
+
+export function setProgress(id, total, progress) {
+  if (total !== null) {
+    download_toasters["total"] = total;
+    download_toasters["progress"] = progress;
+  }
+
+  if (progress !== null) {
+    let tprogress =
+      (Math.round((progress * 100) / download_toasters["total"]) / 100) * 100;
+
+    download_toasters["progress"] = tprogress;
+  }
+}
+
+export function renderProgress(progress, url) {
+  return {
+    icon: "cloud-download",
+    message: (
+      <>
+        <>
+          Downloading asset from{" "}
+          <Tooltip2
+            className={Classes.TOOLTIP_INDICATOR}
+            content={<span>{url}</span>}
+            minimal={true}
+            usePortal={false}
+          >
+            {new URL(url).hostname}
+          </Tooltip2>
+        </>
+        <ProgressBar
+          className={classNames("docs-toast-progress", {
+            [Classes.PROGRESS_NO_STRIPES]: progress >= 100,
+          })}
+          intent={progress < 100 ? "primary" : "success"}
+          value={progress / 100}
+        />
+      </>
+    ),
+    timeout: progress < 100 ? 0 : 1000,
+  };
+}

src/components/AnalysisMode/index.js

@@ -42,6 +42,8 @@ import Gallery from "../Gallery/index";
 
 import { AppToaster } from "../../AppToaster";
 
+import { renderProgress, DownloadToaster } from "../../DownloadToaster";
+
 import { AppContext } from "../../context/AppContext";
 
 import pkgVersion from "../../../package.json";
@@ -59,6 +61,8 @@ const scranWorker = new Worker(
 
 let logs = [];
 
+let download_toasters = {};
+
 export function AnalysisMode(props) {
   // true until wasm is initialized
   const [loading, setLoading] = useState(true);
@@ -135,6 +139,8 @@ export function AnalysisMode(props) {
   const [initDims, setInitDims] = useState(null);
   const [inputData, setInputData] = useState(null);
 
+  const [preInputDone, setPreInputDone] = useState(false);
+
   // STEP: QC; for all three, RNA, ADT, CRISPR
   // dim sizes
   const [qcDims, setQcDims] = useState(null);
@@ -307,6 +313,7 @@ export function AnalysisMode(props) {
   useEffect(() => {
     if (wasmInitialized && preInputFiles) {
       if (preInputFiles.files) {
+        setPreInputDone(false);
         scranWorker.postMessage({
           type: "PREFLIGHT_INPUT",
           payload: {
@@ -318,7 +325,7 @@ export function AnalysisMode(props) {
   }, [preInputFiles, wasmInitialized]);
 
   useEffect(() => {
-    if (wasmInitialized && preInputFiles) {
+    if (wasmInitialized && preInputFiles && preInputDone && preInputOptions) {
       if (preInputFiles.files && preInputOptions.options.length > 0) {
         scranWorker.postMessage({
           type: "PREFLIGHT_OPTIONS",
@@ -329,7 +336,7 @@ export function AnalysisMode(props) {
         });
       }
     }
-  }, [preInputOptions, wasmInitialized]);
+  }, [preInputOptions, wasmInitialized, preInputDone]);
 
   // NEW analysis: files are imported into Kana
   useEffect(() => {
@@ -854,7 +861,7 @@ export function AnalysisMode(props) {
   scranWorker.onmessage = (msg) => {
     const payload = msg.data;
 
-    // console.log("ON MAIN::RCV::", payload);
+    // console.log("IN ANALYSIS MODE, ON MAIN::RCV::", payload);
 
     // process any error messages
     if (payload) {
@@ -904,6 +911,57 @@ export function AnalysisMode(props) {
       }
     }
 
+    if (payload.type.startsWith("DOWNLOAD")) {
+      if (payload.download == "START") {
+        download_toasters[payload.url] = {
+          total: payload.total_bytes,
+          progress: 0,
+        };
+        download_toasters[payload.url]["key"] = DownloadToaster.show(
+          renderProgress(0, payload.url)
+        );
+
+        add_to_logs("start", `Download asset from ${payload.url}`, "started");
+      } else if (payload.download == "PROGRESS") {
+        let tprogress =
+          (Math.round(
+            (payload.downloaded_bytes * 100) /
+              download_toasters[payload.url]["total"]
+          ) /
+            100) *
+          100;
+
+        if (tprogress < 100) {
+          download_toasters[payload.url]["progress"] = tprogress;
+
+          download_toasters[payload.url]["key"] = DownloadToaster.show(
+            renderProgress(tprogress, payload.url),
+            download_toasters[payload.url]["key"]
+          );
+        }
+        add_to_logs("progress", `Downloading ${tprogress}% done`, "");
+      } else if (payload.download == "COMPLETE") {
+        download_toasters[payload.url]["progress"] = 100;
+
+        download_toasters[payload.url]["key"] = DownloadToaster.show(
+          renderProgress(100, payload.url),
+          download_toasters[payload.url]["key"]
+        );
+
+        add_to_logs(
+          "complete",
+          `Asset downloaded from ${payload.url}`,
+          "finished"
+        );
+
+        setTimeout(() => {
+          delete download_toasters[payload.url];
+        }, 500);
+      }
+
+      return;
+    }
+
     const { resp, type } = payload;
 
     if (type === "INIT") {
@@ -927,6 +985,7 @@ export function AnalysisMode(props) {
     } else if (type === "PREFLIGHT_INPUT_DATA") {
       if (resp.details) {
         setPreInputFilesStatus(resp.details);
+        setPreInputDone(true);
       }
     } else if (type === "PREFLIGHT_OPTIONS_DATA") {
       if (resp) {

src/workers/helpers.js

@@ -6,8 +6,91 @@ import * as downloads from "./DownloadsDBHandler.js";
 // Evade CORS problems and enable caching.
 const proxy = "https://cors-proxy.aaron-lun.workers.dev";
 async function proxyAndCache(url) {
-  let buffer = await downloads.get(proxy + "/" + encodeURIComponent(url));
-  return new Uint8Array(buffer);
+  const url_with_proxy = proxy + "/" + encodeURIComponent(url);
+
+  try {
+    const out = await fetchWithProgress(
+      url_with_proxy,
+      (cl) => {
+        postMessage({
+          type: `DOWNLOAD for url: ` + String(url),
+          download: "START",
+          url: String(url),
+          total_bytes: String(cl),
+          msg: "Total size is " + String(cl) + " bytes!",
+        });
+        return url_with_proxy;
+      },
+      (id, sofar) => {
+        postMessage({
+          type: `DOWNLOAD for url: ` + String(url),
+          download: "PROGRESS",
+          url: String(url),
+          downloaded_bytes: String(sofar),
+          msg: "Progress so far, got " + String(sofar) + " bytes!",
+        });
+      },
+      (id, total) => {
+        postMessage({
+          type: `DOWNLOAD for url: ` + String(url),
+          download: "COMPLETE",
+          url: String(url),
+          msg: "Finished, got " + String(total) + " bytes!",
+        });
+      }
+    );
+
+    return out;
+  } catch (error) {
+    // console.log("oops error", error)
+    postMessage({
+      type: `DOWNLOAD for url: ` + String(url),
+      download: "START",
+      url: String(url),
+      total_bytes: 100,
+    });
+    let buffer = await downloads.get(url_with_proxy);
+    postMessage({
+      type: `DOWNLOAD for url: ` + String(url),
+      download: "COMPLETE",
+      url: String(url),
+    });
+    return new Uint8Array(buffer);
+  }
+}
+
+async function fetchWithProgress(url, startFun, iterFun, endFun) {
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error("oops, failed to download '" + url + "'");
+  }
+
+  const cl = res.headers.get("content-length"); // WARNING: this might be NULL!
+  const id = startFun(cl);
+
+  const reader = res.body.getReader();
+  const chunks = [];
+  let total = 0;
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) {
+      break;
+    }
+    chunks.push(value);
+    total += value.length;
+    iterFun(id, total);
+  }
+
+  let output = new Uint8Array(total);
+  let start = 0;
+  for (const x of chunks) {
+    output.set(x, start);
+    start += x.length;
+  }
+
+  endFun(id, total);
+  return output;
 }
 
 bakana.CellLabellingState.setDownload(proxyAndCache);

src/workers/scran.worker.js

@@ -16,6 +16,7 @@ import {
   fetchStepSummary,
   describeColumn,
   isArrayOrView,
+  fetchWithProgress,
 } from "./helpers.js";
 import { code } from "../utils/utils.js";
 /***************************************/
@@ -306,7 +307,7 @@ var loaded;
 onmessage = function (msg) {
   const { type, payload } = msg.data;
 
-  // console.log("WORKER::RCV::", type, payload);
+  // console.log("SCRAN.WORKER ::RCV::", type, payload);
 
   let fatal = false;
   if (type === "INIT") {