better rendering on loading ProgressiBooks via png images

ipyprogressivis/__init__.py

@@ -56,4 +56,55 @@ def _jupyter_server_extension_points():
     return [{"module": "ipyprogressivis.app"}]
 
 
+def pre_save(model, **kwargs):
+    """copy ProgressiVis snapshots to cells outputs before saving notebooks"""
+    # only run on notebooks
+    if model['type'] != 'notebook':
+        return
+    # only run on nbformat v4
+    if model['content']['nbformat'] != 4:
+        return
+    metadata = model["content"]["metadata"]
+    if not (outs := metadata.get("progressivis_outs")):
+        return
+    cell_1 = model['content']['cells'][1]  # if progressivis_outs exists then 1 index exists too
+    if cell_1['cell_type'] != 'code':
+        print("Unconsistent ProgressiBook, exit")
+        return
+    for out in cell_1.get('outputs', []):
+        if out.get("data", {}).get("text/plain") in ("Talker()", "BackupWidget()"):
+            out["data"]["text/plain"] = ""
+    for i, cell in enumerate(model['content']['cells']):
+        if cell['cell_type'] != 'code':
+            continue
+        if i >= len(outs) or not outs[i]:  # an empty dict actually
+            continue
+        _, b64_data = outs[i].split(",", 1)
+        for j, out in enumerate(cell['outputs']):
+            if out["output_type"] not in ("execute_result", "display_data"):
+                continue
+            out["data"] = {
+                "image/png": b64_data
+            }
+    metadata["progressivis_outs"] = []
+    if not (dag_png := metadata.get("progressivis_dag_png")):
+        return
+    _, b64_data = dag_png.split(",", 1)
+    if not (wg_values := metadata.get("widgets", {})
+            .get('application/vnd.jupyter.widget-state+json', {})
+            ).values():
+        return
+    for _, wg in wg_values.get("state", {}).items():
+        if wg.get("model_module") != "@jupyter-widgets/jupyterlab-sidecar":
+            continue
+        for out in wg.get("state", {}).get("outputs", []):
+            if out.get("data", {}).get("text/plain") != "DagWidgetController()":
+                continue
+            out["data"] = {
+                "image/png": b64_data,
+                "output_type": "display_data",
+                "metadata": {}
+            }
+
+
 __all__ = ["__version__"]

ipyprogressivis/js/package.json

@@ -65,6 +65,7 @@
     "d3": "^7.6.1",
     "d3-dag": "^0.11.5",
     "datatables": "^1.10.18",
+    "html2canvas": "^1.4.1",
     "jquery-sparkline": "^2.4.0",
     "jupyter-dataserializers": "^3.0.1",
     "jupyter-vega": "^4.0.0",

ipyprogressivis/js/src/commands.js

@@ -1,4 +1,6 @@
 import { INotebookTracker, NotebookActions } from "@jupyterlab/notebook";
+import * as html2canvas from "html2canvas";
+import $ from "jquery";
 
 export function progressivisTemplate(app, res, data, browser) {
   const content = res.json();
@@ -28,6 +30,9 @@ export function progressivisTemplate(app, res, data, browser) {
   });
 }
 
+// https://discourse.jupyter.org/t/how-to-listen-to-cell-execution/14714
+// https://jupyterlab.readthedocs.io/en/latest/api/classes/notebook.NotebookActions-1.html
+
 export function progressivisCleanup(app, nbtracker) {
   // cleanup sidecars
   let widgets = [];
@@ -112,7 +117,10 @@ export function setRootBackup(nbtracker, backupstring) {
   var notebook = crtWidget.content;
   var backupCell = notebook.widgets[0];
   console.log("backup root markdown", backupCell.model.metadata, backupstring);
-  backupCell.model.sharedModel.setMetadata("progressivis_root_backup", backupstring);
+  backupCell.model.sharedModel.setMetadata(
+    "progressivis_root_backup",
+    backupstring,
+  );
 }
 
 export function createStageCells(nbtracker, tag, md, code, rw, run) {
@@ -165,3 +173,34 @@ export function runCellAt(nbtracker, ix) {
   notebook.activeCellIndex = ix;
   NotebookActions.run(notebook, crtWidget.sessionContext);
 }
+
+export function shotCellAtIndex(notebook, cell, i, delay) {
+  let prevOuts = notebook.model.metadata.progressivis_outs || [];
+  function fun() {
+    html2canvas(cell.outputArea.node.children[0].children[1]).then((canvas) => {
+      let png = canvas.toDataURL("image/png");
+      prevOuts[i] = png;
+      notebook.model.sharedModel.setMetadata("progressivis_outs", prevOuts);
+    });
+  }
+  function fun2() {
+    html2canvas($("[id^='dag_widget_']")[0]).then((canvas) => {
+      let png = canvas.toDataURL("image/png");
+      notebook.model.sharedModel.setMetadata("progressivis_dag_png", png);
+    });
+  }
+  setTimeout(fun, delay);
+  setTimeout(fun2, delay);
+}
+
+export function shotCell(nbtracker, tag, delay) {
+  var crtWidget = nbtracker.currentWidget;
+  var notebook = crtWidget.content;
+  let i = notebook.widgets.findIndex(
+    (x) =>
+      x.model.metadata.progressivis_tag === tag &&
+      x.model.sharedModel.cell_type === "code",
+  );
+  var cell = notebook.widgets[i];
+  shotCellAtIndex(notebook, cell, i, delay);
+}

ipyprogressivis/js/src/dag_widget.js

@@ -58,7 +58,7 @@ export class DagWidgetView extends DOMWidgetView {
 	    console.log("dag widget", this.id);
 	var mainDiv = d3.select('#' + this.id);
             installInterface(mainDiv, '_'+this.id);
-	    d3.select('#' + this.id).show();
+	    try{d3.select('#' + this.id).show();} catch(err){};
 
     });
         this.value_changed();

ipyprogressivis/js/src/labplugin.js

@@ -5,6 +5,9 @@ import { IJupyterWidgetRegistry, DOMWidgetView } from "@jupyter-widgets/base";
 import { INotebookTracker, NotebookActions } from "@jupyterlab/notebook";
 import { IFileBrowserFactory } from "@jupyterlab/filebrowser";
 import { request } from "requests-helper";
+import * as html2canvas from "html2canvas";
+import $ from "jquery";
+
 export const progressivisPlugin = {
   id: "jupyter-progressivis:plugin",
   requires: [IJupyterWidgetRegistry, INotebookTracker, IFileBrowserFactory],
@@ -111,10 +114,23 @@ export const progressivisPlugin = {
         );
       },
     });
+    app.commands.addCommand("progressivis:shot_cell", {
+      label: "Shot cell",
+      caption: "Shot cell",
+      execute: (args) => {
+        cmds.shotCell(nbtracker, args.tag, args.delay);
+      },
+    });
+    NotebookActions.executed.connect((_, args) => {
+      let { cell, notebook, success } = args;
+      let i = notebook.widgets.findIndex((x) => x == cell);
+      if (cell.model.metadata.progressivis_tag === undefined) return;
+      cmds.shotCellAtIndex(notebook, cell, i, 3000);
+    });
     const TalkerView = class extends DOMWidgetView {
       render() {
         this.model.on("msg:custom", (ev) => {
-          console.log("messs", ev);
+          console.log("messg:", ev);
           let args = { ...ev };
           let cmd = args.cmd;
           delete args.cmd;
@@ -130,7 +146,10 @@ export const progressivisPlugin = {
         var notebook = crtWidget.content;
         var backupCell = notebook.widgets[0];
         this.model.set("value", backupCell.model.metadata.progressivis_backup);
-        this.model.set("root_markdown", backupCell.model.metadata.progressivis_root_backup || "");
+        this.model.set(
+          "root_markdown",
+          backupCell.model.metadata.progressivis_root_backup || "",
+        );
         this.touch();
       }
 

ipyprogressivis/widgets/__init__.py

@@ -13,6 +13,6 @@
 from .dag_widget import *
 from .chaining.desc_stats import *
 from .chaining.constructor import *
-from .chaining.utils import create_stage_widget, cleanup_cells, get_header
+from .chaining.utils import create_stage_widget, get_header
 from .chaining.group_by import *
 from .vega import VegaWidget

ipyprogressivis/widgets/chaining/utils.py

@@ -13,7 +13,7 @@
 from progressivis.core.utils import normalize_columns
 from progressivis.core import aio
 import asyncio
-from ._js import jslab_func_remove, jslab_func_cleanup, jslab_func_cell_index
+from ._js import jslab_func_remove
 from ..csv_sniffer import CSVSniffer
 from collections import defaultdict
 from .. import DagWidgetController  # type: ignore
@@ -83,6 +83,15 @@ def dot_progressivis() -> str:
     return ""
 
 
+def shot_cell(func: Callable[..., AnyType]) -> Callable[..., AnyType]:
+    def wrapper(*args: Any, **kwargs: Any) -> None:
+        self_ = args[0]
+        assert isinstance(self_, GuestWidget)
+        func(*args, **kwargs)
+        labcommand("progressivis:shot_cell", tag=self_.carrier.title, delay=3000)
+    return wrapper
+
+
 def runner(func: Callable[..., AnyType]) -> Callable[..., AnyType]:
     def wrapper(*args: Any, **kwargs: Any) -> "NodeCarrier":
         self_ = args[0]
@@ -443,7 +452,7 @@ def replay_sequence(obj: "Constructor") -> None:
     print("widget list:", widget_list)
     for md, code in widget_list:
         labcommand(
-            "progressivis:create_stage_cells", tag=id(md),
+            "progressivis:create_stage_cells", tag=id(md),  # TODO: fix tag
             md=md, code=code, rw=False, run=True
         )
 
@@ -459,7 +468,7 @@ async def _func() -> None:
         extra = backup.root_markdown
         md = f"## root\n {extra}" if extra else "## root"
         labcommand(
-            "progressivis:create_stage_cells", tag=id(md),
+            "progressivis:create_stage_cells", tag="root",
             md=md, code=code, rw=False, run=True
         )
     loop = asyncio.get_event_loop()
@@ -958,17 +967,6 @@ def make_loader_box(self, ftype: str = "csv", disabled: bool = False) -> ipw.HBo
         return ipw.HBox([alias_inp, btn])
 
 
-def cleanup_cells() -> None:
-    manager = PARAMS["header"].manager
-    manager.exec_js(jslab_func_cleanup)
-
-
-def insert_cell_at_index(kind: str, text: str, index: int, tag: str) -> None:
-    get_dag().exec_js(
-        jslab_func_cell_index.format(kind=kind, text=text, index=index, tag=tag)
-    )
-
-
 def get_previous(obj: "ChainingWidget") -> "ChainingWidget":
     if not obj.subwidgets:
         return obj
@@ -1077,12 +1075,12 @@ def add_new_stage(
 ) -> None:
     stage = create_stage_widget(title, frozen, number=number)
     parent_key = key_by_id[id(parent)]
-    tag = id(stage)
     n = stage.number
     end = ""
     if frozen is not None and is_replay():
         end = ".run()"
-    md = "## " + title + (f"[{n}]" if n else "")
+    tag = title + (f"[{n}]" if n else "")
+    md = "## " + tag
     if markdown:
         md = md + "\n" + markdown
     code, rw, run = get_stage_cell(key=title, num=n, end=end, frozen=frozen)
@@ -1112,15 +1110,16 @@ def add_new_loader(
 ) -> None:
     title = f"{ftype.upper()} loader"
     stage = create_loader_widget(title, ftype, alias, frozen=frozen, number=number)
-    tag = id(stage)
     n = stage.number
     end = ""
     if frozen is not None and is_replay():
         end = ".run()"
     if alias:
         md = f"## {alias}"
+        tag = alias
     else:
-        md = "## " + title + (f"[{n}]" if n else "")
+        tag = title + (f"[{n}]" if n else "")
+        md = "## " + tag
     if markdown:
         md = md + "\n" + markdown
     code, rw, run = get_loader_cell(
@@ -1287,6 +1286,7 @@ def get_widget_by_key(self, key: Tuple[str, int]) -> "VBox":
     def dag_running(self) -> None:
         self.carrier.dag_running()
 
+    @shot_cell
     def make_chaining_box(self) -> None:
         self.carrier.make_chaining_box()