Add Jupyter Notebook output

CHANGES.md

@@ -1,5 +1,7 @@
 ### 2.4.1
 
+- Add Jupyter Notebook output (#124, @avsm, @CraigFe, @jonludlam)
+
 #### Changed
 
 - Revert #446: "Allow execution of included OCaml code blocks" (#451, @gpetiot).

bin/dune

@@ -1,11 +1,23 @@
 (library
  (name cli)
  (modules cli)
- (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx))
+ (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx yojson atdgen))
 
 (executable
  (name main)
  (public_name ocaml-mdx)
  (package mdx)
  (modules :standard \ cli)
  (libraries cli mdx))
+
+(rule
+ (targets notebook_j.ml notebook_j.mli)
+ (deps notebook.atd)
+ (action
+  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets notebook_t.ml notebook_t.mli)
+ (deps notebook.atd)
+ (action
+  (run atdgen -t %{deps})))

bin/jupyter.ml

@@ -0,0 +1,119 @@
+open Mdx.Util.Result.Infix
+open Cmdliner
+
+let raw t =
+  Notebook_t.
+    {
+      cell_type = `Raw;
+      metadata = { collapsed = None; scrolled = None };
+      source = String.concat "\n" t;
+      outputs = None;
+      execution_count = None;
+    }
+
+let txt source =
+  Notebook_t.
+    {
+      cell_type = `Markdown;
+      metadata = { collapsed = None; scrolled = None };
+      source;
+      outputs = None;
+      execution_count = None;
+    }
+
+let execution_count = ref 1
+
+let ocaml contents =
+  let cell =
+    Notebook_t.
+      {
+        cell_type = `Code;
+        metadata = { collapsed = None; scrolled = None };
+        source = String.concat "\n" contents;
+        outputs = Some [];
+        execution_count = Some !execution_count;
+      }
+  in
+  incr execution_count;
+  cell
+
+let toplevel x =
+  let cell =
+    Notebook_t.
+      {
+        cell_type = `Code;
+        metadata = { collapsed = None; scrolled = None };
+        source = String.concat "\n" x.Mdx.Toplevel.command;
+        outputs = Some [];
+        execution_count = Some !execution_count;
+      }
+  in
+  incr execution_count;
+  cell
+
+let metadata =
+  Notebook_t.
+    {
+      kernelspec =
+        {
+          display_name = "OCaml 4.07.1";
+          language = "OCaml";
+          name = "ocaml-jupyter";
+        };
+      language_info =
+        {
+          name = "OCaml";
+          version = "4.07.1";
+          codemirror_mode = Some "text/x-ocaml";
+          file_extension = ".ml";
+          mimetype = "text/x-ocaml";
+          nbconverter_exporter = None;
+          pygments_lexer = "OCaml";
+        };
+    }
+
+let rec collapse_text = function
+  | Mdx.Text x :: Mdx.Text y :: xs ->
+      collapse_text (Mdx.Text (x ^ "\n" ^ y) :: xs)
+  | (Mdx.Section _ as s) :: Mdx.Text y :: xs ->
+      let s = Mdx.to_string [ s ] in
+      collapse_text (Mdx.Text (s ^ "\n" ^ y) :: xs)
+  | (Mdx.Section _ as s) :: xs ->
+      let s = Mdx.to_string [ s ] in
+      collapse_text (Mdx.Text s :: xs)
+  | x :: ys -> x :: collapse_text ys
+  | [] -> []
+
+let run _setup (`File file) =
+  Mdx.run_to_stdout file ~f:(fun _file_contents items ->
+      let cells =
+        List.fold_left
+          (fun acc -> function
+            | Mdx.Text "" -> acc
+            | Mdx.Text x -> txt x :: acc
+            | Mdx.Block { value = OCaml { env = User_defined _; _ }; _ }
+            | Mdx.Block { value = Toplevel { env = User_defined _; _ }; _ } ->
+                failwith
+                  "internal error, cannot handle user defined environments"
+            | Mdx.Block { value = OCaml _; contents; _ } ->
+                ocaml contents :: acc
+            | Mdx.Block { value = Toplevel _; contents; loc; _ } ->
+                let blocks = Mdx.Toplevel.of_lines ~loc contents in
+                let newcells = List.rev_map toplevel blocks.Mdx.Toplevel.tests in
+                newcells @ acc
+            | Mdx.Block { value = Raw _; contents; _ } -> raw contents :: acc
+            | x ->
+                failwith
+                  (Printf.sprintf "internal error, cannot handle: %s"
+                     (Mdx.to_string [ x ])))
+          [] (collapse_text items)
+        |> List.rev
+      in
+      Notebook_j.string_of_notebook
+        Notebook_t.{ metadata; nbformat = 4; nbformat_minor = 2; cells })
+  >>! fun () -> 0
+
+let term = Term.(const run $ Cli.setup $ Cli.file)
+let doc = "Convert an mdx file to a jupyter notebook."
+let info = Cmd.info "jupyter" ~doc
+let cmd = Cmd.v info term

bin/main.ml

@@ -16,7 +16,8 @@
 
 open Cmdliner
 
-let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd ]
+let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd; Jupyter.cmd ]
+
 let main (`Setup ()) = `Help (`Pager, None)
 
 let info =

bin/notebook.atd

@@ -0,0 +1,57 @@
+type kernelspec = {
+    display_name : string;
+    language: string;
+    name: string;
+}
+
+type language_info = {
+    name: string;
+    version: string;
+    codemirror_mode: string nullable;
+    file_extension: string;
+    mimetype: string;
+    nbconverter_exporter: string nullable;
+    pygments_lexer: string;
+}
+
+type metadata = {
+    kernelspec: kernelspec;
+    language_info: language_info;
+}
+
+type cell_metadata = {
+    ?collapsed: bool nullable;
+    ?scrolled: bool nullable;
+}
+
+type cell_type = [
+    | Code <json name="code">
+    | Markdown <json name="markdown">
+    | Raw <json name="raw">
+]
+
+type output_type = [
+      Stream <json name="stream">
+    | Display_data <json name="display_data">
+    | Execute_result <json name="execute_result">
+    | Error <json name="error">
+]
+
+type output = {
+    output_type : output_type;
+}
+
+type cell = {
+    cell_type : cell_type;
+    metadata: cell_metadata;
+    source: string;
+    ?outputs: output list nullable;
+    ?execution_count: int option;
+} <json keep_nulls="false">
+
+type notebook = {
+    metadata: metadata;
+    nbformat: int;
+    nbformat_minor: int;
+    cells: cell list
+}

dune-project

@@ -29,11 +29,13 @@
   (csexp
    (>= 1.3.2))
   astring
+  atdgen
   (logs (>= 0.7.0))
   (cmdliner
    (>= 1.1.0))
   (re
    (>= 1.7.2))
+  yojson
   (ocaml-version
    (>= 2.3.0))
   (lwt :with-test)

mdx.opam

@@ -25,9 +25,11 @@ depends: [
   "cppo" {build & >= "1.1.0"}
   "csexp" {>= "1.3.2"}
   "astring"
+  "atdgen"
   "logs" {>= "0.7.0"}
   "cmdliner" {>= "1.1.0"}
   "re" {>= "1.7.2"}
+  "yojson"
   "ocaml-version" {>= "2.3.0"}
   "lwt" {with-test}
   "camlp-streams"

test/bin/misc-test-cases/mdx-jupyter/dune

@@ -0,0 +1,17 @@
+(rule
+ (targets test.jpynb)
+ (deps
+  (:x test.md)
+  (package mdx))
+ (action
+  (with-stdout-to
+   %{targets}
+   (run %{bin:ocaml-mdx} jupyter %{x}))))
+
+(rule
+ (alias runtest)
+ (deps
+  (:x test.jpynb)
+  (:y test.jpynb.expected))
+ (action
+  (diff? %{y} %{x})))

test/bin/misc-test-cases/mdx-jupyter/test.jpynb.expected

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"display_name":"OCaml 4.07.1","language":"OCaml","name":"ocaml-jupyter"},"language_info":{"name":"OCaml","version":"4.07.1","codemirror_mode":"text/x-ocaml","file_extension":".ml","mimetype":"text/x-ocaml","nbconverter_exporter":null,"pygments_lexer":"OCaml"}},"nbformat":4,"nbformat_minor":2,"cells":[{"cell_type":"markdown","metadata":{},"source":"## Generating a Jupyter notebook\n\n\n\n"},{"cell_type":"code","metadata":{},"source":"\nlet x = 1\nlet () = assert true\n","outputs":[],"execution_count":1},{"cell_type":"markdown","metadata":{},"source":"\n\nThis block should not be executed by the tests:\n"},{"cell_type":"code","metadata":{},"source":"\nlet x = 2\nlet () = assert false (* Don't print this! *)\n","outputs":[],"execution_count":2},{"cell_type":"markdown","metadata":{},"source":"\n"},{"cell_type":"code","metadata":{},"source":"\nlet () = print_int x\n","outputs":[],"execution_count":3}]}

test/bin/misc-test-cases/mdx-jupyter/test.md

@@ -0,0 +1,16 @@
+## Generating a Jupyter notebook
+
+```ocaml
+let x = 1
+let () = assert true
+```
+
+This block should not be executed by the tests:
+```ocaml skip
+let x = 2
+let () = assert false (* Don't print this! *)
+```
+
+```ocaml
+let () = print_int x
+```