Merge pull request #16 from xcube-dev/pont-14-handle-magic

Make parameter extraction more robust and ignore magic

Author: pont-us

CHANGES.md

@@ -3,6 +3,7 @@
 * Improve automated parameter extraction (#9)
 * Handle notebooks without parameters (#11)
 * Handle non-code cells (#12)
+* Ignore magic commands in input notebooks (#14)
 
 ## Changes in 0.1.0
 

test/data/paramtest.ipynb

@@ -37,7 +37,7 @@
    "id": "325a90ed-c291-4277-9c0f-99c5543ac218",
    "metadata": {
     "editable": true,
-    "raw_mimetype": "",
+    "raw_mimetype": "text/asciidoc",
     "slideshow": {
      "slide_type": ""
     },
@@ -47,6 +47,94 @@
     "This is a raw cell."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "beb2ee7b-e984-4613-ba37-a486c40527f6",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "Below are some magic commands, to make sure that these are handled during\n",
+    "parameter extraction and conversion."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "d6b05d5c-87a7-4002-b7b1-aead36c08482",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<h3>Hi!</h3>\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "%%html\n",
+    "<h3>Hi!</h3>"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "eef41e77-30e5-4138-ad27-810b31161194",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%dirs"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "567258c5-847f-4d6a-be80-9d69991df102",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "source": [
+    "Now the parameter cell! In a real notebook this should ideally come as early as possible,\n",
+    "but in this test notebook we add some \"tricky\" cells before it to make sure that xcetool\n",
+    "can deal with them."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

xcengine/core.py

@@ -2,10 +2,8 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-import functools
 import io
 import json
-import operator
 import os
 import shutil
 import sys
@@ -55,6 +53,10 @@ def convert_notebook_to_script(
         exporter = nbconvert.PythonExporter()
         (body, resources) = exporter.from_notebook_node(self.notebook)
         with open(output_dir / "user_code.py", "w") as fh:
+            fh.write(
+                    "import unittest.mock\n"
+                    "get_ipython = unittest.mock.MagicMock\n"
+            )
             fh.write(body)
         parent_dir = pathlib.Path(__file__).parent
         shutil.copy2(parent_dir / "wrapper.py", output_dir / "execute.py")
@@ -73,17 +75,23 @@ def process_params_cell(self) -> None:
                 params_cell_index = i
                 break
         if params_cell_index is not None:
-            setup_code = "\n".join(
-                map(
-                    operator.attrgetter("source"),
-                    filter(
-                        lambda c: c.cell_type == "code",
-                        self.notebook.cells[:params_cell_index],
-                    ),
-                )
+            setup_node = nbformat.from_dict(self.notebook)
+            setup_node.cells = setup_node.cells[:params_cell_index]
+            exporter = nbconvert.PythonExporter()
+            (setup_code, _) = exporter.from_notebook_node(setup_node)
+            # Mock out the get_ipython function in case there are any
+            # IPython magic commands in the notebook. This effectively
+            # turns them into no-ops
+            setup_code = (
+                "import unittest.mock\n"
+                "get_ipython = unittest.mock.MagicMock\n"
+                + setup_code
             )
+            params_node = nbformat.from_dict(self.notebook)
+            params_node.cells = [params_node.cells[params_cell_index]]
+            (params_code, _) = exporter.from_notebook_node(params_node)
             self.nb_params = NotebookParameters.from_code(
-                self.notebook.cells[params_cell_index].source,
+                params_code,
                 setup_code=setup_code,
             )
             self.notebook.cells.insert(