start work on notebook to supplement api

Signed-off-by: Lance-Drane <[email protected]>

Author: Lance-Drane

examples-proposed/004-time-loop/sim.conf

@@ -21,13 +21,17 @@ PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org
 # This MUST be an absolute directory
 # if executing portal on a web server also running JupyterHub, you can set this value to manually add your file to this location.
 # NOTE: "/ipsframework/runs/${RUNID}/" will automatically be prepended to the file, guaranteeing that files will NEVER be overwritten.
+# NOTE FOR NERSC SPECIFICALLY: if you want to share notebooks with other users, you must make sure they have the correct file permissions in the directory.
 JUPYTERHUB_DIR = ${PSCRATCH}
 # OPTIONAL
 # if PORTAL_URL is defined + JUPYTERHUB_DIR is defined + this value is defined, the IPSFramework can send a direct link to the JupyterHub file.
 # In the framework, we don't make any assumptions about how JupyterHub's base URL or root server path is configured; you'll generally want to provide the full URL up to the JupyterHub base directory.
 # the URL will always end with /ipsframework/runs/${RUNID}/${JUPYTER_FILE}.ipynb , but:
 #    - /lab/tree may not necessarily be the URL path that JupyterHub users to expose the tree
 #    - the path of the URL after /lab/tree MAY deviate from the value in JUPYTERHUB_DIR
+#
+# NOTES FOR NERSC SPECIFICALLY:
+#   - if you think you will need to execute notebooks on a different node, replace "perlmutter-login-node-base" with the type of node you want to run on.
 JUPYTERHUB_URL = https://jupyter.nersc.gov/user/${USER}/perlmutter-login-node-base/lab/tree${PSCRATCH}
 
 STATE_FILES = state.json

ipsframework/_jupyter/api_v1.py

@@ -51,8 +51,10 @@ def generate_tar_from_runids(runids: Union[Iterable[int], int]) -> str:
     tarball = THIS_DIR / f'{tarball_name}.tar.gz'
     with tarfile.open(tarball, 'w:gz') as archive:
         # add API files inside the tarball
-        for api_file in THIS_DIR.glob('api_v*.py'):
-            archive.add(api_file, arcname=os.path.join(tarball_name, api_file.name))
+        for api_file in THIS_DIR.glob('api_v*'):
+            if api_file.suffix in ('.py', '.ipynb'):
+                arcname = os.path.join(tarball_name, api_file.name)
+                archive.add(api_file, arcname=arcname)
 
         if isinstance(runids, int):
             runids = [runids]

ipsframework/_jupyter/api_v1_notebook.ipynb

@@ -0,0 +1,58 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## IPS workstation\n",
+    "\n",
+    "You can use this notebook to quickly generate a tarfile with desired runids for download. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "import api_v1\n",
+    "from IPython.display import display\n",
+    "from ipywidgets import HTML, Button, Layout, Textarea\n",
+    "\n",
+    "widget1 = Textarea(\n",
+    "    value='',\n",
+    "    placeholder='Enter runids you want to download, delimited by either spaces or newlines',\n",
+    "    description='Enter runids you want to download, delimited by either spaces or newlines',\n",
+    "    layout=Layout(width='50%', display='flex', flex_flow='column')\n",
+    ")\n",
+    "\n",
+    "widget2 = Button(\n",
+    "    description='Generate tar from input',\n",
+    "    layout=Layout(width='300px')\n",
+    ")\n",
+    "\n",
+    "def generate_tarfile(_button_widget):\n",
+    "    runids = [int(v) for v in widget1.value.split()]\n",
+    "    display(f'Generating tar file from runids: {runids}')\n",
+    "    \n",
+    "    file = Path(api_v1.generate_tar_from_runids(runids))\n",
+    "    display(f'Generated tar file {file.name} in directory {file.parent}, right click the file in the file browser to download it')\n",
+    "\n",
+    "widget2.on_click(generate_tarfile)\n",
+    "\n",
+    "display(widget1,widget2,HTML(\"\"\"<style>\n",
+    "    .widget-label { width: unset !important; }\n",
+    "</style>\"\"\"))"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

ipsframework/_jupyter/initializer.py

@@ -49,9 +49,14 @@ def _initial_data_file_code() -> str:
 
 
 def initialize_jupyter_python_api(jupyterhub_dir: str):
-    api_filepath = Path(__file__).parent / f'api_{CURRENT_API_VERSION}.py'
-    file_dest = Path(jupyterhub_dir) / f'api_{CURRENT_API_VERSION}.py'
-    shutil.copyfile(api_filepath, file_dest)
+    """Set up the multirun API files."""
+    source_dir = Path(__file__).parent
+    dest_dir = Path(jupyterhub_dir)
+    for fname in (f'api_{CURRENT_API_VERSION}.py', f'api_{CURRENT_API_VERSION}_notebook.ipynb'):
+        shutil.copyfile(
+            source_dir / fname,
+            dest_dir / fname,
+        )
 
 
 def initialize_jupyter_notebook(notebook_dest: str, notebook_src: str):

ipsframework/services.py

@@ -1870,6 +1870,13 @@ def _init_jupyter(self) -> bool:
             return False
 
         self._jupyterhub_dir = root_dir
+
+        # adds module file to Jupyterhub
+        initialize_jupyter_import_module_file(self._jupyterhub_dir)
+
+        # add the shared python API if it doesn't exist
+        initialize_jupyter_python_api(str(pathlib.Path(self._jupyterhub_dir).parent))
+
         return True
 
     def get_staged_jupyterhub_files(self) -> List[str]:
@@ -1904,7 +1911,6 @@ def _get_jupyterhub_url(self) -> Optional[str]:
         url += f'ipsframework/runs/{portal_url_host}/{runid}/'
         return url
 
-    # TODO consider how we use variable_name in the API and get rid of it if it's not necessary
     def initialize_jupyter_notebook(
         self,
         dest_notebook_name: str,
@@ -1917,18 +1923,11 @@ def initialize_jupyter_notebook(
         Params:
           - dest_notebook_name: name of the JupyterNotebook you want to write (do not include file paths).
           - source_notebook_path: location you want to load the source notebook from
-          - data_module_name: name of the python file you want to generate a data file for (do not include file paths or file extensions)
         """
         if not self._jupyterhub_dir:
             if not self._init_jupyter():
                 raise Exception('Unable to initialize base JupyterHub dir')
 
-        # adds module file to Jupyterhub
-        initialize_jupyter_import_module_file(self._jupyterhub_dir)
-
-        # add the shared python API if it doesn't exist
-        initialize_jupyter_python_api(str(pathlib.Path(self._jupyterhub_dir).parent))
-
         # adds notebook to JupyterHub
         initialize_jupyter_notebook(f'{self._jupyterhub_dir}{dest_notebook_name}', source_notebook_path)