allow for portal assoc. jupyter urls in send_portal_data

Signed-off-by: Lance-Drane <[email protected]>

examples-proposed/004-time-loop/mymodule/components.py

@@ -31,11 +31,11 @@ def step(self, timestamp=0.0):
 
         # create notebook here
         NOTEBOOK_NAME = 'full_state.ipynb'
-        jupyter_files = self.services.get_staged_jupyterhub_files()
-        self.services.create_jupyterhub_notebook(jupyter_files, NOTEBOOK_NAME)
+        jupyter_state_files = self.services.get_staged_jupyterhub_files()
+        self.services.create_jupyterhub_notebook(jupyter_state_files, NOTEBOOK_NAME)
         # NOTE: depending on the names of the files, you may have to use a custom mapping function to get the tag
         # You MUST store the tag somewhere in the file name
-        tags = jupyter_files
+        tags = jupyter_state_files
         self.services.portal_register_jupyter_notebook(NOTEBOOK_NAME, tags)
 
         self.services.call(worker, 'finalize', 0)
@@ -90,7 +90,15 @@ def step(self, timestamp=0.0, **keywords):
             data = f.read()
 
         # example of updating Jupyter state
-        _jupyterhub_state_file = self.services.jupyterhub_make_state(state_file, timestamp)
-        # if you wanted to create a notebook per timestep, call send_portal_data with _jupyterhub_state_file as the argument.
-        print('SEND PORTAL DATA', timestamp, data, file=stderr)
-        self.services.send_portal_data(timestamp, data)
+        jupyterhub_state_file = self.services.jupyterhub_make_state(state_file, timestamp)
+        notebook_param = None
+        # create two notebooks on certain timestamps, create no notebooks otherwise
+        if int(timestamp) % 10 == 0:
+            notebook_param = []
+            for ident in (1, 2):
+                notebook_name = f'state_{timestamp}_{ident}.ipynb'
+                self.services.create_jupyterhub_notebook([jupyterhub_state_file], notebook_name)
+                notebook_param.append(notebook_name)
+
+        print('SEND PORTAL DATA', timestamp, data, notebook_param, file=stderr)
+        self.services.send_portal_data(timestamp, data, notebook_param)

examples-proposed/004-time-loop/sim.conf

@@ -18,11 +18,14 @@ PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org
 # The BASE DIRECTORY of your machine's JupyterHub web server directory. This is used strictly for moving files around on the machine itself.
 # This MUST be an absolute directory
 # if executing portal on a web server also running JupyterHub, you can set this value to manually add your file to this location.
-# NOTE: "/ipsframework/runs/${RUNID}/${TIMESTEP}.ipynb" will automatically be prepended to the file, guaranteeing that files will NEVER be overwritten.
+# NOTE: "/ipsframework/runs/${RUNID}/" will automatically be prepended to the file, guaranteeing that files will NEVER be overwritten.
 JUPYTERHUB_DIR = ${PSCRATCH}
 # OPTIONAL
 # if PORTAL_URL is defined + JUPYTERHUB_DIR is defined + this value is defined, the IPSFramework can send a direct link to the JupyterHub file.
 # In the framework, we don't make any assumptions about how JupyterHub's base URL or root server path is configured; you'll generally want to provide the full URL up to the JupyterHub base directory.
+# the URL will always end with /ipsframework/runs/${RUNID}/${JUPYTER_FILE}.ipynb , but:
+#    - /lab/tree may not necessarily be the URL path that JupyterHub users to expose the tree
+#    - the path of the URL after /lab/tree MAY deviate from the value in JUPYTERHUB_DIR
 JUPYTERHUB_URL = https://jupyter.nersc.gov/user/${USER}/perlmutter-login-node-base/lab/tree${PSCRATCH}
 
 STATE_FILES = state.json

ipsframework/portalBridge.py

@@ -74,15 +74,19 @@ def send_post_data(conn: Connection, stop: EventType, url: str):
             next_val: dict[str, Any] = conn.recv()
             # TODO - consider using multipart/form-data instead
             try:
+                headers = {
+                    'Content-Type': 'application/octet-stream',
+                    'X-IPS-Tag': next_val['tag'],
+                    'X-IPS-Portal-Runid': next_val['portal_runid'],
+                }
+                links = next_val.get('jupyter_links')
+                if links:
+                    headers['X-IPS-Jupyter-Links'] = '\x01'.join(links)
                 resp = http.request(
                     'POST',
                     url,
                     body=next_val['data'],
-                    headers={
-                        'Content-Type': 'application/octet-stream',
-                        'X-IPS-Tag': next_val['tag'],
-                        'X-IPS-Portal-Runid': next_val['portal_runid'],
-                    },
+                    headers=headers,
                 )
             except urllib3.exceptions.MaxRetryError as e:
                 fail_count += 1

ipsframework/services.py

@@ -1763,13 +1763,15 @@ def update_time_stamp(self, new_time_stamp=-1):
 
     # instead of explicit content_type_enum - parse file? Focus just on E2E for now
     # TODO - change API to send a file path instead of raw data
-    def send_portal_data(self, tag: float, data: bytes):
+    def send_portal_data(self, tag: float, data: bytes, juypter_notebooks: Optional[List[str]] = None):
         """
         Send data to the portal
 
         Params:
           - tag: currently, use the timestep for this
           - data: raw data of statefile - must be in bytes format
+          - jupyter_notebooks: optional list of Jupyter notebooks.
+              If provided, associate these urls with this run
         """
         if not isinstance(data, bytes):
             self.error('Data argument passed to "services.send_portal_data" must be bytes')
@@ -1782,6 +1784,10 @@ def send_portal_data(self, tag: float, data: bytes):
         portal_data: dict[str, Any] = {}
         portal_data['tag'] = str(tag)
         portal_data['data'] = data
+        if juypter_notebooks:
+            url = self._get_jupyterhub_url()
+            if url:
+                portal_data['jupyter_links'] = [f'{url}{nb}' for nb in juypter_notebooks]
         portal_data['eventtype'] = 'PORTAL_DATA'
         event_data['portal_data'] = portal_data
         self.publish('_IPS_MONITOR', 'PORTAL_DATA', event_data)
@@ -1869,7 +1875,7 @@ def jupyterhub_make_state(self, state_file_path: str, timestamp: float) -> str:
         shutil.copyfile(state_file_path, new_state_file_path)
         return new_state_file_path
 
-    def _get_jupyterhub_url(self, notebook_name: str) -> Optional[str]:
+    def _get_jupyterhub_url(self) -> Optional[str]:
         url: str = self.get_config_param('JUPYTERHUB_URL')
         if not url:
             self.warning('JUPYTERHUB_URL was not defined in config file, skipping notebook association on portal')
@@ -1922,7 +1928,7 @@ def portal_register_jupyter_notebook(self, notebook_name: str, tags: List[str])
           - notebook_name: name of the notebook (do not provide any directories, use the config file for this)
           - tags: list of tags to associate the notebook with
         """
-        url = self._get_jupyterhub_url(notebook_name)
+        url = self._get_jupyterhub_url()
         if not url:
             return
         url += notebook_name