rewrite jupyter examples

Signed-off-by: Lance-Drane <[email protected]>

examples-proposed/004-jupyter-append/component_driver.py

@@ -0,0 +1,34 @@
+import os
+import time
+from sys import stderr
+
+from ipsframework import Component
+
+DELAY = bool(os.environ.get('EXAMPLE_DELAY'))
+
+class Driver(Component):
+    """In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep."""
+
+    # TODO put delay inside driver instead of another component
+
+    def init(self, timestamp=0.0):
+        self.worker = self.services.get_port('WORKER')
+        self.monitor = self.services.get_port('MONITOR')
+
+        self.services.call(self.worker, 'init', 0)
+        self.services.call(self.monitor, 'init', 0)
+
+    def step(self, timestamp=0.0):
+        # The time loop is configured in its own section of sim.conf
+        # It is shared across all components
+        for t in self.services.get_time_loop():
+            self.services.update_time_stamp(t)
+            self.services.call(self.worker, 'step', t)
+            if DELAY:
+                print('simulating fake delay for 10 seconds', file=stderr)
+                time.sleep(10.0)
+            self.services.call(self.monitor, 'step', t)
+
+    def finalize(self, timestamp=0.0):
+        self.services.call(self.worker, 'finalize', 0)
+        self.services.call(self.monitor, 'finalize', 0)

examples-proposed/004-jupyter-append/component_monitor.py

@@ -0,0 +1,64 @@
+import json
+import os
+from sys import stderr
+
+from ipsframework import Component
+
+NOTEBOOK_1_TEMPLATE = 'basic.ipynb'
+
+
+class Monitor(Component):
+    """
+    The monitor is able to read state files and will separately post data.
+    """
+
+    def init(self, timestamp=0.0):
+        self.services.stage_input_files([NOTEBOOK_1_TEMPLATE])
+
+        # Example of initializing two separate notebooks
+        # Both notebooks should be initialized before the time loop and appended to inside the time loop
+        self.services.initialize_jupyter_notebook(
+            dest_notebook_name='basic.ipynb',  # path is relative to JupyterHub directory
+            source_notebook_path='basic.ipynb',  # path is relative to input directory
+        )
+
+    def step(self, timestamp=0.0, **keywords):
+        msg = f'Running Monitor step with timestamp={timestamp}'
+        print(msg, file=stderr)
+        self.services.send_portal_event(event_comment=msg)
+
+        self.services.stage_state()
+
+        state_file = self.services.get_config_param('STATE_FILES')
+
+        # generate any analysis files from the state file you want
+        with open(state_file) as f:
+            analysis = json.load(f)
+
+        # Do any preprocessing needed prior to adding any file to Jupyter
+        # NOTE: you must make sure every analysis file name is unique in the "append" workflow
+
+        # with the first analysis file, we just dump the results unmodified
+        analysis_file_1 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis.json')
+        with open(analysis_file_1, 'w') as f:
+            json.dump(analysis, f)
+
+        # with the second analysis file, we'll just do a mock "analysis" where we just divide the values by two
+        mapped_analysis = {key: value / 2 for key, value in analysis.items()}
+        analysis_file_2 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis_mapped.json')
+        with open(analysis_file_2, 'w') as f:
+            json.dump(mapped_analysis, f)
+
+        raw_data = json.dumps(analysis).encode()
+
+        print('add analysis data files')
+        self.services.add_analysis_data_files(
+            [analysis_file_1, analysis_file_2],
+            timestamp=timestamp,
+        )
+
+        print('SEND PORTAL DATA', timestamp, raw_data, file=stderr)
+        self.services.send_portal_data(timestamp, raw_data)
+
+    def finalize(self, timestamp=0.0):
+        ...

examples-proposed/004-jupyter-append/component_worker.py

@@ -0,0 +1,35 @@
+import json
+import math
+import random
+from sys import stderr
+
+from ipsframework import Component
+
+
+class Worker(Component):
+    """
+    The worker component performs computations and updates state files.
+    """
+
+    def init(self, timestamp=0.0):
+        self.start = random.random() * math.pi * 2
+
+    def step(self, timestamp=0.0):
+        msg = f'Running Worker step with timestamp={timestamp}'
+        print(msg, file=stderr)
+        self.services.send_portal_event(event_comment=msg)
+
+        data = {
+            'y1': math.sin(self.start + timestamp / 50 * math.pi),
+            'y2': math.sin(self.start + timestamp / 50 * math.pi) ** 2,
+            'y3': math.sin(self.start + timestamp / 50 * math.pi) ** 3,
+        }
+
+        # TODO maybe assume that it's just one?
+        state_file = self.services.get_config_param('STATE_FILES')
+        with open(state_file, 'w') as f:
+            json.dump(data, f)
+        self.services.update_state()
+
+    def finalize(self, timestamp=0.0):
+        ...

examples-proposed/004-time-loop/sim.conf → examples-proposed/004-jupyter-append/sim.conf

@@ -13,7 +13,6 @@ INPUT_DIR = $SIM_ROOT/input_dir/
 USER_W3_DIR = $PWD/www
 USER_W3_BASEURL =
 
-#PORTAL_URL = http://localhost:5000
 PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org
 
 # OPTIONAL
@@ -38,10 +37,7 @@ STATE_FILES = state.json
 STATE_WORK_DIR = $SIM_ROOT/state
 
 [PORTS]
-    NAMES = INIT DRIVER WORKER MONITOR
-
-    [[INIT]]
-        IMPLEMENTATION = init
+    NAMES = DRIVER WORKER MONITOR
 
     [[DRIVER]]
         IMPLEMENTATION = driver
@@ -52,17 +48,6 @@ STATE_WORK_DIR = $SIM_ROOT/state
     [[MONITOR]]
         IMPLEMENTATION = monitor
 
-[init]
-    CLASS = INIT
-    SUB_CLASS =
-    NAME = Init
-    NPROC = 1
-    BIN_PATH =
-    OUTPUT_FILES =
-    INPUT_FILES =
-    SCRIPT =
-    MODULE = mymodule.components
-
 [driver]
     CLASS = DRIVER
     SUB_CLASS =
@@ -71,8 +56,7 @@ STATE_WORK_DIR = $SIM_ROOT/state
     BIN_PATH =
     OUTPUT_FILES =
     INPUT_FILES =
-    SCRIPT =
-    MODULE = mymodule.components
+    SCRIPT = $PWD/component_driver.py
 
 [worker]
     CLASS = WORKER
@@ -82,8 +66,7 @@ STATE_WORK_DIR = $SIM_ROOT/state
     BIN_PATH =
     OUTPUT_FILES =
     INPUT_FILES =
-    SCRIPT =
-    MODULE = mymodule.components
+    SCRIPT = $PWD/component_worker.py
 
 [monitor]
     CLASS = WORKER
@@ -93,8 +76,7 @@ STATE_WORK_DIR = $SIM_ROOT/state
     BIN_PATH =
     OUTPUT_FILES =
     INPUT_FILES =
-    SCRIPT =
-    MODULE = mymodule.components
+    SCRIPT = $PWD/component_monitor.py
 
 [TIME_LOOP]
     MODE = REGULAR

examples-proposed/005-jupyter-replace/.gitignore

@@ -0,0 +1,2 @@
+sim/*
+!sim/input_dir/

examples-proposed/005-jupyter-replace/README.md

@@ -0,0 +1,31 @@
+# Task pool synchronous
+
+This is an example which utilizes the time loop. The config file specifies that the script will execute all timestamps in the range of 1.0 to 100.0 (both inclusive), incrementing by 1.0 for each cycle.
+
+For each timestep, the worker component will update the state file with random JSON data. Note that with this implementation, the state file is overridden on each new timestep (the final timestep called will persist after the application).
+
+## Instructions
+
+Note that this example uses the module syntax, as opposed to the script syntax.
+
+To install, you can run:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .
+```
+
+To run the code, run:
+
+```bash
+./run.sh
+```
+
+By default, this example will always _append_ a state file. If you prefer to see an example of how to _replace_ a state file, run:
+
+```bash
+EXAMPLE_REPLACE=1 ./run.sh
+```
+
+There is also a script `run-delayed.sh` which you can use instead of `run.sh` if you would like to simulate a delay between monitor steps.

examples-proposed/005-jupyter-replace/component_driver.py

@@ -0,0 +1,34 @@
+import os
+import time
+from sys import stderr
+
+from ipsframework import Component
+
+DELAY = bool(os.environ.get('EXAMPLE_DELAY'))
+
+class Driver(Component):
+    """In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep."""
+
+    # TODO put delay inside driver instead of another component
+
+    def init(self, timestamp=0.0):
+        self.worker = self.services.get_port('WORKER')
+        self.monitor = self.services.get_port('MONITOR')
+
+        self.services.call(self.worker, 'init', 0)
+        self.services.call(self.monitor, 'init', 0)
+
+    def step(self, timestamp=0.0):
+        # The time loop is configured in its own section of sim.conf
+        # It is shared across all components
+        for t in self.services.get_time_loop():
+            self.services.update_time_stamp(t)
+            self.services.call(self.worker, 'step', t)
+            if DELAY:
+                print('simulating fake delay for 10 seconds', file=stderr)
+                time.sleep(10.0)
+            self.services.call(self.monitor, 'step', t)
+
+    def finalize(self, timestamp=0.0):
+        self.services.call(self.worker, 'finalize', 0)
+        self.services.call(self.monitor, 'finalize', 0)