diff --git a/examples-proposed/004-time-loop/.gitignore b/examples-proposed/004-jupyter-append/.gitignore similarity index 100% rename from examples-proposed/004-time-loop/.gitignore rename to examples-proposed/004-jupyter-append/.gitignore diff --git a/examples-proposed/004-time-loop/README.md b/examples-proposed/004-jupyter-append/README.md similarity index 100% rename from examples-proposed/004-time-loop/README.md rename to examples-proposed/004-jupyter-append/README.md diff --git a/examples-proposed/004-time-loop/__init__.py b/examples-proposed/004-jupyter-append/__init__.py similarity index 100% rename from examples-proposed/004-time-loop/__init__.py rename to examples-proposed/004-jupyter-append/__init__.py diff --git a/examples-proposed/004-jupyter-append/component_driver.py b/examples-proposed/004-jupyter-append/component_driver.py new file mode 100644 index 00000000..de9ce06f --- /dev/null +++ b/examples-proposed/004-jupyter-append/component_driver.py @@ -0,0 +1,34 @@ +import os +import time +from sys import stderr + +from ipsframework import Component + +DELAY = bool(os.environ.get('EXAMPLE_DELAY')) + +class Driver(Component): + """In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep.""" + + # TODO put delay inside driver instead of another component + + def init(self, timestamp=0.0): + self.worker = self.services.get_port('WORKER') + self.monitor = self.services.get_port('MONITOR') + + self.services.call(self.worker, 'init', 0) + self.services.call(self.monitor, 'init', 0) + + def step(self, timestamp=0.0): + # The time loop is configured in its own section of sim.conf + # It is shared across all components + for t in self.services.get_time_loop(): + self.services.update_time_stamp(t) + self.services.call(self.worker, 'step', t) + if DELAY: + print('simulating fake delay for 10 seconds', file=stderr) + time.sleep(10.0) + self.services.call(self.monitor, 'step', t) + + def finalize(self, timestamp=0.0): + self.services.call(self.worker, 'finalize', 0) + self.services.call(self.monitor, 'finalize', 0) \ No newline at end of file diff --git a/examples-proposed/004-jupyter-append/component_monitor.py b/examples-proposed/004-jupyter-append/component_monitor.py new file mode 100644 index 00000000..131428f2 --- /dev/null +++ b/examples-proposed/004-jupyter-append/component_monitor.py @@ -0,0 +1,64 @@ +import json +import os +from sys import stderr + +from ipsframework import Component + +NOTEBOOK_1_TEMPLATE = 'basic.ipynb' + + +class Monitor(Component): + """ + The monitor is able to read state files and will separately post data. + """ + + def init(self, timestamp=0.0): + self.services.stage_input_files([NOTEBOOK_1_TEMPLATE]) + + # Example of initializing two separate notebooks + # Both notebooks should be initialized before the time loop and appended to inside the time loop + self.services.initialize_jupyter_notebook( + dest_notebook_name='basic.ipynb', # path is relative to JupyterHub directory + source_notebook_path='basic.ipynb', # path is relative to input directory + ) + + def step(self, timestamp=0.0, **keywords): + msg = f'Running Monitor step with timestamp={timestamp}' + print(msg, file=stderr) + self.services.send_portal_event(event_comment=msg) + + self.services.stage_state() + + state_file = self.services.get_config_param('STATE_FILES') + + # generate any analysis files from the state file you want + with open(state_file) as f: + analysis = json.load(f) + + # Do any preprocessing needed prior to adding any file to Jupyter + # NOTE: you must make sure every analysis file name is unique in the "append" workflow + + # with the first analysis file, we just dump the results unmodified + analysis_file_1 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis.json') + with open(analysis_file_1, 'w') as f: + json.dump(analysis, f) + + # with the second analysis file, we'll just do a mock "analysis" where we just divide the values by two + mapped_analysis = {key: value / 2 for key, value in analysis.items()} + analysis_file_2 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis_mapped.json') + with open(analysis_file_2, 'w') as f: + json.dump(mapped_analysis, f) + + raw_data = json.dumps(analysis).encode() + + print('add analysis data files') + self.services.add_analysis_data_files( + [analysis_file_1, analysis_file_2], + timestamp=timestamp, + ) + + print('SEND PORTAL DATA', timestamp, raw_data, file=stderr) + self.services.send_portal_data(timestamp, raw_data) + + def finalize(self, timestamp=0.0): + ... diff --git a/examples-proposed/004-jupyter-append/component_worker.py b/examples-proposed/004-jupyter-append/component_worker.py new file mode 100644 index 00000000..2eb1f0bd --- /dev/null +++ b/examples-proposed/004-jupyter-append/component_worker.py @@ -0,0 +1,35 @@ +import json +import math +import random +from sys import stderr + +from ipsframework import Component + + +class Worker(Component): + """ + The worker component performs computations and updates state files. + """ + + def init(self, timestamp=0.0): + self.start = random.random() * math.pi * 2 + + def step(self, timestamp=0.0): + msg = f'Running Worker step with timestamp={timestamp}' + print(msg, file=stderr) + self.services.send_portal_event(event_comment=msg) + + data = { + 'y1': math.sin(self.start + timestamp / 50 * math.pi), + 'y2': math.sin(self.start + timestamp / 50 * math.pi) ** 2, + 'y3': math.sin(self.start + timestamp / 50 * math.pi) ** 3, + } + + # TODO maybe assume that it's just one? + state_file = self.services.get_config_param('STATE_FILES') + with open(state_file, 'w') as f: + json.dump(data, f) + self.services.update_state() + + def finalize(self, timestamp=0.0): + ... diff --git a/examples-proposed/004-time-loop/platform.conf b/examples-proposed/004-jupyter-append/platform.conf similarity index 100% rename from examples-proposed/004-time-loop/platform.conf rename to examples-proposed/004-jupyter-append/platform.conf diff --git a/examples-proposed/004-time-loop/run-delayed.sh b/examples-proposed/004-jupyter-append/run-delayed.sh similarity index 100% rename from examples-proposed/004-time-loop/run-delayed.sh rename to examples-proposed/004-jupyter-append/run-delayed.sh diff --git a/examples-proposed/004-time-loop/run.sh b/examples-proposed/004-jupyter-append/run.sh similarity index 100% rename from examples-proposed/004-time-loop/run.sh rename to examples-proposed/004-jupyter-append/run.sh diff --git a/examples-proposed/004-time-loop/setup.py b/examples-proposed/004-jupyter-append/setup.py similarity index 100% rename from examples-proposed/004-time-loop/setup.py rename to examples-proposed/004-jupyter-append/setup.py diff --git a/examples-proposed/004-time-loop/sim.conf b/examples-proposed/004-jupyter-append/sim.conf similarity index 85% rename from examples-proposed/004-time-loop/sim.conf rename to examples-proposed/004-jupyter-append/sim.conf index e7785ea3..976cb152 100644 --- a/examples-proposed/004-time-loop/sim.conf +++ b/examples-proposed/004-jupyter-append/sim.conf @@ -13,7 +13,6 @@ INPUT_DIR = $SIM_ROOT/input_dir/ USER_W3_DIR = $PWD/www USER_W3_BASEURL = -#PORTAL_URL = http://localhost:5000 PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org # OPTIONAL @@ -38,10 +37,7 @@ STATE_FILES = state.json STATE_WORK_DIR = $SIM_ROOT/state [PORTS] - NAMES = INIT DRIVER WORKER MONITOR - - [[INIT]] - IMPLEMENTATION = init + NAMES = DRIVER WORKER MONITOR [[DRIVER]] IMPLEMENTATION = driver @@ -52,17 +48,6 @@ STATE_WORK_DIR = $SIM_ROOT/state [[MONITOR]] IMPLEMENTATION = monitor -[init] - CLASS = INIT - SUB_CLASS = - NAME = Init - NPROC = 1 - BIN_PATH = - OUTPUT_FILES = - INPUT_FILES = - SCRIPT = - MODULE = mymodule.components - [driver] CLASS = DRIVER SUB_CLASS = @@ -71,8 +56,7 @@ STATE_WORK_DIR = $SIM_ROOT/state BIN_PATH = OUTPUT_FILES = INPUT_FILES = - SCRIPT = - MODULE = mymodule.components + SCRIPT = $PWD/component_driver.py [worker] CLASS = WORKER @@ -82,8 +66,7 @@ STATE_WORK_DIR = $SIM_ROOT/state BIN_PATH = OUTPUT_FILES = INPUT_FILES = - SCRIPT = - MODULE = mymodule.components + SCRIPT = $PWD/component_worker.py [monitor] CLASS = WORKER @@ -93,8 +76,7 @@ STATE_WORK_DIR = $SIM_ROOT/state BIN_PATH = OUTPUT_FILES = INPUT_FILES = - SCRIPT = - MODULE = mymodule.components + SCRIPT = $PWD/component_monitor.py [TIME_LOOP] MODE = REGULAR diff --git a/examples-proposed/004-time-loop/sim/input_dir/basic.ipynb b/examples-proposed/004-jupyter-append/sim/input_dir/basic.ipynb similarity index 100% rename from examples-proposed/004-time-loop/sim/input_dir/basic.ipynb rename to examples-proposed/004-jupyter-append/sim/input_dir/basic.ipynb diff --git a/examples-proposed/004-time-loop/mymodule/components.py b/examples-proposed/004-time-loop/mymodule/components.py deleted file mode 100644 index 468e1be4..00000000 --- a/examples-proposed/004-time-loop/mymodule/components.py +++ /dev/null @@ -1,120 +0,0 @@ -import json -import math -import os -import random -import time -from sys import stderr - -from ipsframework import Component - -DELAY = bool(os.environ.get('EXAMPLE_DELAY')) -REPLACE = bool(os.environ.get('EXAMPLE_REPLACE')) - -# templates are existing files from the input directory -# names are what the notebook and the associated data file will be labeled with (you can leave off the .ipynb / .py) -NOTEBOOK_1_TEMPLATE = 'basic.ipynb' -NOTEBOOK_1_NAME = 'basic.ipynb' -NOTEBOOK_2_TEMPLATE = 'bokeh-plots.ipynb' -NOTEBOOK_2_NAME = 'bokeh-plots.ipynb' - - -class Init(Component): - """Empty init component.""" - - pass - - -class Driver(Component): - """In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep.""" - - def step(self, timestamp=0.0): - worker = self.services.get_port('WORKER') - monitor = self.services.get_port('MONITOR') - - self.services.call(worker, 'init', 0) - # Needed for notebook template - self.services.stage_input_files([NOTEBOOK_1_TEMPLATE, NOTEBOOK_2_TEMPLATE]) - - # Example of initializing two separate notebooks - # Both notebooks should be initialized before the time loop and appended to inside the time loop - self.services.initialize_jupyter_notebook( - dest_notebook_name=NOTEBOOK_1_NAME, # path is relative to JupyterHub directory - source_notebook_path=NOTEBOOK_1_TEMPLATE, # path is relative to input directory - ) - self.services.initialize_jupyter_notebook( - dest_notebook_name=NOTEBOOK_2_NAME, # path is relative to JupyterHub directory - source_notebook_path=NOTEBOOK_2_TEMPLATE, # path is relative to input directory - ) - - # The time loop is configured in its own section of sim.conf - # It is shared across all components - for t in self.services.get_time_loop(): - self.services.update_time_stamp(t) - self.services.call(worker, 'step', t) - # TODO - perhaps monitor timestep does not need to be called every step, but only every 20 steps? - self.services.call(monitor, 'step', t) - - self.services.call(worker, 'finalize', 0) - - -class Worker(Component): - """ - The worker component performs computations and updates state files. - """ - - def init(self, timestamp=0.0): - self.start = random.random() * math.pi * 2 - - def step(self, timestamp=0.0): - msg = f'Running Worker step with timestamp={timestamp}' - print(msg, file=stderr) - self.services.send_portal_event(event_comment=msg) - - data = { - 'y1': math.sin(self.start + timestamp / 50 * math.pi), - 'y2': math.sin(self.start + timestamp / 50 * math.pi) ** 2, - 'y3': math.sin(self.start + timestamp / 50 * math.pi) ** 3, - } - - state_file = self.services.get_config_param('STATE_FILES') - with open(state_file, 'w') as f: - json.dump(data, f) - self.services.update_state() - - # copy the state file to a unique path for the monitor - data_loc = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp if not REPLACE else 0.0}_{state_file}') - with open(data_loc, 'w') as f: - json.dump(data, f) - - -class Monitor(Component): - """ - The monitor is able to read state files and will separately post data. - """ - - def step(self, timestamp=0.0, **keywords): - msg = f'Running Monitor step with timestamp={timestamp}' - print(msg, file=stderr) - if DELAY: - print('simulating fake delay for 10 seconds', file=stderr) - time.sleep(10.0) - self.services.send_portal_event(event_comment=msg) - - self.services.stage_state() - - state_file = self.services.get_config_param('STATE_FILES') - data_loc = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp if not REPLACE else 0.0}_{state_file}') - with open(data_loc, 'rb') as f: - data = f.read() - - # stage the state file in the JupyterHub directory and update the module file to handle it - if REPLACE: - self.services.add_analysis_data_files([data_loc], replace=True) - else: - self.services.add_analysis_data_files( - [data_loc], - timestamp=timestamp, - ) - - print('SEND PORTAL DATA', timestamp, data, file=stderr) - self.services.send_portal_data(timestamp, data) diff --git a/examples-proposed/005-jupyter-replace/.gitignore b/examples-proposed/005-jupyter-replace/.gitignore new file mode 100644 index 00000000..fea47dab --- /dev/null +++ b/examples-proposed/005-jupyter-replace/.gitignore @@ -0,0 +1,2 @@ +sim/* +!sim/input_dir/ diff --git a/examples-proposed/005-jupyter-replace/README.md b/examples-proposed/005-jupyter-replace/README.md new file mode 100644 index 00000000..142d57b6 --- /dev/null +++ b/examples-proposed/005-jupyter-replace/README.md @@ -0,0 +1,31 @@ +# Task pool synchronous + +This is an example which utilizes the time loop. The config file specifies that the script will execute all timestamps in the range of 1.0 to 100.0 (both inclusive), incrementing by 1.0 for each cycle. + +For each timestep, the worker component will update the state file with random JSON data. Note that with this implementation, the state file is overridden on each new timestep (the final timestep called will persist after the application). + +## Instructions + +Note that this example uses the module syntax, as opposed to the script syntax. + +To install, you can run: + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +To run the code, run: + +```bash +./run.sh +``` + +By default, this example will always _append_ a state file. If you prefer to see an example of how to _replace_ a state file, run: + +```bash +EXAMPLE_REPLACE=1 ./run.sh +``` + +There is also a script `run-delayed.sh` which you can use instead of `run.sh` if you would like to simulate a delay between monitor steps. diff --git a/examples-proposed/004-time-loop/mymodule/__init__.py b/examples-proposed/005-jupyter-replace/__init__.py similarity index 100% rename from examples-proposed/004-time-loop/mymodule/__init__.py rename to examples-proposed/005-jupyter-replace/__init__.py diff --git a/examples-proposed/005-jupyter-replace/component_driver.py b/examples-proposed/005-jupyter-replace/component_driver.py new file mode 100644 index 00000000..de9ce06f --- /dev/null +++ b/examples-proposed/005-jupyter-replace/component_driver.py @@ -0,0 +1,34 @@ +import os +import time +from sys import stderr + +from ipsframework import Component + +DELAY = bool(os.environ.get('EXAMPLE_DELAY')) + +class Driver(Component): + """In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep.""" + + # TODO put delay inside driver instead of another component + + def init(self, timestamp=0.0): + self.worker = self.services.get_port('WORKER') + self.monitor = self.services.get_port('MONITOR') + + self.services.call(self.worker, 'init', 0) + self.services.call(self.monitor, 'init', 0) + + def step(self, timestamp=0.0): + # The time loop is configured in its own section of sim.conf + # It is shared across all components + for t in self.services.get_time_loop(): + self.services.update_time_stamp(t) + self.services.call(self.worker, 'step', t) + if DELAY: + print('simulating fake delay for 10 seconds', file=stderr) + time.sleep(10.0) + self.services.call(self.monitor, 'step', t) + + def finalize(self, timestamp=0.0): + self.services.call(self.worker, 'finalize', 0) + self.services.call(self.monitor, 'finalize', 0) \ No newline at end of file diff --git a/examples-proposed/005-jupyter-replace/component_monitor.py b/examples-proposed/005-jupyter-replace/component_monitor.py new file mode 100644 index 00000000..a6bb93e8 --- /dev/null +++ b/examples-proposed/005-jupyter-replace/component_monitor.py @@ -0,0 +1,52 @@ +import json +import os +from sys import stderr + +from ipsframework import Component + +# templates are existing files from the input directory +# names are what the notebook and the associated data file will be labeled with (you can leave off the .ipynb / .py) +NOTEBOOK_1_TEMPLATE = 'basic.ipynb' + + +# TODO - use two different examples instead of REPLACE loop +class Monitor(Component): + """ + The monitor is able to read state files and will separately post data. + """ + + def init(self, timestamp=0.0): + self.services.stage_input_files([NOTEBOOK_1_TEMPLATE]) + + # Example of initializing two separate notebooks + # Both notebooks should be initialized before the time loop and appended to inside the time loop + self.services.initialize_jupyter_notebook( + dest_notebook_name='basic.ipynb', # path is relative to JupyterHub directory + source_notebook_path='basic.ipynb', # path is relative to input directory + ) + + def step(self, timestamp=0.0, **keywords): + msg = f'Running Monitor step with timestamp={timestamp}' + print(msg, file=stderr) + self.services.send_portal_event(event_comment=msg) + + self.services.stage_state() + + state_file = self.services.get_config_param('STATE_FILES') + + # generate any analysis files from the state file you want + with open(state_file) as f: + analysis = json.load(f) + + analysis_file_1 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis.json') + with open(analysis_file_1, 'w') as f: + json.dump(analysis, f) + + self.services.add_analysis_data_files([analysis_file_1], replace=True) + + raw_data = json.dumps(analysis).encode() + print('SEND PORTAL DATA', timestamp, raw_data, file=stderr) + self.services.send_portal_data(timestamp, raw_data) + + def finalize(self, timestamp=0.0): + ... diff --git a/examples-proposed/005-jupyter-replace/component_worker.py b/examples-proposed/005-jupyter-replace/component_worker.py new file mode 100644 index 00000000..2eb1f0bd --- /dev/null +++ b/examples-proposed/005-jupyter-replace/component_worker.py @@ -0,0 +1,35 @@ +import json +import math +import random +from sys import stderr + +from ipsframework import Component + + +class Worker(Component): + """ + The worker component performs computations and updates state files. + """ + + def init(self, timestamp=0.0): + self.start = random.random() * math.pi * 2 + + def step(self, timestamp=0.0): + msg = f'Running Worker step with timestamp={timestamp}' + print(msg, file=stderr) + self.services.send_portal_event(event_comment=msg) + + data = { + 'y1': math.sin(self.start + timestamp / 50 * math.pi), + 'y2': math.sin(self.start + timestamp / 50 * math.pi) ** 2, + 'y3': math.sin(self.start + timestamp / 50 * math.pi) ** 3, + } + + # TODO maybe assume that it's just one? + state_file = self.services.get_config_param('STATE_FILES') + with open(state_file, 'w') as f: + json.dump(data, f) + self.services.update_state() + + def finalize(self, timestamp=0.0): + ... diff --git a/examples-proposed/005-jupyter-replace/platform.conf b/examples-proposed/005-jupyter-replace/platform.conf new file mode 100644 index 00000000..4051494c --- /dev/null +++ b/examples-proposed/005-jupyter-replace/platform.conf @@ -0,0 +1,9 @@ +HOST = +MPIRUN = mpirun +NODE_DETECTION = manual +TOTAL_PROCS = 8 +NODES = 1 +PROCS_PER_NODE = 8 +CORES_PER_NODE = 8 +SOCKETS_PER_NODE = 1 +NODE_ALLOCATION_MODE = SHARED diff --git a/examples-proposed/005-jupyter-replace/run-delayed.sh b/examples-proposed/005-jupyter-replace/run-delayed.sh new file mode 100755 index 00000000..85516620 --- /dev/null +++ b/examples-proposed/005-jupyter-replace/run-delayed.sh @@ -0,0 +1,2 @@ +#!/bin/sh +PYTHONPATH=$(dirname "$0") PSCRATCH=${PSCRATCH:-/tmp} EXAMPLE_DELAY=true ips.py --config=sim.conf --platform=platform.conf --log=ips.log #--debug --verbose diff --git a/examples-proposed/005-jupyter-replace/run.sh b/examples-proposed/005-jupyter-replace/run.sh new file mode 100755 index 00000000..4de3e069 --- /dev/null +++ b/examples-proposed/005-jupyter-replace/run.sh @@ -0,0 +1,2 @@ +#!/bin/sh +PYTHONPATH=$(dirname "$0") PSCRATCH=${PSCRATCH:-/tmp} ips.py --config=sim.conf --platform=platform.conf --log=ips.log #--debug --verbose diff --git a/examples-proposed/005-jupyter-replace/setup.py b/examples-proposed/005-jupyter-replace/setup.py new file mode 100644 index 00000000..7a2abd1f --- /dev/null +++ b/examples-proposed/005-jupyter-replace/setup.py @@ -0,0 +1,8 @@ +from setuptools import find_packages, setup + +setup( + name='mymodule', + version='0.0.1', + install_requires=['ipsframework'], + packages=find_packages(), +) diff --git a/examples-proposed/005-jupyter-replace/sim.conf b/examples-proposed/005-jupyter-replace/sim.conf new file mode 100644 index 00000000..976cb152 --- /dev/null +++ b/examples-proposed/005-jupyter-replace/sim.conf @@ -0,0 +1,85 @@ +SIM_ROOT = $PWD/sim +SIM_NAME = sim +LOG_FILE = sim.log +LOG_LEVEL = DEBUG +SIMULATION_MODE = NORMAL +RUN_COMMENT = From PC +RUN_ID = sim run id +TOKAMAK_ID = tokamak +SHOT_NUMBER = 1 +TAG = tag +INPUT_DIR = $SIM_ROOT/input_dir/ + +USER_W3_DIR = $PWD/www +USER_W3_BASEURL = + +PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org + +# OPTIONAL +# The BASE DIRECTORY of your machine's JupyterHub web server directory. This is used strictly for moving files around on the machine itself. +# This MUST be an absolute directory +# if executing portal on a web server also running JupyterHub, you can set this value to manually add your file to this location. +# NOTE: "/ipsframework/runs/${RUNID}/" will automatically be prepended to the file, guaranteeing that files will NEVER be overwritten. +# NOTE FOR NERSC SPECIFICALLY: if you want to share notebooks with other users, you must make sure they have the correct file permissions in the directory. +JUPYTERHUB_DIR = ${PSCRATCH} +# OPTIONAL +# if PORTAL_URL is defined + JUPYTERHUB_DIR is defined + this value is defined, the IPSFramework can send a direct link to the JupyterHub file. +# In the framework, we don't make any assumptions about how JupyterHub's base URL or root server path is configured; you'll generally want to provide the full URL up to the JupyterHub base directory. +# the URL will always end with /ipsframework/runs/${RUNID}/${JUPYTER_FILE}.ipynb , but: +# - /lab/tree may not necessarily be the URL path that JupyterHub users to expose the tree +# - the path of the URL after /lab/tree MAY deviate from the value in JUPYTERHUB_DIR +# +# NOTES FOR NERSC SPECIFICALLY: +# - if you think you will need to execute notebooks on a different node, replace "perlmutter-login-node-base" with the type of node you want to run on. +JUPYTERHUB_URL = https://jupyter.nersc.gov/user/${USER}/perlmutter-login-node-base/lab/tree${PSCRATCH} + +STATE_FILES = state.json +STATE_WORK_DIR = $SIM_ROOT/state + +[PORTS] + NAMES = DRIVER WORKER MONITOR + + [[DRIVER]] + IMPLEMENTATION = driver + + [[WORKER]] + IMPLEMENTATION = worker + + [[MONITOR]] + IMPLEMENTATION = monitor + +[driver] + CLASS = DRIVER + SUB_CLASS = + NAME = Driver + NPROC = 1 + BIN_PATH = + OUTPUT_FILES = + INPUT_FILES = + SCRIPT = $PWD/component_driver.py + +[worker] + CLASS = WORKER + SUB_CLASS = + NAME = Worker + NPROC = 1 + BIN_PATH = + OUTPUT_FILES = + INPUT_FILES = + SCRIPT = $PWD/component_worker.py + +[monitor] + CLASS = WORKER + SUB_CLASS = + NAME = Monitor + NPROC = 1 + BIN_PATH = + OUTPUT_FILES = + INPUT_FILES = + SCRIPT = $PWD/component_monitor.py + +[TIME_LOOP] + MODE = REGULAR + START = 1 + FINISH = 30 + NSTEP = 30 diff --git a/examples-proposed/005-jupyter-replace/sim/input_dir/basic.ipynb b/examples-proposed/005-jupyter-replace/sim/input_dir/basic.ipynb new file mode 100644 index 00000000..7c50b5ba --- /dev/null +++ b/examples-proposed/005-jupyter-replace/sim/input_dir/basic.ipynb @@ -0,0 +1,30 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "5d75faa3", + "metadata": {}, + "outputs": [], + "source": [ + "# Notebook template, the IPS Framework will add a cell before this one\n", + "# defining DATA_FILES as a list of state file paths.\n", + "\n", + "# In this example, this notebook is generated during the time loop.\n", + "\n", + "mapping = {}\n", + "for file in DATA_FILES.values():\n", + " with open(file, 'rb') as f:\n", + " mapping[file] = f.read()\n", + "print(mapping)\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples-proposed/006-jupyter-multiple-notebooks/.gitignore b/examples-proposed/006-jupyter-multiple-notebooks/.gitignore new file mode 100644 index 00000000..fea47dab --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/.gitignore @@ -0,0 +1,2 @@ +sim/* +!sim/input_dir/ diff --git a/examples-proposed/006-jupyter-multiple-notebooks/README.md b/examples-proposed/006-jupyter-multiple-notebooks/README.md new file mode 100644 index 00000000..142d57b6 --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/README.md @@ -0,0 +1,31 @@ +# Task pool synchronous + +This is an example which utilizes the time loop. The config file specifies that the script will execute all timestamps in the range of 1.0 to 100.0 (both inclusive), incrementing by 1.0 for each cycle. + +For each timestep, the worker component will update the state file with random JSON data. Note that with this implementation, the state file is overridden on each new timestep (the final timestep called will persist after the application). + +## Instructions + +Note that this example uses the module syntax, as opposed to the script syntax. + +To install, you can run: + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +To run the code, run: + +```bash +./run.sh +``` + +By default, this example will always _append_ a state file. If you prefer to see an example of how to _replace_ a state file, run: + +```bash +EXAMPLE_REPLACE=1 ./run.sh +``` + +There is also a script `run-delayed.sh` which you can use instead of `run.sh` if you would like to simulate a delay between monitor steps. diff --git a/examples-proposed/006-jupyter-multiple-notebooks/__init__.py b/examples-proposed/006-jupyter-multiple-notebooks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples-proposed/006-jupyter-multiple-notebooks/component_driver.py b/examples-proposed/006-jupyter-multiple-notebooks/component_driver.py new file mode 100644 index 00000000..de9ce06f --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/component_driver.py @@ -0,0 +1,34 @@ +import os +import time +from sys import stderr + +from ipsframework import Component + +DELAY = bool(os.environ.get('EXAMPLE_DELAY')) + +class Driver(Component): + """In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep.""" + + # TODO put delay inside driver instead of another component + + def init(self, timestamp=0.0): + self.worker = self.services.get_port('WORKER') + self.monitor = self.services.get_port('MONITOR') + + self.services.call(self.worker, 'init', 0) + self.services.call(self.monitor, 'init', 0) + + def step(self, timestamp=0.0): + # The time loop is configured in its own section of sim.conf + # It is shared across all components + for t in self.services.get_time_loop(): + self.services.update_time_stamp(t) + self.services.call(self.worker, 'step', t) + if DELAY: + print('simulating fake delay for 10 seconds', file=stderr) + time.sleep(10.0) + self.services.call(self.monitor, 'step', t) + + def finalize(self, timestamp=0.0): + self.services.call(self.worker, 'finalize', 0) + self.services.call(self.monitor, 'finalize', 0) \ No newline at end of file diff --git a/examples-proposed/006-jupyter-multiple-notebooks/component_monitor.py b/examples-proposed/006-jupyter-multiple-notebooks/component_monitor.py new file mode 100644 index 00000000..c39e4a2b --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/component_monitor.py @@ -0,0 +1,64 @@ +import json +import os +from sys import stderr + +from ipsframework import Component + +# templates are existing files from the input directory +# names are what the notebook and the associated data file will be labeled with (you can leave off the .ipynb / .py) +NOTEBOOK_1_TEMPLATE = 'basic.ipynb' +NOTEBOOK_2_TEMPLATE = 'bokeh-plots.ipynb' + + +# TODO - use two different examples instead of REPLACE loop +class Monitor(Component): + """ + The monitor is able to read state files and will separately post data. + """ + + def init(self, timestamp=0.0): + self.services.stage_input_files([NOTEBOOK_1_TEMPLATE, NOTEBOOK_2_TEMPLATE]) + + # Example of initializing two separate notebooks + # Both notebooks should be initialized before the time loop and appended to inside the time loop + self.services.initialize_jupyter_notebook( + dest_notebook_name=NOTEBOOK_1_TEMPLATE, # path is relative to JupyterHub directory + source_notebook_path=NOTEBOOK_1_TEMPLATE, # path is relative to input directory + ) + self.services.initialize_jupyter_notebook( + dest_notebook_name=NOTEBOOK_2_TEMPLATE, # path is relative to JupyterHub directory + source_notebook_path=NOTEBOOK_2_TEMPLATE, # path is relative to input directory + ) + + def step(self, timestamp=0.0, **keywords): + msg = f'Running Monitor step with timestamp={timestamp}' + print(msg, file=stderr) + self.services.send_portal_event(event_comment=msg) + + self.services.stage_state() + + state_file = self.services.get_config_param('STATE_FILES') + + # Do some arbitrary processing and make sure analysis file is distinct from state file + # i.e. just a sampling of the data + # analysis file does NOT have to be a replica of the state file - we want the name to be unique + + # generate any analysis files from the state file you want + with open(state_file) as f: + analysis = json.load(f) + + analysis_file_1 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis.json') + with open(analysis_file_1, 'w') as f: + json.dump(analysis, f) + data = json.dumps(analysis).encode() + + self.services.add_analysis_data_files( + [analysis_file_1], + timestamp=timestamp, + ) + + print('SEND PORTAL DATA', timestamp, data, file=stderr) + self.services.send_portal_data(timestamp, data) + + def finalize(self, timestamp=0.0): + ... diff --git a/examples-proposed/006-jupyter-multiple-notebooks/component_worker.py b/examples-proposed/006-jupyter-multiple-notebooks/component_worker.py new file mode 100644 index 00000000..2eb1f0bd --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/component_worker.py @@ -0,0 +1,35 @@ +import json +import math +import random +from sys import stderr + +from ipsframework import Component + + +class Worker(Component): + """ + The worker component performs computations and updates state files. + """ + + def init(self, timestamp=0.0): + self.start = random.random() * math.pi * 2 + + def step(self, timestamp=0.0): + msg = f'Running Worker step with timestamp={timestamp}' + print(msg, file=stderr) + self.services.send_portal_event(event_comment=msg) + + data = { + 'y1': math.sin(self.start + timestamp / 50 * math.pi), + 'y2': math.sin(self.start + timestamp / 50 * math.pi) ** 2, + 'y3': math.sin(self.start + timestamp / 50 * math.pi) ** 3, + } + + # TODO maybe assume that it's just one? + state_file = self.services.get_config_param('STATE_FILES') + with open(state_file, 'w') as f: + json.dump(data, f) + self.services.update_state() + + def finalize(self, timestamp=0.0): + ... diff --git a/examples-proposed/006-jupyter-multiple-notebooks/platform.conf b/examples-proposed/006-jupyter-multiple-notebooks/platform.conf new file mode 100644 index 00000000..4051494c --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/platform.conf @@ -0,0 +1,9 @@ +HOST = +MPIRUN = mpirun +NODE_DETECTION = manual +TOTAL_PROCS = 8 +NODES = 1 +PROCS_PER_NODE = 8 +CORES_PER_NODE = 8 +SOCKETS_PER_NODE = 1 +NODE_ALLOCATION_MODE = SHARED diff --git a/examples-proposed/006-jupyter-multiple-notebooks/run-delayed.sh b/examples-proposed/006-jupyter-multiple-notebooks/run-delayed.sh new file mode 100755 index 00000000..85516620 --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/run-delayed.sh @@ -0,0 +1,2 @@ +#!/bin/sh +PYTHONPATH=$(dirname "$0") PSCRATCH=${PSCRATCH:-/tmp} EXAMPLE_DELAY=true ips.py --config=sim.conf --platform=platform.conf --log=ips.log #--debug --verbose diff --git a/examples-proposed/006-jupyter-multiple-notebooks/run.sh b/examples-proposed/006-jupyter-multiple-notebooks/run.sh new file mode 100755 index 00000000..4de3e069 --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/run.sh @@ -0,0 +1,2 @@ +#!/bin/sh +PYTHONPATH=$(dirname "$0") PSCRATCH=${PSCRATCH:-/tmp} ips.py --config=sim.conf --platform=platform.conf --log=ips.log #--debug --verbose diff --git a/examples-proposed/006-jupyter-multiple-notebooks/setup.py b/examples-proposed/006-jupyter-multiple-notebooks/setup.py new file mode 100644 index 00000000..7a2abd1f --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/setup.py @@ -0,0 +1,8 @@ +from setuptools import find_packages, setup + +setup( + name='mymodule', + version='0.0.1', + install_requires=['ipsframework'], + packages=find_packages(), +) diff --git a/examples-proposed/006-jupyter-multiple-notebooks/sim.conf b/examples-proposed/006-jupyter-multiple-notebooks/sim.conf new file mode 100644 index 00000000..976cb152 --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/sim.conf @@ -0,0 +1,85 @@ +SIM_ROOT = $PWD/sim +SIM_NAME = sim +LOG_FILE = sim.log +LOG_LEVEL = DEBUG +SIMULATION_MODE = NORMAL +RUN_COMMENT = From PC +RUN_ID = sim run id +TOKAMAK_ID = tokamak +SHOT_NUMBER = 1 +TAG = tag +INPUT_DIR = $SIM_ROOT/input_dir/ + +USER_W3_DIR = $PWD/www +USER_W3_BASEURL = + +PORTAL_URL = https://lb.ipsportal.development.svc.spin.nersc.org + +# OPTIONAL +# The BASE DIRECTORY of your machine's JupyterHub web server directory. This is used strictly for moving files around on the machine itself. +# This MUST be an absolute directory +# if executing portal on a web server also running JupyterHub, you can set this value to manually add your file to this location. +# NOTE: "/ipsframework/runs/${RUNID}/" will automatically be prepended to the file, guaranteeing that files will NEVER be overwritten. +# NOTE FOR NERSC SPECIFICALLY: if you want to share notebooks with other users, you must make sure they have the correct file permissions in the directory. +JUPYTERHUB_DIR = ${PSCRATCH} +# OPTIONAL +# if PORTAL_URL is defined + JUPYTERHUB_DIR is defined + this value is defined, the IPSFramework can send a direct link to the JupyterHub file. +# In the framework, we don't make any assumptions about how JupyterHub's base URL or root server path is configured; you'll generally want to provide the full URL up to the JupyterHub base directory. +# the URL will always end with /ipsframework/runs/${RUNID}/${JUPYTER_FILE}.ipynb , but: +# - /lab/tree may not necessarily be the URL path that JupyterHub users to expose the tree +# - the path of the URL after /lab/tree MAY deviate from the value in JUPYTERHUB_DIR +# +# NOTES FOR NERSC SPECIFICALLY: +# - if you think you will need to execute notebooks on a different node, replace "perlmutter-login-node-base" with the type of node you want to run on. +JUPYTERHUB_URL = https://jupyter.nersc.gov/user/${USER}/perlmutter-login-node-base/lab/tree${PSCRATCH} + +STATE_FILES = state.json +STATE_WORK_DIR = $SIM_ROOT/state + +[PORTS] + NAMES = DRIVER WORKER MONITOR + + [[DRIVER]] + IMPLEMENTATION = driver + + [[WORKER]] + IMPLEMENTATION = worker + + [[MONITOR]] + IMPLEMENTATION = monitor + +[driver] + CLASS = DRIVER + SUB_CLASS = + NAME = Driver + NPROC = 1 + BIN_PATH = + OUTPUT_FILES = + INPUT_FILES = + SCRIPT = $PWD/component_driver.py + +[worker] + CLASS = WORKER + SUB_CLASS = + NAME = Worker + NPROC = 1 + BIN_PATH = + OUTPUT_FILES = + INPUT_FILES = + SCRIPT = $PWD/component_worker.py + +[monitor] + CLASS = WORKER + SUB_CLASS = + NAME = Monitor + NPROC = 1 + BIN_PATH = + OUTPUT_FILES = + INPUT_FILES = + SCRIPT = $PWD/component_monitor.py + +[TIME_LOOP] + MODE = REGULAR + START = 1 + FINISH = 30 + NSTEP = 30 diff --git a/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/basic.ipynb b/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/basic.ipynb new file mode 100644 index 00000000..7c50b5ba --- /dev/null +++ b/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/basic.ipynb @@ -0,0 +1,30 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "5d75faa3", + "metadata": {}, + "outputs": [], + "source": [ + "# Notebook template, the IPS Framework will add a cell before this one\n", + "# defining DATA_FILES as a list of state file paths.\n", + "\n", + "# In this example, this notebook is generated during the time loop.\n", + "\n", + "mapping = {}\n", + "for file in DATA_FILES.values():\n", + " with open(file, 'rb') as f:\n", + " mapping[file] = f.read()\n", + "print(mapping)\n" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples-proposed/004-time-loop/sim/input_dir/bokeh-plots.ipynb b/examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/bokeh-plots.ipynb similarity index 100% rename from examples-proposed/004-time-loop/sim/input_dir/bokeh-plots.ipynb rename to examples-proposed/006-jupyter-multiple-notebooks/sim/input_dir/bokeh-plots.ipynb diff --git a/examples-proposed/005-task-pool-sync/README.md b/examples-proposed/008-task-pool-sync/README.md similarity index 100% rename from examples-proposed/005-task-pool-sync/README.md rename to examples-proposed/008-task-pool-sync/README.md diff --git a/examples-proposed/005-task-pool-sync/dask_sim.config b/examples-proposed/008-task-pool-sync/dask_sim.config similarity index 100% rename from examples-proposed/005-task-pool-sync/dask_sim.config rename to examples-proposed/008-task-pool-sync/dask_sim.config diff --git a/examples-proposed/005-task-pool-sync/dask_worker.py b/examples-proposed/008-task-pool-sync/dask_worker.py similarity index 100% rename from examples-proposed/005-task-pool-sync/dask_worker.py rename to examples-proposed/008-task-pool-sync/dask_worker.py diff --git a/examples-proposed/005-task-pool-sync/driver.py b/examples-proposed/008-task-pool-sync/driver.py similarity index 100% rename from examples-proposed/005-task-pool-sync/driver.py rename to examples-proposed/008-task-pool-sync/driver.py diff --git a/examples-proposed/005-task-pool-sync/myscript b/examples-proposed/008-task-pool-sync/myscript similarity index 100% rename from examples-proposed/005-task-pool-sync/myscript rename to examples-proposed/008-task-pool-sync/myscript diff --git a/examples-proposed/005-task-pool-sync/platform.conf b/examples-proposed/008-task-pool-sync/platform.conf similarity index 100% rename from examples-proposed/005-task-pool-sync/platform.conf rename to examples-proposed/008-task-pool-sync/platform.conf