From 318388049ecebe566dc4c20a34ee29395119bf7d Mon Sep 17 00:00:00 2001
From: Laurie Stephey <lastephey@lbl.gov>
Date: Tue, 18 May 2021 10:26:36 -0700
Subject: [PATCH 1/2] cupy workaround, hackathon config

---
 delta/analysis/helpers.py         |  2 +-
 delta/configs/hackathon_test.json | 78 +++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+), 1 deletion(-)
 create mode 100644 delta/configs/hackathon_test.json

diff --git a/delta/analysis/helpers.py b/delta/analysis/helpers.py
index 3cb1bbb..9f64dbe 100644
--- a/delta/analysis/helpers.py
+++ b/delta/analysis/helpers.py
@@ -8,7 +8,7 @@
 from analysis.task_spectral_cy import task_coherence_cy, task_crosspower_cy, task_crossphase_cy
 try:
     from analysis.task_spectral_cu import task_coherence_cu, task_crosscorr_cu, task_crossphase_cu, task_crosspower_cu
-except ModuleNotFoundError:
+except:
     None
 
 from analysis.task_spectral_numba import task_spectral_GAP
diff --git a/delta/configs/hackathon_test.json b/delta/configs/hackathon_test.json
new file mode 100644
index 0000000..e5cc155
--- /dev/null
+++ b/delta/configs/hackathon_test.json
@@ -0,0 +1,78 @@
+{
+  "diagnostic":
+  {
+    "name": "kstarecei",
+    "shotnr": 25259,
+    "dev": "GT",
+    "datasource":
+    {
+      "source_file": "/global/cscratch1/sd/rkube/KSTAR/kstar_streaming/025259/ECEI.025259.GT.h5",
+      "chunk_size": 10000,
+      "num_chunks": 500,
+      "channel_range": ["0101-2408"],
+      "datatype": "float",
+      "t_norm": [-0.099, -0.089]
+    }
+  },
+  "transport_rx":
+  {
+    "engine": "BP4",
+    "params":
+    {
+      "IPAddress": "203.230.120.125",
+      "Timeout": "120",
+      "Port": "50001",
+      "TransportMode": "fast"
+    }
+  },
+  "transport_tx":
+  {
+    "engine": "BP4",
+    "params":
+    {
+      "IPAddress": "128.55.205.18",
+      "Timeout": "120",
+      "Port": "50055",
+      "TransportMode": "fast"
+    }
+  },
+  "storage":
+  {
+    "backend": "null"
+  },
+ "preprocess": {
+   "bandpass_fir": {"N": 5, "Wn": [0.14, 0.2], "btype": "bandpass", "output": "sos"},
+   "no_wavelet": {"wavelet": "db5", "method": "BayesShrink", "wavelet_levels": 5, "rescale_sigma": false},
+   "no_plot": {"time_range": [2.7175, 2.7178], "plot_dir": "/global/homes/r/rkube/delta_runs/plots/"},
+   "stft": {"nfft": 512, "fs": 500000, "window": "hann", "overlap": 0.5, "noverlap": 256, "detrend": "constant", "full": true}
+   },
+  "analysis_gpu": {
+    "spectral_GAP": {
+      "channel_chunk_size": 32768, 
+      "ref_channels": [1, 1, 24, 8], 
+      "cmp_channels": [1, 1, 24, 8]
+    }
+  },
+  "analysis": {
+    "crosscorr": {
+      "channel_chunk_size": 32768, 
+      "ref_channels": [1, 1, 24, 8], 
+      "cmp_channels": [1, 1, 24, 8]
+    },
+    "crosspower_cy": {
+      "channel_chunk_size": 32768, 
+      "ref_channels": [1, 1, 24, 8], 
+      "cmp_channels": [1, 1, 24, 8]
+    },
+    "crossphase_cy": {
+      "channel_chunk_size": 32768, 
+      "ref_channels": [1, 1, 24, 8], 
+      "cmp_channels": [1, 1, 24, 8]
+    },
+    "coherence_cy": {
+      "channel_chunk_size": 32768, 
+      "ref_channels": [1, 1, 24, 8], 
+      "cmp_channels": [1, 1, 24, 8]
+    }
+  }
+}

From 3f3a62f3b731fe619f2b815d4b51797918e2088e Mon Sep 17 00:00:00 2001
From: Laurie Stephey <lastephey@lbl.gov>
Date: Tue, 18 May 2021 11:37:43 -0700
Subject: [PATCH 2/2] dockerfile how to

---
 DockerfileREADME.md | 42 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/DockerfileREADME.md b/DockerfileREADME.md
index fe90e8f..8e68ea3 100644
--- a/DockerfileREADME.md
+++ b/DockerfileREADME.md
@@ -10,6 +10,29 @@ The image is called `registry.nersc.gov/das/delta:3.0`.
 
 As a backup, there is also an image that has delta installed inside: `registry.nersc.gov/das/delta-inside:1.0`
 
+
+### Haswell
+
+```
+salloc -N 4 -C haswell -q interactive --image=registry.nersc.gov/das/delta:3.0
+```
+
+cd to wherever you have installed delta. For me it's 
+
+```
+cd /global/cscratch1/sd/stephey/delta/delta
+```
+
+And then launch delta:
+
+
+```
+OMP_NUM_THREADS=16 srun -n 16 -c 16 --cpu-bind=cores shifter python3 -m mpi4py.futures processor.py --config configs/hackathon_test.json --transport transport_tx  --num_ranks_preprocess=4 --num_ranks_analysis=12 --num_queue_threads=4 --run_id=test_25259_GT
+```
+
+
+### corigpu
+
 On corigpu, request the image during your interactive job:
 
 ```
@@ -17,19 +40,22 @@ module load cgpu
 salloc -N 1 -C gpu -G 1 -t 120 -A m499 --image=registry.nersc.gov/das/delta:3.0
 ```
 
-On Cori, `module unload cgpu` and adjust your salloc/sbatch accordingly.
+```
+OMP_NUM_THREADS=16 srun -n 16 -c 1 --cpu-bind=cores shifter python3 -m mpi4py.futures processor.py --config configs/hackathon_test.json --transport transport_tx  --num_ranks_preprocess=2 --num_ranks_analysis=2 --num_queue_threads=2 --run_id=test_25259_GT
+
 
-To open a shell inside the container and run tests/scripts inside:
+### Debugging inside the container
 
 ```
-shifter --volume="/global/cscratch1/sd/stephey/delta:/opt/delta" /bin/bash
+shifter /bin/bash
 ```
 
-The `--volume` command will bind-mount your delta installation into
-`/opt/delta` inside the image. Note that the bind location `/opt/delta` must
-exist inside the image.
+### Known issues
 
-To use shifter to run your tests/scripts:
+Pytest has trouble running in the bind-mounted configuration for reasons I
+don't understand. To run pytest you'll have to run in the container:
 
-shifter --volume="/global/cscratch1/sd/stephey/delta:/opt/delta" ls /opt/delta/tests
+```
+shifter --volume="/global/cscratch1/sd/stephey/delta/delta:/opt/delta" /bin/bash
+pytest /opt/delta/tests
 ```