From 07f0def4be393457a42cd0bc34b417bfb57c0a06 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Wed, 4 Oct 2023 14:53:00 -0500 Subject: [PATCH 1/3] simulator: Fix bad UX with shared volumes We can make an educated guess that fixes 99% of people's struggle. The person that might want to extra flexibility can run the simulator by hand with the knowledge they posses. Signed-off-by: Andy Doan --- jobserv/api/run.py | 8 ++------ runner/jobserv_runner/simulator.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/jobserv/api/run.py b/jobserv/api/run.py index 92ea6845..1357a046 100644 --- a/jobserv/api/run.py +++ b/jobserv/api/run.py @@ -344,10 +344,6 @@ def run_get_simulate_sh(proj, build_id, run): rundef_str = rundef_str.replace("$", "\\$") rundef_str = rundef_str.replace("\\", "\\\\") - volumes = "" - for vol in (rundef.get("shared-volumes") or {}).keys(): - volumes += " -v " + vol + "=" - script = """#!/bin/sh -e SIMDIR="${{SIMDIR-/tmp/sim-run}}" @@ -360,9 +356,9 @@ def run_get_simulate_sh(proj, build_id, run): EIEIO wget -O runner {runner} -PYTHONPATH=./runner python3 -m jobserv_runner.simulator -w `pwd` {volumes} rundef.json +PYTHONPATH=./runner python3 -m jobserv_runner.simulator -w `pwd` rundef.json """.format( - rundef=rundef_str, volumes=volumes, runner=runner + rundef=rundef_str, runner=runner ) return script, 200, {"Content-Type": "text/plain"} diff --git a/runner/jobserv_runner/simulator.py b/runner/jobserv_runner/simulator.py index d0b61e22..a6b60d10 100644 --- a/runner/jobserv_runner/simulator.py +++ b/runner/jobserv_runner/simulator.py @@ -15,7 +15,7 @@ def main(args): m.handler.execute(args.worker_dir, args.runner_dir, args.rundef) -def _update_shared_volumes_mapping(volumes, rundef): +def _update_shared_volumes_mapping(worker_dir, volumes, rundef): """Convert rundef mappings: name1: /path/in/container1 name2: /path/in/container2 @@ -36,7 +36,12 @@ def _update_shared_volumes_mapping(volumes, rundef): host_path = volumes[name] mapping[host_path] = container_path except KeyError: - sys.exit(f"Please specify a shared volume path for: {name}") + volpath = os.path.join(worker_dir, "shared-volumes", name) + print(f"Shared volume not specified for: {name}. Default to: {volpath}") + try: + os.makedirs(volpath) + except FileExistsError: + pass rundef["shared-volumes"] = mapping @@ -65,7 +70,7 @@ def get_args(args=None): args.rundef = json.load(args.rundef) args.rundef["simulator"] = True - _update_shared_volumes_mapping(vols, args.rundef) + _update_shared_volumes_mapping(args.worker_dir, vols, args.rundef) if not os.path.isdir(args.worker_dir): sys.exit("worker-dir does not exist: " + args.worker_dir) From 02d6d7f036e3da3be2876cfb4208f797e85eee41 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 6 Oct 2023 11:04:26 -0500 Subject: [PATCH 2/3] simulator: Add ability to process "inputs" This lets us create a rundef that can help guide the user as to what fields they need to set. Signed-off-by: Andy Doan --- runner/jobserv_runner/simulator.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/runner/jobserv_runner/simulator.py b/runner/jobserv_runner/simulator.py index a6b60d10..0492b2ea 100644 --- a/runner/jobserv_runner/simulator.py +++ b/runner/jobserv_runner/simulator.py @@ -2,6 +2,7 @@ # Author: Andy Doan import argparse +from base64 import b64encode import importlib import json import os @@ -45,6 +46,25 @@ def _update_shared_volumes_mapping(worker_dir, volumes, rundef): rundef["shared-volumes"] = mapping +def _handle_inputs(rundef_path, rundef): + inputs = rundef.get("simulator-inputs") + if not inputs: + return + for item in inputs: + value = input(item["prompt"]) + for name, handler in item["secrets"].items(): + transform = handler.get("transform") + if transform and transform == "BasicAuth": + encoded = b64encode(value.encode()).decode() + value = f"Authorization: basic {encoded}" + elif transform: + sys.exit(f"unknown secret transform for {item}") + rundef["secrets"][name] = value + del rundef["simulator-inputs"] + with open(rundef_path, "w") as f: + json.dump(rundef, f, indent=2) + + def get_args(args=None): parser = argparse.ArgumentParser(description="Execute a JobServ run definition") parser.add_argument("-w", "--worker-dir", help="Location to store the run") @@ -68,9 +88,11 @@ def get_args(args=None): sys.exit(f"Invalid shared-volume: {k}. {v} does not exist") vols[k] = v + rundef_path = args.rundef.name args.rundef = json.load(args.rundef) args.rundef["simulator"] = True _update_shared_volumes_mapping(args.worker_dir, vols, args.rundef) + _handle_inputs(rundef_path, args.rundef) if not os.path.isdir(args.worker_dir): sys.exit("worker-dir does not exist: " + args.worker_dir) From a8ede07f734a2c16aeb54750267794e6893ebeeb Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 6 Oct 2023 11:10:54 -0500 Subject: [PATCH 3/3] simulator: Exit with error when secret is not set Signed-off-by: Andy Doan --- runner/jobserv_runner/simulator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runner/jobserv_runner/simulator.py b/runner/jobserv_runner/simulator.py index 0492b2ea..e5a73c5a 100644 --- a/runner/jobserv_runner/simulator.py +++ b/runner/jobserv_runner/simulator.py @@ -94,6 +94,10 @@ def get_args(args=None): _update_shared_volumes_mapping(args.worker_dir, vols, args.rundef) _handle_inputs(rundef_path, args.rundef) + for name, val in (args.rundef.get("secrets") or {}).items(): + if val == "TODO": + sys.exit(f"Missing required secret value in run definition: {name}") + if not os.path.isdir(args.worker_dir): sys.exit("worker-dir does not exist: " + args.worker_dir) args.runner_dir = os.path.join(args.worker_dir, "run")