From 2ebfbf4e34b6cec4d4c1912d9feac824e54a5e18 Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Fri, 3 May 2024 13:43:08 -0400 Subject: [PATCH 1/8] add target option to push --- push | 85 +++++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 18 deletions(-) diff --git a/push b/push index c0d319d1f..bb4421a1d 100755 --- a/push +++ b/push @@ -16,10 +16,13 @@ from itertools import chain from zipfile import ZipFile import tempfile from contextlib import contextmanager +import json +import re _DEFAULT_EXTRAS = {'stdout': sys.stdout, 'stderr': sys.stderr} _SSH_EXTRA_OPTS = ['-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null'] +_MANIFEST_FILE_PATH = "/usr/lib/firmware/opentrons-firmware.json" class CantFindUtilityException(RuntimeError): def __init__(self, which_util): @@ -40,9 +43,19 @@ def _scp_to_robot(scp_util, host, local, remote, **extras): **extras ) +def _scp_from_robot(scp_util, host, local, remote, **extras): + _cmd( + [scp_util] + + _SSH_EXTRA_OPTS + + ['root@{host}:{remote}'.format(host=host, remote=remote), local], + **extras + ) + + def _cmd(cmdlist, **extras): _extras = {k: v for k, v in chain(_DEFAULT_EXTRAS.items(), extras.items())} + # breakpoint() print(' '.join(cmdlist)) subprocess.run(cmdlist, **_extras).check_returncode() @@ -54,32 +67,63 @@ def _controlled_tempdir(): finally: shutil.rmtree(td) -def _build_fw(zip_path, apps_path): +def _build_fw(zip_path, apps_path, targets): + regex_list = [re.compile(f"{target}" + r"(.*).hex") for target in targets] with ZipFile(zip_path, 'w') as zf: for fname in os.listdir(apps_path): - zf.write(os.path.join(apps_path, fname), fname) - -def _transfer_firmware(host, repo_path, scp, ssh, sensors): + if any([reg.search(fname) for reg in regex_list]): + # breakpoint() + zf.write(os.path.join(apps_path, fname), fname) + # breakpoint() + + +def _update_shortsha(json_data, targets): + shortsha = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode().strip() + with open(json_data, 'r+') as output: + manifest = json.load(output) + for target in targets: + if target not in manifest['subsystems']: + print(f"target {target} not found, continuing.") + continue + manifest['subsystems'][target]['shortsha'] = shortsha + json.dump(manifest, output) + +def _transfer_firmware(host, repo_path, scp, ssh, targets, sensors): dist_dir = "dist" + # apps_path = os.path.join(repo_path, 'dist', 'applications') + # breakpoint() if sensors: dist_dir = dist_dir+"-sensor" apps_path = os.path.join(repo_path, dist_dir, 'applications') with _controlled_tempdir() as td: local_zip_path = os.path.join(td, 'fw.zip') robot_zip_path = '/tmp/fw.zip' - _build_fw(local_zip_path, apps_path) + local_temp_manifest_path = os.path.join(td, 'temp_manifest.json') + breakpoint() + _scp_from_robot(scp, host, local_temp_manifest_path, _MANIFEST_FILE_PATH) + # modify this + _build_fw(local_zip_path, apps_path, targets) + if targets: + _update_shortsha(local_temp_manifest_path, targets) _scp_to_robot(scp, host, local_zip_path, robot_zip_path) _ssh(ssh, host, 'unzip -o {zip_path} -d /usr/lib/firmware/'.format(zip_path=robot_zip_path)) _ssh(ssh, host, 'rm {zip_path}'.format(zip_path=robot_zip_path)) -def _prep_firmware(repo_path, cmake, sensors): - preset = "firmware-g4" +def _prep_firmware(repo_path, cmake, sensors, targets): working_dir = "./build-cross" - if sensors: - preset = preset+"-sensors" - working_dir = working_dir+"-sensor" - _cmd([cmake, '--build', f'--preset={preset}', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) - _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) + # make sure -sensors still works + + if targets: + for target in targets: + # breakpoint() + _cmd([cmake, '--build', 'build-cross', '--target', f'{target}-images'], cwd=repo_path) + # might not need this line becasue --install updates the manifest file + # _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) + + else: + _cmd([cmake, '--build', f'--preset=firmware-g4', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) + _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) + @contextmanager def _prep_robot(host, ssh): @@ -104,19 +148,20 @@ def _find_utils(): def _restart_robot(host, ssh): _ssh(ssh, host, 'nohup systemctl restart opentrons-robot-server &') -def _do_push(host, repo_path, build, restart, sensors): +def _do_push(host, repo_path, build, restart, sensors, targets): + ssh, scp, cmake = _find_utils() if build: - _prep_firmware(repo_path, cmake, sensors) + _prep_firmware(repo_path, cmake, sensors, targets) with _prep_robot(host, ssh): - _transfer_firmware(host, repo_path, scp, ssh, sensors) + _transfer_firmware(host, repo_path, scp, ssh, targets, sensors) if restart: _restart_robot(host, ssh) -def push(host, repo_path=None, build=True, restart=True, sensors=False): +def push(host, repo_path=None, build=True, restart=True, sensors=False, targets=[]): repo = repo_path or os.dirname(__file__) try: - _do_push(host, repo, build, restart, sensors) + _do_push(host, repo, build, restart, sensors, targets) return 0 except subprocess.CalledProcessError as e: print( @@ -133,7 +178,7 @@ def _push_from_argparse(args): if args.key: _SSH_EXTRA_OPTS.append('-i') _SSH_EXTRA_OPTS.append(args.key) - return push(args.host, os.path.abspath(args.repo_path), not args.no_build, not args.no_restart, args.sensors) + return push(args.host, os.path.abspath(args.repo_path), not args.no_build, not args.no_restart, args.sensors, args.targets) def _arg_parser(parent=None): parents = [] @@ -169,6 +214,10 @@ def _arg_parser(parent=None): action='store_true', help='Private SSH key to use' ) + parser.add_argument( + '--targets', + nargs='+' + ) return parser def _main(): From 055b9b7d504db35389e5bb1b07da0a74627e9241 Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Fri, 3 May 2024 16:20:09 -0400 Subject: [PATCH 2/8] add target option, need to add sensor option back in --- push | 62 ++++++++++++++++++++++++++---------------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/push b/push index bb4421a1d..59d27651e 100755 --- a/push +++ b/push @@ -22,7 +22,8 @@ import re _DEFAULT_EXTRAS = {'stdout': sys.stdout, 'stderr': sys.stderr} _SSH_EXTRA_OPTS = ['-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null'] -_MANIFEST_FILE_PATH = "/usr/lib/firmware/opentrons-firmware.json" +_ROBOT_MANIFEST_FILE_PATH = "/usr/lib/firmware/opentrons-firmware.json" +_DIST_DIR = "dist" class CantFindUtilityException(RuntimeError): def __init__(self, which_util): @@ -51,11 +52,8 @@ def _scp_from_robot(scp_util, host, local, remote, **extras): **extras ) - - def _cmd(cmdlist, **extras): _extras = {k: v for k, v in chain(_DEFAULT_EXTRAS.items(), extras.items())} - # breakpoint() print(' '.join(cmdlist)) subprocess.run(cmdlist, **_extras).check_returncode() @@ -68,61 +66,55 @@ def _controlled_tempdir(): shutil.rmtree(td) def _build_fw(zip_path, apps_path, targets): - regex_list = [re.compile(f"{target}" + r"(.*).hex") for target in targets] - with ZipFile(zip_path, 'w') as zf: - for fname in os.listdir(apps_path): - if any([reg.search(fname) for reg in regex_list]): - # breakpoint() + if targets: + regex_list = [re.compile(f"{target}" + r"(.*).hex") for target in targets] + with ZipFile(zip_path, 'w') as zf: + for fname in os.listdir(apps_path): + # only write to zip file to be copied if filename matches target + if any([reg.search(fname) for reg in regex_list]): + zf.write(os.path.join(apps_path, fname), fname) + else: + with ZipFile(zip_path, 'w') as zf: + for fname in os.listdir(apps_path): + # write all image files to zip file zf.write(os.path.join(apps_path, fname), fname) - # breakpoint() - -def _update_shortsha(json_data, targets): +def _update_shortsha(scp, host, json_data_path, targets): shortsha = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode().strip() - with open(json_data, 'r+') as output: - manifest = json.load(output) + # copy data to local file + _scp_from_robot(scp, host, json_data_path, _ROBOT_MANIFEST_FILE_PATH) + with open(json_data_path, 'r+') as output_file: + manifest = json.load(output_file) for target in targets: - if target not in manifest['subsystems']: - print(f"target {target} not found, continuing.") - continue manifest['subsystems'][target]['shortsha'] = shortsha - json.dump(manifest, output) + output_file.seek(0) + json.dump(manifest, output_file) + # copy updated subsystem data to the robot + _scp_to_robot(scp, host, json_data_path, _ROBOT_MANIFEST_FILE_PATH) def _transfer_firmware(host, repo_path, scp, ssh, targets, sensors): - dist_dir = "dist" - # apps_path = os.path.join(repo_path, 'dist', 'applications') - # breakpoint() - if sensors: - dist_dir = dist_dir+"-sensor" - apps_path = os.path.join(repo_path, dist_dir, 'applications') + apps_path = os.path.join(repo_path, _DIST_DIR, 'applications') with _controlled_tempdir() as td: local_zip_path = os.path.join(td, 'fw.zip') robot_zip_path = '/tmp/fw.zip' - local_temp_manifest_path = os.path.join(td, 'temp_manifest.json') - breakpoint() - _scp_from_robot(scp, host, local_temp_manifest_path, _MANIFEST_FILE_PATH) - # modify this _build_fw(local_zip_path, apps_path, targets) if targets: - _update_shortsha(local_temp_manifest_path, targets) + local_temp_manifest_path = os.path.join(td, 'temp_manifest.json') + _update_shortsha(scp, host, local_temp_manifest_path, targets) _scp_to_robot(scp, host, local_zip_path, robot_zip_path) _ssh(ssh, host, 'unzip -o {zip_path} -d /usr/lib/firmware/'.format(zip_path=robot_zip_path)) _ssh(ssh, host, 'rm {zip_path}'.format(zip_path=robot_zip_path)) def _prep_firmware(repo_path, cmake, sensors, targets): working_dir = "./build-cross" - # make sure -sensors still works if targets: for target in targets: - # breakpoint() _cmd([cmake, '--build', 'build-cross', '--target', f'{target}-images'], cwd=repo_path) - # might not need this line becasue --install updates the manifest file - # _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) - else: _cmd([cmake, '--build', f'--preset=firmware-g4', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) - _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) + + _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) @contextmanager From dd29abd150888bcf3e711f3f2fb4ab8fb4e300df Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Fri, 3 May 2024 17:10:43 -0400 Subject: [PATCH 3/8] should add sensor functionality back in --- push | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/push b/push index 59d27651e..8b677c3e7 100755 --- a/push +++ b/push @@ -23,8 +23,6 @@ _DEFAULT_EXTRAS = {'stdout': sys.stdout, 'stderr': sys.stderr} _SSH_EXTRA_OPTS = ['-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null'] _ROBOT_MANIFEST_FILE_PATH = "/usr/lib/firmware/opentrons-firmware.json" -_DIST_DIR = "dist" - class CantFindUtilityException(RuntimeError): def __init__(self, which_util): self.util = which_util @@ -93,7 +91,10 @@ def _update_shortsha(scp, host, json_data_path, targets): _scp_to_robot(scp, host, json_data_path, _ROBOT_MANIFEST_FILE_PATH) def _transfer_firmware(host, repo_path, scp, ssh, targets, sensors): - apps_path = os.path.join(repo_path, _DIST_DIR, 'applications') + dist_dir = "dist" + if sensors: + dist_dir = dist_dir+"-sensor" + apps_path = os.path.join(repo_path, dist_dir, 'applications') with _controlled_tempdir() as td: local_zip_path = os.path.join(td, 'fw.zip') robot_zip_path = '/tmp/fw.zip' @@ -108,11 +109,16 @@ def _transfer_firmware(host, repo_path, scp, ssh, targets, sensors): def _prep_firmware(repo_path, cmake, sensors, targets): working_dir = "./build-cross" - if targets: - for target in targets: - _cmd([cmake, '--build', 'build-cross', '--target', f'{target}-images'], cwd=repo_path) + if sensors: + working_dir = working_dir+"-sensor" + _cmd([cmake, '--build', f'--preset=firmware-g4-sensors', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) else: - _cmd([cmake, '--build', f'--preset=firmware-g4', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) + if targets: + for target in targets: + _cmd([cmake, '--build', 'build-cross', '--target', f'{target}-images'], cwd=repo_path) + else: + _cmd([cmake, '--build', f'--preset=firmware-g4', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) + _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) @@ -151,6 +157,10 @@ def _do_push(host, repo_path, build, restart, sensors, targets): _restart_robot(host, ssh) def push(host, repo_path=None, build=True, restart=True, sensors=False, targets=[]): + # sensors is logically independent from targets here- if you specify both: + # - all hex files under firmware-g4-sensors will be built and installed, but + # - only the targets selected will actually be copied over to the robot + repo = repo_path or os.dirname(__file__) try: _do_push(host, repo, build, restart, sensors, targets) From 943c3d9dd87d26f00a85885dc519503974f11491 Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Mon, 6 May 2024 11:18:56 -0400 Subject: [PATCH 4/8] couple formatting changes --- push | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/push b/push index 8b677c3e7..cc27d5287 100755 --- a/push +++ b/push @@ -23,6 +23,7 @@ _DEFAULT_EXTRAS = {'stdout': sys.stdout, 'stderr': sys.stderr} _SSH_EXTRA_OPTS = ['-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null'] _ROBOT_MANIFEST_FILE_PATH = "/usr/lib/firmware/opentrons-firmware.json" + class CantFindUtilityException(RuntimeError): def __init__(self, which_util): self.util = which_util @@ -90,7 +91,7 @@ def _update_shortsha(scp, host, json_data_path, targets): # copy updated subsystem data to the robot _scp_to_robot(scp, host, json_data_path, _ROBOT_MANIFEST_FILE_PATH) -def _transfer_firmware(host, repo_path, scp, ssh, targets, sensors): +def _transfer_firmware(host, repo_path, scp, ssh, sensors, targets): dist_dir = "dist" if sensors: dist_dir = dist_dir+"-sensor" @@ -108,21 +109,20 @@ def _transfer_firmware(host, repo_path, scp, ssh, targets, sensors): def _prep_firmware(repo_path, cmake, sensors, targets): working_dir = "./build-cross" + full_build_preset = "firmware-g4" if sensors: working_dir = working_dir+"-sensor" - _cmd([cmake, '--build', f'--preset=firmware-g4-sensors', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) + full_build_preset = full_build_preset+"-sensors" + if targets: + for target in targets: + _cmd([cmake, '--build', 'build-cross', '--target', f'{target}-images'], cwd=repo_path) else: - if targets: - for target in targets: - _cmd([cmake, '--build', 'build-cross', '--target', f'{target}-images'], cwd=repo_path) - else: - _cmd([cmake, '--build', f'--preset=firmware-g4', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) + _cmd([cmake, '--build', f'--preset={full_build_preset}', '--target', 'firmware-applications', 'firmware-images'], cwd=repo_path) _cmd([cmake, '--install', f'{working_dir}', '--component', 'Applications'], cwd=repo_path) - @contextmanager def _prep_robot(host, ssh): _ssh(ssh, host, 'mount -o remount,rw /') @@ -152,7 +152,7 @@ def _do_push(host, repo_path, build, restart, sensors, targets): if build: _prep_firmware(repo_path, cmake, sensors, targets) with _prep_robot(host, ssh): - _transfer_firmware(host, repo_path, scp, ssh, targets, sensors) + _transfer_firmware(host, repo_path, scp, ssh, sensors, targets) if restart: _restart_robot(host, ssh) From b5a3817706d5f780ea4f22e3dbc2b7083f2f083a Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Mon, 6 May 2024 13:50:06 -0400 Subject: [PATCH 5/8] address change requests --- push | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/push b/push index cc27d5287..376ebfa47 100755 --- a/push +++ b/push @@ -39,7 +39,7 @@ def _scp_to_robot(scp_util, host, local, remote, **extras): _cmd( [scp_util] + _SSH_EXTRA_OPTS - + [local, 'root@{host}:{remote}'.format(host=host, remote=remote)], + + [local, f'root@{host}:{remote}'], **extras ) @@ -47,7 +47,7 @@ def _scp_from_robot(scp_util, host, local, remote, **extras): _cmd( [scp_util] + _SSH_EXTRA_OPTS - + ['root@{host}:{remote}'.format(host=host, remote=remote), local], + + [f'root@{host}:{remote}', local], **extras ) @@ -66,7 +66,7 @@ def _controlled_tempdir(): def _build_fw(zip_path, apps_path, targets): if targets: - regex_list = [re.compile(f"{target}" + r"(.*).hex") for target in targets] + regex_list = [re.compile(f"{target}" + r"(.*)(.hex|.bin)") for target in targets] with ZipFile(zip_path, 'w') as zf: for fname in os.listdir(apps_path): # only write to zip file to be copied if filename matches target @@ -218,7 +218,7 @@ def _arg_parser(parent=None): ) parser.add_argument( '--targets', - nargs='+' + nargs='*' ) return parser From 26fab8e51c72f0d1c50a470c150a0d2b76c9699c Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Mon, 6 May 2024 16:02:06 -0400 Subject: [PATCH 6/8] ignore targets within prep_fw if sensors --- push | 2 ++ 1 file changed, 2 insertions(+) diff --git a/push b/push index 376ebfa47..5c59be9f2 100755 --- a/push +++ b/push @@ -114,6 +114,8 @@ def _prep_firmware(repo_path, cmake, sensors, targets): if sensors: working_dir = working_dir+"-sensor" full_build_preset = full_build_preset+"-sensors" + # if sensors is true, disregard targets within the scope of this function + targets = None if targets: for target in targets: _cmd([cmake, '--build', 'build-cross', '--target', f'{target}-images'], cwd=repo_path) From 030a2cbc41d5a32ca3f37595fa4fa4addffeed62 Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Mon, 13 May 2024 11:15:01 -0400 Subject: [PATCH 7/8] check targets --- push | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/push b/push index 5c59be9f2..b3e82a175 100755 --- a/push +++ b/push @@ -23,6 +23,21 @@ _DEFAULT_EXTRAS = {'stdout': sys.stdout, 'stderr': sys.stderr} _SSH_EXTRA_OPTS = ['-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null'] _ROBOT_MANIFEST_FILE_PATH = "/usr/lib/firmware/opentrons-firmware.json" +TARGETS = [ + "pipettes", + "pipettes-rev1", + "pipettes-single", + "pipettes-multi", + "pipettes-96", + "gripper", + "hepa-uv", + "gantry", + "gantry-x", + "gantry-y", + "head", + "rear-panel", + "bootloader", +] class CantFindUtilityException(RuntimeError): def __init__(self, which_util): @@ -145,12 +160,21 @@ def _find_utils(): raise CantFindUtilityException('cmake') return ssh, scp, cmake +def _check_targets(targets): + for t in targets: + if t not in TARGETS: + print(f"preset {t} is not in target options, ignoring") + targets.remove(t) + return targets + def _restart_robot(host, ssh): _ssh(ssh, host, 'nohup systemctl restart opentrons-robot-server &') def _do_push(host, repo_path, build, restart, sensors, targets): ssh, scp, cmake = _find_utils() + if targets: + targets = _check_targets(targets) if build: _prep_firmware(repo_path, cmake, sensors, targets) with _prep_robot(host, ssh): From 4e737e7371dc36ff1f06d19cec82c0fb6c423212 Mon Sep 17 00:00:00 2001 From: caila-marashaj Date: Mon, 13 May 2024 11:48:18 -0400 Subject: [PATCH 8/8] update all subsystems for targets that cover multiple --- push | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/push b/push index b3e82a175..9eb3939f3 100755 --- a/push +++ b/push @@ -38,6 +38,10 @@ TARGETS = [ "rear-panel", "bootloader", ] +_MULTI_SUBSYSTEM_TARGETS = { + "pipettes": ["pipettes-single", "pipettes-multi", "pipettes-96"], + "gantry": ["gantry-x", "gantry-y"] +} class CantFindUtilityException(RuntimeError): def __init__(self, which_util): @@ -93,13 +97,26 @@ def _build_fw(zip_path, apps_path, targets): # write all image files to zip file zf.write(os.path.join(apps_path, fname), fname) + +def _subsystems_from_targets(targets): + # assuming all targets are valid at this point, convert + # presets that encompass multiple subsystems to their + # respective subsystems + for t in targets: + if t in _MULTI_SUBSYSTEM_TARGETS: + t_index = targets.index(t) + # replace the target with multiple subsystems + targets[t_index:t_index+1] = tuple(_MULTI_SUBSYSTEM_TARGETS[t]) + return targets + + def _update_shortsha(scp, host, json_data_path, targets): shortsha = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode().strip() # copy data to local file _scp_from_robot(scp, host, json_data_path, _ROBOT_MANIFEST_FILE_PATH) with open(json_data_path, 'r+') as output_file: manifest = json.load(output_file) - for target in targets: + for target in _subsystems_from_targets(targets): manifest['subsystems'][target]['shortsha'] = shortsha output_file.seek(0) json.dump(manifest, output_file)