Use Hierarchical Machine Config

src/murfey/server/api/clem.py

@@ -83,7 +83,7 @@ def validate_and_sanitise(
     machine_config = get_machine_config(instrument_name=instrument_name)[
         instrument_name
     ]
-    rsync_basepath = machine_config.rsync_basepath.resolve()
+    rsync_basepath = (machine_config.rsync_basepath or Path("")).resolve()
 
     # Check that full file path doesn't contain unallowed characters
     #   Currently allows only:

src/murfey/server/api/file_io_frontend.py

@@ -50,10 +50,11 @@
     machine_config = get_machine_config(instrument_name=instrument_name)[
         instrument_name
     ]
-    symlink_full_path = machine_config.rsync_basepath / symlink_params.symlink
+    rsync_basepath = (machine_config.rsync_basepath or Path("")).resolve()
+    symlink_full_path = rsync_basepath / symlink_params.symlink
     if symlink_full_path.is_symlink() and symlink_params.override:
         symlink_full_path.unlink()
     if symlink_full_path.exists():
         return ""
-    symlink_full_path.symlink_to(machine_config.rsync_basepath / symlink_params.target)
+    symlink_full_path.symlink_to(rsync_basepath / symlink_params.target)
     return str(symlink_params.symlink)

src/murfey/server/api/file_io_instrument.py

@@ -57,7 +57,8 @@ def suggest_path(
         )
 
     # Construct the full path to where the dataset is to be saved
-    check_path = machine_config.rsync_basepath / base_path
+    rsync_basepath = (machine_config.rsync_basepath or Path("")).resolve()
+    check_path = rsync_basepath / base_path
 
     # Check previous year to account for the year rolling over during data collection
     if not check_path.parent.exists():
@@ -69,7 +70,7 @@ def suggest_path(
                 base_path_parts[year_idx] = str(int(part) - 1)
         base_path = "/".join(base_path_parts)
         check_path_prev = check_path
-        check_path = machine_config.rsync_basepath / base_path
+        check_path = rsync_basepath / base_path
 
         # If it's not in the previous year either, it's a genuine error
         if not check_path.parent.exists():
@@ -88,7 +89,7 @@ def suggest_path(
         check_path.mkdir(mode=0o750)
         if params.extra_directory:
             (check_path / secure_filename(params.extra_directory)).mkdir(mode=0o750)
-    return {"suggested_path": check_path.relative_to(machine_config.rsync_basepath)}
+    return {"suggested_path": check_path.relative_to(rsync_basepath)}
 
 
 class Dest(BaseModel):
@@ -107,7 +108,9 @@ def make_rsyncer_destination(session_id: int, destination: Dest, db=murfey_db):
     ]
     if not machine_config:
         raise ValueError("No machine configuration set when making rsyncer destination")
-    full_destination_path = machine_config.rsync_basepath / destination_path
+    full_destination_path = (
+        machine_config.rsync_basepath or Path("")
+    ).resolve() / destination_path
     for parent_path in full_destination_path.parents:
         parent_path.mkdir(mode=0o750, exist_ok=True)
     return destination
@@ -151,7 +154,7 @@ async def write_eer_fractionation_file(
         ) / secure_filename(fractionation_params.fractionation_file_name)
     else:
         file_path = (
-            Path(machine_config.rsync_basepath)
+            (machine_config.rsync_basepath or Path("")).resolve()
             / str(datetime.now().year)
             / secure_filename(visit_name)
             / machine_config.gain_directory_name

src/murfey/server/api/file_io_shared.py

@@ -37,8 +37,9 @@ async def process_gain(
         executables = machine_config.external_executables
     env = machine_config.external_environment
     safe_path_name = secure_filename(gain_reference_params.gain_ref.name)
+    rsync_basepath = machine_config.rsync_basepath or Path("")
     filepath = (
-        Path(machine_config.rsync_basepath)
+        rsync_basepath
         / str(datetime.now().year)
         / secure_filename(visit_name)
         / machine_config.gain_directory_name
@@ -48,7 +49,7 @@ async def process_gain(
     if not filepath.exists():
         filepath_prev = filepath
         filepath = (
-            Path(machine_config.rsync_basepath)
+            rsync_basepath
             / str(datetime.now().year - 1)
             / secure_filename(visit_name)
             / machine_config.gain_directory_name
@@ -80,14 +81,12 @@ async def process_gain(
         )
     if new_gain_ref and new_gain_ref_superres:
         return {
-            "gain_ref": new_gain_ref.relative_to(Path(machine_config.rsync_basepath)),
-            "gain_ref_superres": new_gain_ref_superres.relative_to(
-                Path(machine_config.rsync_basepath)
-            ),
+            "gain_ref": new_gain_ref.relative_to(rsync_basepath),
+            "gain_ref_superres": new_gain_ref_superres.relative_to(rsync_basepath),
         }
     elif new_gain_ref:
         return {
-            "gain_ref": new_gain_ref.relative_to(Path(machine_config.rsync_basepath)),
+            "gain_ref": new_gain_ref.relative_to(rsync_basepath),
             "gain_ref_superres": None,
         }
     else:

src/murfey/server/api/workflow.py

@@ -214,15 +214,14 @@ def start_dc(
     machine_config = get_machine_config(instrument_name=instrument_name)[
         instrument_name
     ]
+    rsync_basepath = (machine_config.rsync_basepath or Path("")).resolve()
     logger.info(
         f"Starting data collection on microscope {instrument_name!r} "
-        f"with basepath {sanitise(str(machine_config.rsync_basepath))} and directory {sanitise(dc_params.image_directory)}"
+        f"with basepath {sanitise(str(rsync_basepath))} and directory {sanitise(dc_params.image_directory)}"
     )
     dc_parameters = {
         "visit": visit_name,
-        "image_directory": str(
-            machine_config.rsync_basepath / dc_params.image_directory
-        ),
+        "image_directory": str(rsync_basepath / dc_params.image_directory),
         "start_time": str(datetime.now()),
         "voltage": dc_params.voltage,
         "pixel_size": str(float(dc_params.pixel_size_on_image) * 1e9),
@@ -713,7 +712,10 @@ async def request_tomography_preprocessing(
                 "fm_dose": proc_file.dose_per_frame,
                 "frame_count": proc_file.frame_count,
                 "gain_ref": (
-                    str(machine_config.rsync_basepath / proc_file.gain_ref)
+                    str(
+                        (machine_config.rsync_basepath or Path("")).resolve()
+                        / proc_file.gain_ref
+                    )
                     if proc_file.gain_ref and machine_config.data_transfer_enabled
                     else proc_file.gain_ref
                 ),
@@ -1029,7 +1031,7 @@ async def make_gif(
         instrument_name
     ]
     output_dir = (
-        Path(machine_config.rsync_basepath)
+        (machine_config.rsync_basepath or Path("")).resolve()
         / secure_filename(year)
         / secure_filename(visit_name)
         / "processed"

src/murfey/server/feedback.py

@@ -1100,7 +1100,9 @@ def _register_class_selection(message: dict, _db, demo: bool = False):
 def _find_initial_model(visit: str, machine_config: MachineConfig) -> Path | None:
     if machine_config.initial_model_search_directory:
         visit_directory = (
-            machine_config.rsync_basepath / str(datetime.now().year) / visit
+            (machine_config.rsync_basepath or Path("")).resolve()
+            / str(datetime.now().year)
+            / visit
         )
         possible_models = [
             p
@@ -1512,7 +1514,10 @@ def _flush_tomography_preprocessing(message: dict, _db):
                 "fm_dose": proc_params.dose_per_frame,
                 "frame_count": proc_params.frame_count,
                 "gain_ref": (
-                    str(machine_config.rsync_basepath / proc_params.gain_ref)
+                    str(
+                        (machine_config.rsync_basepath or Path("")).resolve()
+                        / proc_params.gain_ref
+                    )
                     if proc_params.gain_ref
                     else proc_params.gain_ref
                 ),
@@ -2042,7 +2047,10 @@ def feedback_callback(header: dict, message: dict, _db=murfey_db) -> None:
                     angpix=float(message["pixel_size_on_image"]) * 1e10,
                     dose_per_frame=message["dose_per_frame"],
                     gain_ref=(
-                        str(machine_config.rsync_basepath / message["gain_ref"])
+                        str(
+                            (machine_config.rsync_basepath or Path("")).resolve()
+                            / message["gain_ref"]
+                        )
                         if message["gain_ref"] and machine_config.data_transfer_enabled
                         else message["gain_ref"]
                     ),

src/murfey/util/config.py

@@ -36,14 +36,14 @@ class MachineConfig(BaseModel):  # type: ignore
     # Hardware and software -----------------------------------------------------------
     camera: str = "FALCON"
     superres: bool = False
-    calibrations: dict[str, Any]
-    acquisition_software: list[str]
+    calibrations: dict[str, Any] = {}
+    acquisition_software: list[str] = []
     software_versions: dict[str, str] = {}
     software_settings_output_directories: dict[str, list[str]] = {}
     data_required_substrings: dict[str, dict[str, list[str]]] = {}
 
     # Client side directory setup -----------------------------------------------------
-    data_directories: list[Path]
+    data_directories: list[Path] = []
     create_directories: list[str] = ["atlas"]
     analyse_created_directories: list[str] = []
     gain_reference_directory: Optional[Path] = None
@@ -58,7 +58,7 @@ class MachineConfig(BaseModel):  # type: ignore
     data_transfer_enabled: bool = True
     rsync_url: str = ""
     rsync_module: str = ""
-    rsync_basepath: Path
+    rsync_basepath: Optional[Path] = None
     allow_removal: bool = False
 
     # Upstream data download setup
@@ -86,7 +86,7 @@ class MachineConfig(BaseModel):  # type: ignore
     }
 
     # Particle picking setup
-    default_model: Path
+    default_model: Optional[Path] = None
     picking_model_search_directory: str = "processing"
     initial_model_search_directory: str = "processing/initial_model"
 

src/murfey/util/processing_params.py

@@ -42,7 +42,9 @@ def cryolo_model_path(visit: str, instrument_name: str) -> Path:
     ]
     if machine_config.picking_model_search_directory:
         visit_directory = (
-            machine_config.rsync_basepath / str(datetime.now().year) / visit
+            (machine_config.rsync_basepath or Path("")).resolve()
+            / str(datetime.now().year)
+            / visit
         )
         possible_models = list(
             (visit_directory / machine_config.picking_model_search_directory).glob(
@@ -51,7 +53,7 @@ def cryolo_model_path(visit: str, instrument_name: str) -> Path:
         )
         if possible_models:
             return sorted(possible_models, key=lambda x: x.stat().st_ctime)[-1]
-    return machine_config.default_model
+    return (machine_config.default_model or Path("")).resolve()
 
 
 class CLEMProcessingParameters(BaseModel):

src/murfey/workflows/clem/__init__.py

@@ -64,7 +64,7 @@ def _validate_and_sanitise(
     machine_config = get_machine_config(instrument_name=instrument_name)[
         instrument_name
     ]
-    rsync_basepath = machine_config.rsync_basepath.resolve()
+    rsync_basepath = (machine_config.rsync_basepath or Path("")).resolve()
 
     # Check that full file path doesn't contain unallowed characters
     # Currently allows only: