Merge branch 'main' into 564_organise_testing_utils

README.rst

@@ -13,6 +13,26 @@ Documentation  https://DiamondLightSource.github.io/dodal
 Releases       https://github.com/DiamondLightSource/dodal/releases
 ============== ==============================================================
 
+Testing Connectivity
+--------------------
+
+You can test your connection to a beamline if it's PVs are visible to your machine with:
+
+.. code:: shell
+
+    # On any workstation:
+    dodal connect <BEAMLINE>
+
+    # On a beamline workstation, this should suffice:
+    dodal connect ${BEAMLINE}
+
+
+For more options, including a list of valid beamlines, type
+
+.. code:: shell
+
+    dodal connect --help
+
 
 .. |code_ci| image:: https://github.com/DiamondLightSource/dodal/actions/workflows/code.yml/badge.svg?branch=main
     :target: https://github.com/DiamondLightSource/dodal/actions/workflows/code.yml

docs/user/how-to/create-beamline.rst

@@ -10,8 +10,8 @@ The following example creates a fictitious beamline ``w41``, with a simulated tw
 
 .. code-block:: python
 
-    from dodal.beamlines.beamline_utils import device_instantiation
-    from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+    from dodal.common.beamlines.beamline_utils import device_instantiation
+    from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
     from dodal.devices.areadetector.adaravis import AdAravisDetector
     from dodal.devices.synchrotron import Synchrotron
     from dodal.log import set_beamline as set_log_beamline

pull_request_template.md

@@ -7,4 +7,5 @@ Fixes #ISSUE
 ### Checks for reviewer
 - [ ] Would the PR title make sense to a scientist on a set of release notes
 - [ ] If a new device has been added does it follow the [standards](https://github.com/DiamondLightSource/dodal/wiki/Device-Standards)
-- [ ] If changing the API for a pre-existing device, ensure that any beamlines using this device have updated their Bluesky plans accordingly
+- [ ] If changing the API for a pre-existing device, ensure that any beamlines using this device have updated their Bluesky plans accordingly
+- [ ] Have the connection tests for the relevant beamline(s) been run via `dodal connect ${BEAMLINE}`

pyproject.toml

@@ -12,6 +12,7 @@ classifiers = [
 ]
 description = "Ophyd devices and other utils that could be used across DLS beamlines"
 dependencies = [
+    "click",
     "ophyd",
     "ophyd-async>=0.3a5",
     "bluesky",
@@ -58,6 +59,9 @@ dev = [
     "types-aiofiles",
 ]
 
+[project.scripts]
+dodal = "dodal.__main__:main"
+
 [project.urls]
 GitHub = "https://github.com/DiamondLightSource/dodal"
 

src/dodal/__main__.py

@@ -1,16 +1,8 @@
-from argparse import ArgumentParser
-
-from . import __version__
+from .cli import main
 
 __all__ = ["main"]
 
 
-def main(args=None):
-    parser = ArgumentParser()
-    parser.add_argument("-v", "--version", action="version", version=__version__)
-    args = parser.parse_args(args)
-
-
 # test with: python -m dodal
 if __name__ == "__main__":
     main()

src/dodal/beamlines/README.md

@@ -0,0 +1,3 @@
+# Beamlines
+
+Beamline modules are code-as-configuration. They define the set of devices and common device settings needed for a particular beamline or group of similar beamlines (e.g. a beamline and its digital twin). Some of our tooling depends on the convention of _only_ beamline modules going in this package. Common utilities should go somewhere else e.g. `dodal.utils` or `dodal.beamlines.common`.

src/dodal/beamlines/__init__.py

@@ -0,0 +1,85 @@
+import importlib.util
+from functools import lru_cache
+from pathlib import Path
+from typing import Iterable, Mapping
+
+# Where beamline names (per the ${BEAMLINE} environment variable don't always
+# match up, we have to map between them bidirectionally). The most common use case is
+# beamlines with a "-"" in the name such as "i04-1", which is not valid in a Python
+# module name. Add any new beamlines whose name differs from their module name to this
+# dictionary, which maps ${BEAMLINE} to dodal.beamlines.<MODULE NAME>
+_BEAMLINE_NAME_OVERRIDES = {
+    "i04-1": "i04_1",
+    "i20-1": "i20_1",
+    "s03": "i03",
+}
+
+
+def all_beamline_modules() -> Iterable[str]:
+    """
+    Get the names of all importable modules in beamlines
+
+    Returns:
+        Iterable[str]: Generator of beamline module names
+    """
+
+    # This is done by inspecting file names rather than modules to avoid
+    # premature importing
+    spec = importlib.util.find_spec(__name__)
+    if spec is not None:
+        search_paths = [Path(path) for path in spec.submodule_search_locations]
+        for path in search_paths:
+            for subpath in path.glob("**/*"):
+                if (
+                    subpath.name.endswith(".py")
+                    and subpath.name != "__init__.py"
+                    and ("__pycache__" not in str(subpath))
+                ):
+                    yield subpath.with_suffix("").name
+    else:
+        raise KeyError(f"Unable to find {__name__} module")
+
+
+def all_beamline_names() -> Iterable[str]:
+    """
+    Get the names of all beamlines as per the ${BEAMLINE} environment variable
+
+    Returns:
+        Iterable[str]: Generator of beamline names that dodal supports
+    """
+    inverse_mapping = _module_name_overrides()
+    for module_name in all_beamline_modules():
+        yield from inverse_mapping.get(module_name, set()).union({module_name})
+
+
+@lru_cache
+def _module_name_overrides() -> Mapping[str, set[str]]:
+    """
+    Get the inverse of _BEAMLINE_NAME_OVERRIDES so that modules can be mapped back to
+    beamlines. _BEAMLINE_NAME_OVERRIDES is expected to be a constant so the return
+    value is cached.
+
+    Returns:
+        Mapping[str, set[str]]: A dictionary mapping the name of a dodal module to the
+        set of beamlines it supports.
+    """
+
+    inverse_mapping: dict[str, set[str]] = {}
+    for beamline, module in _BEAMLINE_NAME_OVERRIDES.items():
+        inverse_mapping[module] = inverse_mapping.get(module, set()).union({beamline})
+    return inverse_mapping
+
+
+def module_name_for_beamline(beamline: str) -> str:
+    """
+    Get the module name for a particular beamline, it may differ from the beamline
+    name e.g. i04-1 -> i04_1
+
+    Args:
+        beamline: The beamline name as per the ${BEAMLINE} environment variable
+
+    Returns:
+        str: The importable module name
+    """
+
+    return _BEAMLINE_NAME_OVERRIDES.get(beamline, beamline)

src/dodal/beamlines/i03.py

@@ -1,11 +1,11 @@
 from ophyd_async.panda import HDFPanda
 
-from dodal.beamlines.beamline_utils import (
+from dodal.common.beamlines.beamline_utils import (
     device_instantiation,
     get_directory_provider,
     set_directory_provider,
 )
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
 from dodal.common.udc_directory_provider import PandASubdirectoryProvider
 from dodal.devices.aperturescatterguard import AperturePositions, ApertureScatterguard
 from dodal.devices.attenuator import Attenuator

src/dodal/beamlines/i04.py

@@ -1,5 +1,5 @@
-from dodal.beamlines.beamline_utils import device_instantiation
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import device_instantiation
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
 from dodal.devices.aperturescatterguard import AperturePositions, ApertureScatterguard
 from dodal.devices.attenuator import Attenuator
 from dodal.devices.backlight import Backlight

src/dodal/beamlines/i04_1.py

@@ -1,5 +1,5 @@
-from dodal.beamlines.beamline_utils import device_instantiation
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import device_instantiation
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
 from dodal.devices.backlight import Backlight
 from dodal.devices.detector import DetectorParams
 from dodal.devices.eiger import EigerDetector

src/dodal/beamlines/i20_1.py

@@ -1,5 +1,5 @@
-from dodal.beamlines.beamline_utils import device_instantiation
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import device_instantiation
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
 from dodal.devices.turbo_slit import TurboSlit
 from dodal.log import set_beamline as set_log_beamline
 from dodal.utils import get_beamline_name

src/dodal/beamlines/i22.py

@@ -3,12 +3,13 @@
 from ophyd_async.epics.areadetector import AravisDetector, PilatusDetector
 from ophyd_async.panda import HDFPanda
 
-from dodal.beamlines.beamline_utils import (
+from dodal.common.beamlines.beamline_utils import (
     device_instantiation,
     get_directory_provider,
     set_directory_provider,
 )
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.device_helpers import numbered_slits
 from dodal.common.visit import LocalDirectoryServiceClient, StaticVisitDirectoryProvider
 from dodal.devices.focusing_mirror import FocusingMirror
 from dodal.devices.i22.fswitch import FSwitch
@@ -17,10 +18,6 @@
 from dodal.log import set_beamline as set_log_beamline
 from dodal.utils import get_beamline_name, skip_device
 
-from ._device_helpers import numbered_slits
-from .beamline_utils import device_instantiation, get_directory_provider
-from .beamline_utils import set_beamline as set_utils_beamline
-
 BL = get_beamline_name("i22")
 set_log_beamline(BL)
 set_utils_beamline(BL)

src/dodal/beamlines/i23.py

@@ -1,5 +1,5 @@
-from dodal.beamlines.beamline_utils import device_instantiation
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import device_instantiation
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
 from dodal.devices.i23.gonio import Gonio
 from dodal.devices.oav.pin_image_recognition import PinTipDetection
 from dodal.log import set_beamline as set_log_beamline

src/dodal/beamlines/i24.py

@@ -1,5 +1,5 @@
-from dodal.beamlines.beamline_utils import BL, device_instantiation
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import BL, device_instantiation
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
 from dodal.devices.detector import DetectorParams
 from dodal.devices.eiger import EigerDetector
 from dodal.devices.i24.dual_backlight import DualBacklight

src/dodal/beamlines/p38.py

@@ -3,23 +3,20 @@
 from ophyd_async.epics.areadetector import AravisDetector
 from ophyd_async.panda import HDFPanda
 
-from dodal.beamlines.beamline_utils import (
+from dodal.common.beamlines.beamline_utils import (
     device_instantiation,
     get_directory_provider,
     set_directory_provider,
 )
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.device_helpers import numbered_slits
 from dodal.common.visit import LocalDirectoryServiceClient, StaticVisitDirectoryProvider
 from dodal.devices.focusing_mirror import FocusingMirror
 from dodal.devices.slits import Slits
 from dodal.devices.tetramm import TetrammDetector
 from dodal.log import set_beamline as set_log_beamline
 from dodal.utils import get_beamline_name, skip_device
 
-from ._device_helpers import numbered_slits
-from .beamline_utils import device_instantiation, get_directory_provider
-from .beamline_utils import set_beamline as set_utils_beamline
-
 BL = get_beamline_name("p38")
 set_log_beamline(BL)
 set_utils_beamline(BL)

src/dodal/beamlines/p45.py

@@ -1,12 +1,12 @@
 from ophyd_async.epics.areadetector import AravisDetector
 from ophyd_async.panda import HDFPanda
 
-from dodal.beamlines.beamline_utils import (
+from dodal.common.beamlines.beamline_utils import (
     device_instantiation,
     get_directory_provider,
     set_directory_provider,
 )
-from dodal.beamlines.beamline_utils import set_beamline as set_utils_beamline
+from dodal.common.beamlines.beamline_utils import set_beamline as set_utils_beamline
 from dodal.common.visit import StaticVisitDirectoryProvider
 from dodal.devices.p45 import Choppers, TomoStageWithStretchAndSkew
 from dodal.log import set_beamline as set_log_beamline

src/dodal/cli.py

@@ -0,0 +1,62 @@
+import os
+
+import click
+from bluesky.run_engine import RunEngine
+
+from dodal.beamlines import all_beamline_names, module_name_for_beamline
+from dodal.utils import make_all_devices
+
+from . import __version__
+
+
+@click.group(invoke_without_command=True)
+@click.version_option(version=__version__, message="%(version)s")
+@click.pass_context
+def main(ctx: click.Context) -> None:
+    if ctx.invoked_subcommand is None:
+        print("Please invoke subcommand!")
+
+
+@main.command(name="connect")
+@click.argument(
+    "beamline",
+    type=click.Choice(list(all_beamline_names())),
+    required=True,
+)
+@click.option(
+    "-a",
+    "--all",
+    is_flag=True,
+    help="Attempt to connect to devices marked as skipped",
+    default=False,
+)
+@click.option(
+    "-s",
+    "--sim-backend",
+    is_flag=True,
+    help="Connect to a sim backend, this initializes all device objects but does not "
+    "attempt any I/O. Useful as a a dry-run.",
+    default=False,
+)
+def connect(beamline: str, all: bool, sim_backend: bool) -> None:
+    """Initialises a beamline module, connects to all devices, reports
+    any connection issues."""
+
+    os.environ["BEAMLINE"] = beamline
+
+    module_name = module_name_for_beamline(beamline)
+    full_module_path = f"dodal.beamlines.{module_name}"
+
+    # We need to make a RunEngine to allow ophyd-async devices to connect.
+    # See https://blueskyproject.io/ophyd-async/main/explanations/event-loop-choice.html
+    RunEngine()
+
+    print(f"Attempting connection to {beamline} (using {full_module_path})")
+    devices = make_all_devices(
+        full_module_path,
+        include_skipped=all,
+        fake_with_ophyd_sim=sim_backend,
+    )
+    sim_statement = "sim mode" if sim_backend else ""
+    print(f"{len(devices)} devices connected ({sim_statement}): ")
+    print("\n".join([f"\t{key}" for key in devices.keys()]))

src/dodal/beamlines/beamline_utils.py → src/dodal/common/beamlines/beamline_utils.py

@@ -44,7 +44,7 @@ def active_device_is_same_type(
     return inspect.isclass(device) and isinstance(active_device, device)
 
 
-def _wait_for_connection(
+def wait_for_connection(
     device: AnyDevice,
     timeout: float = DEFAULT_CONNECTION_TIMEOUT,
     mock: bool = False,
@@ -109,7 +109,7 @@ def device_instantiation(
         )
         ACTIVE_DEVICES[name] = device_instance
         if wait:
-            _wait_for_connection(device_instance, mock=fake)
+            wait_for_connection(device_instance, mock=fake)
 
     else:
         if not active_device_is_same_type(already_existing_device, device_factory):

src/dodal/beamlines/_device_helpers.py → src/dodal/common/beamlines/device_helpers.py

@@ -1,8 +1,7 @@
+from dodal.common.beamlines.beamline_utils import device_instantiation
 from dodal.devices.slits import Slits
 from dodal.utils import skip_device
 
-from .beamline_utils import device_instantiation
-
 
 @skip_device()
 def numbered_slits(

src/dodal/common/visit.py

@@ -9,7 +9,7 @@
 from ophyd_async.core import DirectoryInfo
 from pydantic import BaseModel
 
-from dodal.beamlines import beamline_utils
+from dodal.common.beamlines import beamline_utils
 from dodal.common.types import MsgGenerator, UpdatingDirectoryProvider
 from dodal.log import LOGGER