CLEM Workflow Update (#669)

* Fixed minor missed issues to the CLEM workflow after the recent patch bump.
* Updated the Analyser and Context logic for the CLEM workflow to identify and process TIFF datasets for any combination of tiles, frames, and colour channels
* Updated and streamlined the feedback-callback workflows for the CLEM workflow to reflect the change in structure of the received RabbitMQ messages.

Author: tieneupin

pyproject.toml

@@ -104,8 +104,7 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey"
 "clem.process_raw_lifs" = "murfey.workflows.clem.process_raw_lifs:zocalo_cluster_request"
 "clem.process_raw_tiffs" = "murfey.workflows.clem.process_raw_tiffs:zocalo_cluster_request"
 "clem.register_align_and_merge_result" = "murfey.workflows.clem.register_align_and_merge_results:register_align_and_merge_result"
-"clem.register_lif_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_lif_preprocessing_result"
-"clem.register_tiff_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:register_tiff_preprocessing_result"
+"clem.register_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:run"
 "pato" = "murfey.workflows.notifications:notification_setup"
 "picked_particles" = "murfey.workflows.spa.picking:particles_picked"
 "spa.flush_spa_preprocess" = "murfey.workflows.spa.flush_spa_preprocess:flush_spa_preprocess"

src/murfey/client/analyser.py

@@ -129,10 +129,9 @@ def _find_context(self, file_path: Path) -> bool:
             self._context = CLEMContext("leica", self._basepath, self._token)
             return True
         # Look for TIFF files associated with CLEM workflow
-        # Leica's autosave mode seems to name the TIFFs in the format
-        # PostionXX--ZXX--CXX.tif
-        if all(
-            pattern in file_path.name for pattern in ("--Z", "--C")
+        # CLEM TIFF files will have "--Stage", "--Z", and/or "--C" in their file stem
+        if any(
+            pattern in file_path.stem for pattern in ("--Stage", "--Z", "--C")
         ) and file_path.suffix in (".tiff", ".tif"):
             self._context = CLEMContext("leica", self._basepath, self._token)
             return True

src/murfey/client/contexts/clem.py

@@ -6,7 +6,6 @@
 import logging
 from pathlib import Path
 from typing import Dict, Generator, List, Optional
-from urllib.parse import quote
 from xml.etree import ElementTree as ET
 
 from defusedxml.ElementTree import parse
@@ -146,58 +145,40 @@ def post_transfer(
 
             # Process TIF/TIFF files
             if transferred_file.suffix in (".tif", ".tiff"):
-
-                # Files should be named "PositionX--ZXX--CXX.tif" by default
-                # If Position is repeated, it will add an additional --00X to the end
-                if len(transferred_file.stem.split("--")) not in [3, 4]:
+                # CLEM TIFF files will have "--Stage", "--C", and/or "--Z" in their file stem
+                if not any(
+                    pattern in transferred_file.stem
+                    for pattern in ("--Stage", "--Z", "--C")
+                ):
                     logger.warning(
                         f"File {transferred_file.name!r} is likely not part of the CLEM workflow"
                     )
                     return False
-
                 logger.debug(
                     f"File {transferred_file.name!r} is part of a TIFF image series"
                 )
 
                 # Create a unique name for the series
-                # For standard file name
-                if len(transferred_file.stem.split("--")) == 3:
-                    series_name = "--".join(
-                        [
-                            *destination_file.parent.parts[
-                                -2:
-                            ],  # Upper 2 parent directories
-                            destination_file.stem.split("--")[0],
-                        ]
-                    )
-                # When this a repeated position
-                elif len(transferred_file.stem.split("--")) == 4:
-                    series_name = "--".join(
-                        [
-                            *destination_file.parent.parts[
-                                -2:
-                            ],  # Upper 2 parent directories
-                            "--".join(
-                                destination_file.stem.split("--")[i] for i in [0, -1]
-                            ),
-                        ]
-                    )
-                else:
-                    logger.error(
-                        f"Series name could not be generated from file {transferred_file.name!r}"
-                    )
-                    return False
-                logger.debug(
-                    f"File {transferred_file.name!r} given the series name {series_name!r}"
+                series_name = "--".join(
+                    [
+                        *destination_file.parent.parts[
+                            -2:
+                        ],  # Upper 2 parent directories
+                        destination_file.stem.split("--")[0].replace(" ", "_"),
+                    ]
                 )
 
                 # Create key-value pairs containing empty list if not already present
                 if series_name not in self._tiff_series.keys():
                     self._tiff_series[series_name] = []
+                    logger.debug(
+                        f"Created new dictionary entry for TIFF series {series_name!r}"
+                    )
+
                 # Append information to list
                 self._tiff_series[series_name].append(str(destination_file))
                 logger.debug(
-                    f"Created TIFF file dictionary entries for {series_name!r}"
+                    f"File {transferred_file.name!r} added to series {series_name!r}"
                 )
 
                 # Register the TIFF file in the database
@@ -231,43 +212,40 @@ def post_transfer(
                 # XLIF files don't have the "--ZXX--CXX" additions in the file name
                 # But they have "/Metadata/" as the immediate parent
                 series_name = "--".join(
-                    [*destination_file.parent.parent.parts[-2:], destination_file.stem]
+                    [
+                        *destination_file.parent.parent.parts[-2:],
+                        destination_file.stem.replace(" ", "_"),
+                    ]
                 )  # The previous 2 parent directories should be unique enough
-                logger.debug(
-                    f"File {transferred_file.name!r} given the series name {series_name!r}"
-                )
 
                 # Extract metadata to get the expected size of the series
                 metadata = parse(transferred_file).getroot()
                 metadata = _get_image_elements(metadata)[0]
 
                 # Get channel and dimension information
-                channels = metadata.findall(
-                    "Data/Image/ImageDescription/Channels/ChannelDescription"
-                )
-                dimensions = metadata.findall(
-                    "Data/Image/ImageDescription/Dimensions/DimensionDescription"
-                )
+                channels = metadata.findall(".//ChannelDescription")
+                dimensions = metadata.findall(".//DimensionDescription")
 
                 # Calculate expected number of files for this series
                 num_channels = len(channels)
-                num_frames = (
-                    int(dimensions[2].attrib["NumberOfElements"])
-                    if len(dimensions) > 2
-                    else 1
-                )
-                num_files = num_channels * num_frames
+                num_frames = 1
+                num_tiles = 1
+                for dim in dimensions:
+                    if dim.attrib["DimID"] == "3":
+                        num_frames = int(dim.attrib["NumberOfElements"])
+                    if dim.attrib["DimID"] == "10":
+                        num_tiles = int(dim.attrib["NumberOfElements"])
+                num_files = num_channels * num_frames * num_tiles
                 logger.debug(
                     f"Expected number of files in {series_name!r}: {num_files}"
                 )
 
                 # Update dictionary entries
                 self._files_in_series[series_name] = num_files
                 self._series_metadata[series_name] = str(destination_file)
-                logger.debug(f"Created dictionary entries for {series_name!r} metadata")
-
-                # A new copy of the metadata file is created in 'processed', so no need
-                # to register this instance of it
+                logger.debug(
+                    f"File {transferred_file.name!r} added to series {series_name!r}"
+                )
 
             # Post message if all files for the associated series have been collected
             # .get(series_name, 0) returns 0 if no associated key is found
@@ -281,21 +259,26 @@ def post_transfer(
                 return True
             elif len(
                 self._tiff_series.get(series_name, [])
-            ) == self._files_in_series.get(series_name, 0):
+            ) >= self._files_in_series.get(series_name, 0):
                 logger.debug(
                     f"Collected expected number of TIFF files for series {series_name!r}; posting job to server"
                 )
 
                 # Post the message and log any errors that arise
                 tiff_dataset = {
                     "series_name": series_name,
-                    "tiff_files": self._tiff_series[series_name],
+                    "tiff_files": self._tiff_series[series_name][0:1],
                     "series_metadata": self._series_metadata[series_name],
                 }
                 post_result = self.process_tiff_series(tiff_dataset, environment)
                 if post_result is False:
                     return False
                 logger.info(f"Started preprocessing of TIFF series {series_name!r}")
+
+                # Clean up memory after posting
+                del self._tiff_series[series_name]
+                del self._series_metadata[series_name]
+                del self._files_in_series[series_name]
             else:
                 logger.debug(f"TIFF series {series_name!r} is still being processed")
 
@@ -361,7 +344,7 @@ def register_lif_file(
                 function_name="register_lif_file",
                 token=self._token,
                 session_id=environment.murfey_session,
-                data={"lif_file": quote(str(lif_file), safe="")},
+                data={"lif_file": str(lif_file)},
             )
             return True
         except Exception as e:
@@ -387,7 +370,7 @@ def process_lif_file(
                 function_name="process_raw_lifs",
                 token=self._token,
                 session_id=environment.murfey_session,
-                data={"lif_file": quote(str(lif_file), safe="")},
+                data={"lif_file": str(lif_file)},
             )
             return True
         except Exception as e:
@@ -411,7 +394,7 @@ def register_tiff_file(
                 function_name="register_tiff_file",
                 token=self._token,
                 session_id=environment.murfey_session,
-                data={"tiff_file": quote(str(tiff_file), safe="")},
+                data={"tiff_file": str(tiff_file)},
             )
             return True
         except Exception as e:

src/murfey/server/api/clem.py

@@ -204,13 +204,6 @@ class LifInfo(BaseModel):
     child_stacks: list[Path] = []
 
 
-class TiffInfo(BaseModel):
-    tiff_file: Path
-    associated_metadata: Optional[Path] = None
-    associated_series: Optional[str] = None
-    associated_stack: Optional[Path] = None
-
-
 @router.post("/sessions/{session_id}/clem/lif_files")
 def register_lif_file(
     lif_file: LifInfo,
@@ -314,6 +307,13 @@ def register_lif_file(
     return True
 
 
+class TiffInfo(BaseModel):
+    tiff_file: Path
+    associated_metadata: Optional[Path] = None
+    associated_series: Optional[str] = None
+    associated_stack: Optional[Path] = None
+
+
 @router.post("/sessions/{session_id}/clem/tiff_files")
 def register_tiff_file(
     tiff_file: TiffInfo,

src/murfey/workflows/clem/process_raw_lifs.py

@@ -48,7 +48,7 @@ def zocalo_cluster_request(
 
         # Construct recipe and submit it for processing
         recipe = {
-            "recipes": ["clem-lif-to-stack"],
+            "recipes": ["clem-process-raw-lifs"],
             "parameters": {
                 # Job parameters
                 "lif_file": f"{str(file)}",

src/murfey/workflows/clem/process_raw_tiffs.py

@@ -55,7 +55,7 @@ def zocalo_cluster_request(
 
         # Construct recipe and submit it for processing
         recipe = {
-            "recipes": ["clem-tiff-to-stack"],
+            "recipes": ["clem-process-raw-tiffs"],
             "parameters": {
                 # Job parameters
                 "tiff_list": "null",