Merge pull request #791 from henrypinkard/mmpycorexrefactor

Factor out mmpycorex into seperate repo

Author: henrypinkard

pycromanager/__init__.py

@@ -1,12 +1,13 @@
 name = "pycromanager"
 
-from pycromanager.acquisition.java_backend_acquisitions import JavaBackendAcquisition, MagellanAcquisition, XYTiledAcquisition, ExploreAcquisition
+from pycromanager.acquisition.java_backend_acquisitions import (JavaBackendAcquisition, MagellanAcquisition,
+                                                                XYTiledAcquisition, ExploreAcquisition)
 from pycromanager.acquisition.acquisition_superclass import multi_d_acquisition_events
 from pycromanager.acquisition.acq_constructor import Acquisition
-from pycromanager.headless import start_headless, stop_headless
 from pycromanager.mm_java_classes import Studio, Magellan
-from pycromanager.core import Core
-from pyjavaz import JavaObject, JavaClass, PullSocket, PushSocket
 from pycromanager.acquisition.acq_eng_py.main.acq_notification import AcqNotification
-from ndstorage import Dataset
+
+from pycromanager.headless import start_headless, stop_headless
+from mmpycorex import download_and_install_mm, find_existing_mm_install, Core
+
 from ._version import __version__, version_info

pycromanager/_version.py

@@ -1,2 +1,2 @@
-version_info = (0, 34, 8)
+version_info = (0, 35, 0)
 __version__ = ".".join(map(str, version_info))

pycromanager/acquisition/acq_constructor.py

@@ -1,8 +1,8 @@
-from pycromanager.headless import _PYMMCORES
 from pycromanager.acquisition.java_backend_acquisitions import JavaBackendAcquisition
 from pycromanager.acquisition.python_backend_acquisitions import PythonBackendAcquisition
 from pycromanager.acquisition.acquisition_superclass import Acquisition as PycromanagerAcquisitionBase
 from inspect import signature
+from mmpycorex import is_pymmcore_active
 
 # This is a convenience class that automatically selects the appropriate acquisition
 # type based on backend is running. It is subclassed from the base acquisition class
@@ -29,7 +29,7 @@ def __new__(cls,
                                      dict(signature(Acquisition.__init__).parameters.items())[arg_name].default)
                                      for arg_name in arg_names }
 
-        if _PYMMCORES:
+        if is_pymmcore_active():
             # Python backend detected, so create a python backend acquisition
             specific_arg_names = [k for k in signature(PythonBackendAcquisition.__init__).parameters.keys() if k != 'self']
             for name in specific_arg_names:

pycromanager/acq_future.py renamed to pycromanager/acquisition/acq_future.py

@@ -5,19 +5,16 @@
 import copy
 import types
 import numpy as np
-from typing import Union, List, Iterable
+from typing import List, Iterable
 import warnings
 from abc import ABCMeta, abstractmethod
 from docstring_inheritance import NumpyDocstringInheritanceMeta
 import queue
 import weakref
-from pycromanager.acq_future import AcqNotification, AcquisitionFuture
-import os
+from pycromanager.acquisition.acq_future import AcqNotification, AcquisitionFuture
 import threading
 from inspect import signature
-from typing import Generator
 from types import GeneratorType
-import time
 
 from queue import Queue
 from typing import Generator, Dict, Union

pycromanager/acquisition/acquisition_superclass.py

@@ -1,16 +1,13 @@
 """
 The Pycro-manager Acquisiton system
 """
-import json
 import logging
 import warnings
-import weakref
 
 import numpy as np
 import multiprocessing
 import threading
 from inspect import signature
-import time
 from pyjavaz import deserialize_array
 from pyjavaz import PullSocket, PushSocket, JavaObject, JavaClass
 from pyjavaz import DEFAULT_BRIDGE_PORT as DEFAULT_PORT
@@ -19,11 +16,10 @@
 
 from ndstorage import Dataset
 import os.path
-import queue
 from docstring_inheritance import NumpyDocstringInheritanceMeta
 from pycromanager.acquisition.acquisition_superclass import Acquisition
 import traceback
-from pycromanager.acq_future import AcqNotification, AcquisitionFuture
+from pycromanager.acquisition.acq_future import AcqNotification
 import json
 
 logger = logging.getLogger(__name__)
@@ -337,7 +333,7 @@ def __init__(
                     import napari
                 except:
                     raise Exception('Napari must be installed in order to use this feature')
-                from pycromanager.napari_util import start_napari_signalling
+                from pycromanager.acquisition.napari_util import start_napari_signalling
                 assert isinstance(napari_viewer, napari.Viewer), 'napari_viewer must be an instance of napari.Viewer'
                 self._napari_viewer = napari_viewer
                 start_napari_signalling(self._napari_viewer, self.get_dataset())

pycromanager/acquisition/java_backend_acquisitions.py

@@ -3,7 +3,7 @@
 from pycromanager.acquisition.acq_eng_py.main.AcqEngPy_Acquisition import Acquisition as pymmcore_Acquisition
 from pycromanager.acquisition.acquisition_superclass import _validate_acq_events, Acquisition
 from pycromanager.acquisition.acq_eng_py.main.acquisition_event import AcquisitionEvent
-from pycromanager.acq_future import AcqNotification
+from pycromanager.acquisition.acq_future import AcqNotification
 import threading
 from inspect import signature
 import traceback
@@ -110,7 +110,7 @@ def post_notification(notification):
                 import napari
             except:
                 raise Exception('Napari must be installed in order to use this feature')
-            from pycromanager.napari_util import start_napari_signalling
+            from pycromanager.acquisition.napari_util import start_napari_signalling
             assert isinstance(napari_viewer, napari.Viewer), 'napari_viewer must be an instance of napari.Viewer'
             self._napari_viewer = napari_viewer
             start_napari_signalling(self._napari_viewer, self.get_dataset())

pycromanager/napari_util.py renamed to pycromanager/acquisition/napari_util.py

@@ -1,137 +1,21 @@
-import logging
-import subprocess
-import platform
-import atexit
-import threading
-import types
-import os
-
+from mmpycorex import create_core_instance, terminate_core_instances
+from mmpycorex import Core
 from pycromanager.acquisition.acq_eng_py.internal.engine import Engine
-from pymmcore import CMMCore
+from pyjavaz import DEFAULT_BRIDGE_PORT
+import atexit
 import pymmcore
-from pyjavaz import DEFAULT_BRIDGE_PORT, server_terminated
-
-import re
-
-logger = logging.getLogger(__name__)
-
-class TaggedImage:
-
-    def __init__(self, tags, pix):
-        self.tags = tags
-        self.pix = pix
-
-def _camel_to_snake(name):
-    """
-    Convert camelCase string to snake_case
-    """
-    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
-    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
-
-def _create_pymmcore_instance():
-    """
-    Make a subclass of CMMCore with two differences:
-
-    1. All methods are converted to snake_case
-    2. add convenience methods to match the MMCoreJ API:
-    """
-
-    # Create a new dictionary for the class attributes
-    new_attributes = {}
-
-    # Iterate through the original attributes
-    for attr_name, attr_value in vars(CMMCore).items():
-        # If it's a dunder method, skip it (we don't want to override these)
-        if attr_name.startswith("__") and attr_name.endswith("__"):
-            continue
-        # If the attribute is callable (i.e., a method), convert its name to snake_case and add it
-        if callable(attr_value):
-            new_attr_name = _camel_to_snake(attr_name)
-            new_attributes[new_attr_name] = attr_value
-
-    # Create and return a new class that subclasses the original class and has the new attributes
-    clz = type(CMMCore.__name__ + "SnakeCase", (CMMCore,), new_attributes)
-
-    instance = clz()
-
-    def pop_next_tagged_image(self):
-        md = pymmcore.Metadata()
-        pix = self.pop_next_image_md(0, 0, md)
-        tags = {key: md.GetSingleTag(key).GetValue() for key in md.GetKeys()}
-        return TaggedImage(tags, pix)
-
-    def get_tagged_image(core, cam_index, camera, height, width, binning=None, pixel_type=None, roi_x_start=None,
-                         roi_y_start=None):
-        """
-        Different signature than the Java version because of difference in metadata handling in the swig layers
-        """
-        pix = core.get_image()
-        md = pymmcore.Metadata()
-        # most of the same tags from pop_next_tagged_image, which may not be the same as the MMCoreJ version of this function
-        tags = {'Camera': camera, 'Height': height, 'Width': width, 'PixelType': pixel_type,
-                'CameraChannelIndex': cam_index}
-        # Could optionally add these for completeness but there might be a performance hit
-        if binning is not None:
-            tags['Binning'] = binning
-        if roi_x_start is not None:
-            tags['ROI-X-start'] = roi_x_start
-        if roi_y_start is not None:
-            tags['ROI-Y-start'] = roi_y_start
-
-        return TaggedImage(tags, pix)
-
-    instance.get_tagged_image = types.MethodType(get_tagged_image, instance)
-    instance.pop_next_tagged_image = types.MethodType(pop_next_tagged_image, instance)
-
-    # attach TaggedImage class
-    instance.TaggedImage = TaggedImage
-    return instance
-
-
-_JAVA_HEADLESS_SUBPROCESSES = []
-_PYMMCORES = []
-
-def stop_headless(debug=False):
-    
-    for p in _JAVA_HEADLESS_SUBPROCESSES:
-        port = p.port
-        if debug:
-            logger.debug('Stopping headless process with pid {}'.format(p.pid))
-        p.terminate()
-        server_terminated(port)
-        if debug:
-            logger.debug('Waiting for process with pid {} to terminate'.format(p.pid))
-        p.wait()  # wait for process to terminate
-        if debug:
-            logger.debug('Process with pid {} terminated'.format(p.pid))
-    _JAVA_HEADLESS_SUBPROCESSES.clear()
-    if debug:
-        logger.debug('Stopping {} pymmcore instances'.format(len(_PYMMCORES)))
-    for c in _PYMMCORES:
-        if debug:
-            logger.debug('Stopping pymmcore instance')
-        c.unloadAllDevices()
-        if debug:
-            logger.debug('Unloaded all devices')
-        Engine.get_instance().shutdown()
-        if debug:
-            logger.debug('Engine shut down')
-    _PYMMCORES.clear()
-    if debug:
-        logger.debug('Headless stopped')
+import types
 
-# make sure any Java processes are cleaned up when Python exits
-atexit.register(stop_headless)
 
 def start_headless(
     mm_app_path: str, config_file: str=None, java_loc: str=None,
         python_backend=False, core_log_path: str='',
         buffer_size_mb: int=1024, max_memory_mb: int=2000,
         port: int=DEFAULT_BRIDGE_PORT, debug=False):
     """
-    Start a Java process that contains the neccessary libraries for pycro-manager to run,
-    so that it can be run independently of the Micro-Manager GUI/application. This calls
-    will create and initialize MMCore with the configuration file provided.
+    Start an instance of the Micro-Manager core and acquisition engine in headless mode. This can be
+    either a Python (i.e. pymmcore) or Java (i.e. MMCoreJ) backend. If a Python backend is used,
+    the core will be started in the same process.
 
     On windows plaforms, the Java Runtime Environment will be grabbed automatically
     as it is installed along with the Micro-Manager application.
@@ -161,68 +45,17 @@ def start_headless(
     debug : bool
         Print debug messages
     """
-
+    create_core_instance(
+        mm_app_path=mm_app_path, config_file=config_file, java_loc=java_loc,
+        python_backend=python_backend, core_log_path=core_log_path,
+        buffer_size_mb=buffer_size_mb, max_memory_mb=max_memory_mb,
+        port=port, debug=debug)
     if python_backend:
-        mmc = _create_pymmcore_instance()
-        mmc.set_device_adapter_search_paths([mm_app_path])
-        if config_file is not None and config_file != "":
-            mmc.load_system_configuration(config_file)
-        mmc.set_circular_buffer_memory_footprint(buffer_size_mb)
-        _PYMMCORES.append(mmc) # Store so it doesn't get garbage collected
-        Engine(mmc)
-    else:
-        classpath = mm_app_path + '/plugins/Micro-Manager/*'
-        if java_loc is None:
-            if platform.system() == "Windows":
-                # windows comes with its own JRE
-                java_loc = mm_app_path + "/jre/bin/javaw.exe"
-            else:
-                java_loc = "java"
-        if debug:
-            logger.debug(f'Java location: {java_loc}')
-            #print classpath
-            logger.debug(f'Classpath: {classpath}')
-            # print stuff in the classpath directory
-            logger.debug('Contents of classpath directory:')
-            for f in os.listdir(classpath.split('*')[0]):
-                logger.debug(f)
-
-        # This starts Java process and instantiates essential objects (core,
-        # acquisition engine, ZMQServer)
-        process = subprocess.Popen(
-                [
-                    java_loc,
-                    "-classpath",
-                    classpath,
-                    "-Dsun.java2d.dpiaware=false",
-                    f"-Xmx{max_memory_mb}m",
-                    # This is used by MM desktop app but breaks things on MacOS...Don't think its neccessary
-                    # "-XX:MaxDirectMemorySize=1000",
-                    "org.micromanager.remote.HeadlessLauncher",
-                    str(port),
-                    config_file if config_file is not None else '',
-                    str(buffer_size_mb),
-                    core_log_path,
-                ], cwd=mm_app_path, stdout=subprocess.PIPE
-            )
-        process.port = port
-        _JAVA_HEADLESS_SUBPROCESSES.append(process)
-
-        started = False
-        output = True
-        # Some drivers output various status messages which need to be skipped over to look for the STARTED token.
-        while output and not started:
-            output = process.stdout.readline()
-            started = "STARTED" in output.decode('utf-8')
-        if not started:
-            raise Exception('Error starting headless mode')
-        if debug:
-            logger.debug('Headless mode started')
-            def loggerFunction():
-                while process in _JAVA_HEADLESS_SUBPROCESSES:
-                    line = process.stdout.readline().decode('utf-8')
-                    if line.strip() != '':
-                        logger.debug(line)
-            threading.Thread(target=loggerFunction).start()
+        Engine(Core())
 
+def stop_headless(debug=False):
+    terminate_core_instances(debug=debug)
+    Engine.get_instance().shutdown()
 
+# make sure any Java processes are cleaned up when Python exits
+atexit.register(stop_headless)