Enable image logging also for display controllers

To draw the current mouse location while either clicking or while
indicating focus during typing we generalize previous location
drawing functionality to the image logger class and retrieve a
coordinates tuple from the location class.

The controllers will only dump image logs for mouse clicking and
key pressing and typing since mouse and key toggling are sometimes
called internally for some controllers and not for others. In order
to keep image logging expectations simple, it is best to only log
in these more externally used methods of each controller.

AutoPy controller changes are not covered in all cases until we can
recover more of this DC backend later on. Isolation tests are now
provided for the controller image logging where needed.

While at it, provide raw regular expressions for imglog dump testing.

guibot/config.py

@@ -642,6 +642,12 @@ def __init__(self, configure: bool = True, synchronize: bool = True) -> None:
         self.categories["type"] = "backend_types"
         self.algorithms["backend_types"] = ("cv", "dc")
 
+        # other attributes
+        from .imagelogger import ImageLogger
+
+        self.imglog = ImageLogger()
+        self.imglog.log = self.log
+
         if configure:
             self.__configure_backend()
         if synchronize:

guibot/controller.py

@@ -29,12 +29,13 @@
 import re
 import time
 import logging
-
+import numpy
 import PIL.Image
 from tempfile import NamedTemporaryFile
 
 from . import inputmap
 from .config import GlobalConfig, LocalConfig
+from .imagelogger import ImageLogger
 from .target import Image
 from .location import Location
 from .errors import *
@@ -348,6 +349,8 @@ def keys_press(self, keys: list[str] | str) -> None:
         :param keys: characters or special keys depending on the backend
                      (see :py:class:`inputmap.Key` for extensive list)
         """
+        self.imglog.type = "keys"
+        self.imglog.log(30)
         time.sleep(self.params["control"]["delay_before_keys"])
         # BUG: pressing multiple times the same key does not work?
         self.keys_toggle(keys, True)
@@ -366,6 +369,35 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None:
             "Method is not available for this controller implementation"
         )
 
+    def log(self, lvl: int) -> None:
+        """
+        Log images with an arbitrary logging level.
+
+        :param lvl: logging level for the message
+        """
+        # below selected logging level
+        if lvl < self.imglog.logging_level:
+            self.imglog.clear()
+            return
+
+        self.imglog.hotmaps += [numpy.array(self.capture_screen().pil_image)]
+        self.imglog.draw_locations(
+            [self.get_mouse_location().coords],
+            self.imglog.hotmaps[-1],
+            30,
+            0,
+            0,
+            0,
+        )
+        name = "imglog%s-1control-%s.png" % (
+            self.imglog.printable_step,
+            self.imglog.type,
+        )
+        self.imglog.dump_hotmap(name, self.imglog.hotmaps[-1])
+
+        self.imglog.clear()
+        ImageLogger.step += 1
+
 
 class AutoPyController(Controller):
     """
@@ -528,6 +560,8 @@ def mouse_click(
 
         See base method for details.
         """
+        self.imglog.type = "mouse"
+        self.imglog.log(30)
         button = self._mousemap.LEFT_BUTTON if button is None else button
         if modifiers is not None:
             self.keys_toggle(modifiers, True)
@@ -579,6 +613,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None:
 
         See base method for details.
         """
+        self.imglog.type = "keys"
+        self.imglog.log(30)
         time.sleep(self.params["control"]["delay_before_keys"])
         if modifiers is not None:
             self.keys_toggle(modifiers, True)
@@ -752,6 +788,8 @@ def mouse_click(
 
         See base method for details.
         """
+        self.imglog.type = "mouse"
+        self.imglog.log(30)
         button = self._mousemap.LEFT_BUTTON if button is None else button
         if modifiers is not None:
             self.keys_toggle(modifiers, True)
@@ -807,6 +845,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None:
 
         See base method for details.
         """
+        self.imglog.type = "keys"
+        self.imglog.log(30)
         time.sleep(self.params["control"]["delay_before_keys"])
         if modifiers is not None:
             self.keys_toggle(modifiers, True)
@@ -971,6 +1011,8 @@ def mouse_click(
 
         See base method for details.
         """
+        self.imglog.type = "mouse"
+        self.imglog.log(30)
         button = self._mousemap.LEFT_BUTTON if button is None else button
         if modifiers is not None:
             self.keys_toggle(modifiers, True)
@@ -1033,6 +1075,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None:
 
         See base method for details.
         """
+        self.imglog.type = "keys"
+        self.imglog.log(30)
         time.sleep(self.params["control"]["delay_before_keys"])
         if modifiers is not None:
             self.keys_toggle(modifiers, True)
@@ -1184,6 +1228,8 @@ def mouse_click(
 
         See base method for details.
         """
+        self.imglog.type = "mouse"
+        self.imglog.log(30)
         button = self._mousemap.LEFT_BUTTON if button is None else button
         if modifiers is not None:
             self.keys_toggle(modifiers, True)
@@ -1253,6 +1299,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None:
 
         See base method for details.
         """
+        self.imglog.type = "keys"
+        self.imglog.log(30)
         time.sleep(self.params["control"]["delay_before_keys"])
         if modifiers is not None:
             self.keys_toggle(modifiers, True)

guibot/finder.py

@@ -40,7 +40,6 @@
 from .imagelogger import ImageLogger
 from .fileresolver import FileResolver
 from .errors import *
-from .location import Location
 
 
 log = logging.getLogger("guibot.finder")
@@ -360,10 +359,6 @@ def __init__(self, configure: bool = True, synchronize: bool = True) -> None:
             "hybrid",
         ]
 
-        # other attributes
-        self.imglog = ImageLogger()
-        self.imglog.log = self.log
-
         # additional preparation (no synchronization available)
         if configure:
             self.__configure_backend(reset=True)
@@ -1771,9 +1766,15 @@ def _project_features(
             )
             return None
         else:
-            self._log_features(
-                30, self.imglog.locations, self.imglog.hotmaps[-1], 3, 0, 0, 255
-            )
+            if 30 >= self.imglog.logging_level:
+                self.imglog.draw_locations(
+                    self.imglog.locations,
+                    self.imglog.hotmaps[-1],
+                    3,
+                    0,
+                    0,
+                    255,
+                )
             return locations_in_haystack
 
     def _detect_features(
@@ -1837,7 +1838,15 @@ def _detect_features(
             len(hkeypoints),
         )
         hkp_locations = [hkp.pt for hkp in hkeypoints]
-        self._log_features(10, hkp_locations, self.imglog.hotmaps[-4], 3, 255, 0, 0)
+        if 10 >= self.imglog.logging_level:
+            self.imglog.draw_locations(
+                hkp_locations,
+                self.imglog.hotmaps[-4],
+                3,
+                255,
+                0,
+                0,
+            )
 
         return (nkeypoints, ndescriptors, hkeypoints, hdescriptors)
 
@@ -1958,7 +1967,15 @@ def symmetry_test(nmatches: list[Any], hmatches: list[Any]) -> list[Any]:
 
         # these matches are half the way to being good
         mhkp_locations = [mhkp.pt for mhkp in match_hkeypoints]
-        self._log_features(10, mhkp_locations, self.imglog.hotmaps[-3], 2, 255, 255, 0)
+        if 10 >= self.imglog.logging_level:
+            self.imglog.draw_locations(
+                mhkp_locations,
+                self.imglog.hotmaps[-3],
+                2,
+                255,
+                255,
+                0,
+            )
 
         match_similarity = float(len(match_nkeypoints)) / float(len(nkeypoints))
         # update the current achieved similarity if matching similarity is used:
@@ -2035,7 +2052,15 @@ def _project_locations(
             if mask[i][0] == 1:
                 true_matches.append(kp)
         tmhkp_locations = [tmhkp.pt for tmhkp in true_matches]
-        self._log_features(20, tmhkp_locations, self.imglog.hotmaps[-2], 1, 0, 255, 0)
+        if 20 >= self.imglog.logging_level:
+            self.imglog.draw_locations(
+                tmhkp_locations,
+                self.imglog.hotmaps[-2],
+                1,
+                0,
+                255,
+                0,
+            )
 
         # calculate and project all point coordinates in the needle
         projected = []
@@ -2099,24 +2124,6 @@ def log(self, lvl: int) -> None:
         self.imglog.clear()
         ImageLogger.step += 1
 
-    def _log_features(
-        self,
-        lvl: int,
-        locations: list[tuple[float, float]],
-        hotmap: "Matlike",
-        radius: int = 0,
-        r: int = 255,
-        g: int = 255,
-        b: int = 255,
-    ) -> None:
-        if lvl < self.imglog.logging_level:
-            return
-        import cv2
-
-        for loc in locations:
-            x, y = loc
-            cv2.circle(hotmap, (int(x), int(y)), radius, (r, g, b))
-
 
 class CascadeFinder(Finder):
     """

guibot/imagelogger.py

@@ -69,6 +69,8 @@ def __init__(self) -> None:
         self.similarities = []
         self.locations = []
 
+        self.type = "find"
+
         # sync these static methods with the general settings at each use
         ImageLogger.logging_level = GlobalConfig.image_logging_level
         # NOTE: the executing code decides when to clean this directory
@@ -85,6 +87,17 @@ def get_printable_step(self) -> str:
 
     printable_step = property(fget=get_printable_step)
 
+    def log(self, _lvl: int) -> None:
+        """
+        Log images with an arbitrary logging level.
+
+        :param lvl: logging level for the message
+        """
+        raise NotImplementedError(
+            "Each finder or controller that does image logging "
+            "has to implement this itself"
+        )
+
     def debug(self) -> None:
         """Log images with a DEBUG logging level."""
         self.log(10)
@@ -105,6 +118,31 @@ def critical(self) -> None:
         """Log images with a CRITICAL logging level."""
         self.log(50)
 
+    def draw_locations(
+        self,
+        locations: list[tuple[float, float]],
+        canvas: "Matlike",
+        radius: int = 0,
+        r: int = 255,
+        g: int = 255,
+        b: int = 255,
+    ) -> None:
+        """
+        Draw locations on a canvas image to visualize points of interest.
+
+        :param locations: list of locations to draw
+        :param canvas: canvas image to draw on
+        :param radius: radius for the circled locations
+        :param r: red value for the color of the circles locations
+        :param g: green value for the color of the circles locations
+        :param b: blue value for the color of the circles locations
+        """
+        import cv2
+
+        for loc in locations:
+            x, y = loc
+            cv2.circle(canvas, (int(x), int(y)), radius, (r, g, b))
+
     def dump_matched_images(self) -> None:
         """
         Write file with the current needle and haystack.
@@ -161,3 +199,4 @@ def clear(self) -> None:
         self.hotmaps = []
         self.similarities = []
         self.locations = []
+        self.type = "find"

guibot/location.py

@@ -64,3 +64,13 @@ def get_y(self) -> int:
         return self._ypos
 
     y = property(fget=get_y)
+
+    def get_coords(self) -> tuple[int, int]:
+        """
+        Getter for readonly attributes.
+
+        :returns: tuple of (x, y) coordinates of the location
+        """
+        return self._xpos, self._ypos
+
+    coords = property(fget=get_coords)