diff --git a/guibot/config.py b/guibot/config.py index 0c53516c..73e89daa 100644 --- a/guibot/config.py +++ b/guibot/config.py @@ -642,6 +642,12 @@ def __init__(self, configure: bool = True, synchronize: bool = True) -> None: self.categories["type"] = "backend_types" self.algorithms["backend_types"] = ("cv", "dc") + # other attributes + from .imagelogger import ImageLogger + + self.imglog = ImageLogger() + self.imglog.log = self.log + if configure: self.__configure_backend() if synchronize: diff --git a/guibot/controller.py b/guibot/controller.py index 2bcd175b..bb212422 100644 --- a/guibot/controller.py +++ b/guibot/controller.py @@ -29,12 +29,13 @@ import re import time import logging - +import numpy import PIL.Image from tempfile import NamedTemporaryFile from . import inputmap from .config import GlobalConfig, LocalConfig +from .imagelogger import ImageLogger from .target import Image from .location import Location from .errors import * @@ -348,6 +349,8 @@ def keys_press(self, keys: list[str] | str) -> None: :param keys: characters or special keys depending on the backend (see :py:class:`inputmap.Key` for extensive list) """ + self.imglog.type = "keys" + self.imglog.log(30) time.sleep(self.params["control"]["delay_before_keys"]) # BUG: pressing multiple times the same key does not work? self.keys_toggle(keys, True) @@ -366,6 +369,35 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None: "Method is not available for this controller implementation" ) + def log(self, lvl: int) -> None: + """ + Log images with an arbitrary logging level. + + :param lvl: logging level for the message + """ + # below selected logging level + if lvl < self.imglog.logging_level: + self.imglog.clear() + return + + self.imglog.hotmaps += [numpy.array(self.capture_screen().pil_image)] + self.imglog.draw_locations( + [self.get_mouse_location().coords], + self.imglog.hotmaps[-1], + 30, + 0, + 0, + 0, + ) + name = "imglog%s-1control-%s.png" % ( + self.imglog.printable_step, + self.imglog.type, + ) + self.imglog.dump_hotmap(name, self.imglog.hotmaps[-1]) + + self.imglog.clear() + ImageLogger.step += 1 + class AutoPyController(Controller): """ @@ -528,6 +560,8 @@ def mouse_click( See base method for details. """ + self.imglog.type = "mouse" + self.imglog.log(30) button = self._mousemap.LEFT_BUTTON if button is None else button if modifiers is not None: self.keys_toggle(modifiers, True) @@ -579,6 +613,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None: See base method for details. """ + self.imglog.type = "keys" + self.imglog.log(30) time.sleep(self.params["control"]["delay_before_keys"]) if modifiers is not None: self.keys_toggle(modifiers, True) @@ -752,6 +788,8 @@ def mouse_click( See base method for details. """ + self.imglog.type = "mouse" + self.imglog.log(30) button = self._mousemap.LEFT_BUTTON if button is None else button if modifiers is not None: self.keys_toggle(modifiers, True) @@ -807,6 +845,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None: See base method for details. """ + self.imglog.type = "keys" + self.imglog.log(30) time.sleep(self.params["control"]["delay_before_keys"]) if modifiers is not None: self.keys_toggle(modifiers, True) @@ -971,6 +1011,8 @@ def mouse_click( See base method for details. """ + self.imglog.type = "mouse" + self.imglog.log(30) button = self._mousemap.LEFT_BUTTON if button is None else button if modifiers is not None: self.keys_toggle(modifiers, True) @@ -1033,6 +1075,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None: See base method for details. """ + self.imglog.type = "keys" + self.imglog.log(30) time.sleep(self.params["control"]["delay_before_keys"]) if modifiers is not None: self.keys_toggle(modifiers, True) @@ -1184,6 +1228,8 @@ def mouse_click( See base method for details. """ + self.imglog.type = "mouse" + self.imglog.log(30) button = self._mousemap.LEFT_BUTTON if button is None else button if modifiers is not None: self.keys_toggle(modifiers, True) @@ -1253,6 +1299,8 @@ def keys_type(self, text: list[str] | str, modifiers: list[str] = None) -> None: See base method for details. """ + self.imglog.type = "keys" + self.imglog.log(30) time.sleep(self.params["control"]["delay_before_keys"]) if modifiers is not None: self.keys_toggle(modifiers, True) diff --git a/guibot/finder.py b/guibot/finder.py index f3ff3d7d..aaeafe00 100644 --- a/guibot/finder.py +++ b/guibot/finder.py @@ -40,7 +40,6 @@ from .imagelogger import ImageLogger from .fileresolver import FileResolver from .errors import * -from .location import Location log = logging.getLogger("guibot.finder") @@ -360,10 +359,6 @@ def __init__(self, configure: bool = True, synchronize: bool = True) -> None: "hybrid", ] - # other attributes - self.imglog = ImageLogger() - self.imglog.log = self.log - # additional preparation (no synchronization available) if configure: self.__configure_backend(reset=True) @@ -1771,9 +1766,15 @@ def _project_features( ) return None else: - self._log_features( - 30, self.imglog.locations, self.imglog.hotmaps[-1], 3, 0, 0, 255 - ) + if 30 >= self.imglog.logging_level: + self.imglog.draw_locations( + self.imglog.locations, + self.imglog.hotmaps[-1], + 3, + 0, + 0, + 255, + ) return locations_in_haystack def _detect_features( @@ -1837,7 +1838,15 @@ def _detect_features( len(hkeypoints), ) hkp_locations = [hkp.pt for hkp in hkeypoints] - self._log_features(10, hkp_locations, self.imglog.hotmaps[-4], 3, 255, 0, 0) + if 10 >= self.imglog.logging_level: + self.imglog.draw_locations( + hkp_locations, + self.imglog.hotmaps[-4], + 3, + 255, + 0, + 0, + ) return (nkeypoints, ndescriptors, hkeypoints, hdescriptors) @@ -1958,7 +1967,15 @@ def symmetry_test(nmatches: list[Any], hmatches: list[Any]) -> list[Any]: # these matches are half the way to being good mhkp_locations = [mhkp.pt for mhkp in match_hkeypoints] - self._log_features(10, mhkp_locations, self.imglog.hotmaps[-3], 2, 255, 255, 0) + if 10 >= self.imglog.logging_level: + self.imglog.draw_locations( + mhkp_locations, + self.imglog.hotmaps[-3], + 2, + 255, + 255, + 0, + ) match_similarity = float(len(match_nkeypoints)) / float(len(nkeypoints)) # update the current achieved similarity if matching similarity is used: @@ -2035,7 +2052,15 @@ def _project_locations( if mask[i][0] == 1: true_matches.append(kp) tmhkp_locations = [tmhkp.pt for tmhkp in true_matches] - self._log_features(20, tmhkp_locations, self.imglog.hotmaps[-2], 1, 0, 255, 0) + if 20 >= self.imglog.logging_level: + self.imglog.draw_locations( + tmhkp_locations, + self.imglog.hotmaps[-2], + 1, + 0, + 255, + 0, + ) # calculate and project all point coordinates in the needle projected = [] @@ -2099,24 +2124,6 @@ def log(self, lvl: int) -> None: self.imglog.clear() ImageLogger.step += 1 - def _log_features( - self, - lvl: int, - locations: list[tuple[float, float]], - hotmap: "Matlike", - radius: int = 0, - r: int = 255, - g: int = 255, - b: int = 255, - ) -> None: - if lvl < self.imglog.logging_level: - return - import cv2 - - for loc in locations: - x, y = loc - cv2.circle(hotmap, (int(x), int(y)), radius, (r, g, b)) - class CascadeFinder(Finder): """ diff --git a/guibot/imagelogger.py b/guibot/imagelogger.py index f8ebc167..f9159354 100644 --- a/guibot/imagelogger.py +++ b/guibot/imagelogger.py @@ -69,6 +69,8 @@ def __init__(self) -> None: self.similarities = [] self.locations = [] + self.type = "find" + # sync these static methods with the general settings at each use ImageLogger.logging_level = GlobalConfig.image_logging_level # NOTE: the executing code decides when to clean this directory @@ -85,6 +87,17 @@ def get_printable_step(self) -> str: printable_step = property(fget=get_printable_step) + def log(self, _lvl: int) -> None: + """ + Log images with an arbitrary logging level. + + :param lvl: logging level for the message + """ + raise NotImplementedError( + "Each finder or controller that does image logging " + "has to implement this itself" + ) + def debug(self) -> None: """Log images with a DEBUG logging level.""" self.log(10) @@ -105,6 +118,31 @@ def critical(self) -> None: """Log images with a CRITICAL logging level.""" self.log(50) + def draw_locations( + self, + locations: list[tuple[float, float]], + canvas: "Matlike", + radius: int = 0, + r: int = 255, + g: int = 255, + b: int = 255, + ) -> None: + """ + Draw locations on a canvas image to visualize points of interest. + + :param locations: list of locations to draw + :param canvas: canvas image to draw on + :param radius: radius for the circled locations + :param r: red value for the color of the circles locations + :param g: green value for the color of the circles locations + :param b: blue value for the color of the circles locations + """ + import cv2 + + for loc in locations: + x, y = loc + cv2.circle(canvas, (int(x), int(y)), radius, (r, g, b)) + def dump_matched_images(self) -> None: """ Write file with the current needle and haystack. @@ -161,3 +199,4 @@ def clear(self) -> None: self.hotmaps = [] self.similarities = [] self.locations = [] + self.type = "find" diff --git a/guibot/location.py b/guibot/location.py index bdf92443..25ca05b1 100644 --- a/guibot/location.py +++ b/guibot/location.py @@ -64,3 +64,13 @@ def get_y(self) -> int: return self._ypos y = property(fget=get_y) + + def get_coords(self) -> tuple[int, int]: + """ + Getter for readonly attributes. + + :returns: tuple of (x, y) coordinates of the location + """ + return self._xpos, self._ypos + + coords = property(fget=get_coords) diff --git a/tests/test_controller.py b/tests/test_controller.py index 7798ed80..4272096d 100755 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -28,6 +28,7 @@ from guibot.region import Region from guibot.location import Location from guibot.config import GlobalConfig +from guibot.imagelogger import ImageLogger class ControllerTest(unittest.TestCase): @@ -53,6 +54,16 @@ def setUpClass(cls) -> None: cls._server = subprocess.Popen(["x11vnc", "-q", "-forever", "-display", ":99", "-rfbauth", passfile]) + # preserve values of static attributes + cls.prev_loglevel = GlobalConfig.image_logging_level + cls.prev_logpath = GlobalConfig.image_logging_destination + cls.prev_logwidth = GlobalConfig.image_logging_step_width + + cls.logpath = os.path.join(common_test.unittest_dir, 'tmp') + GlobalConfig.image_logging_level = 0 + GlobalConfig.image_logging_destination = cls.logpath + GlobalConfig.image_logging_step_width = 4 + @classmethod def tearDownClass(cls) -> None: # the VNC display controller is disabled on OS-es like Windows @@ -65,6 +76,10 @@ def tearDownClass(cls) -> None: if os.path.exists(vnc_config_dir): shutil.rmtree(vnc_config_dir) + GlobalConfig.image_logging_level = cls.prev_loglevel + GlobalConfig.image_logging_destination = cls.prev_logpath + GlobalConfig.image_logging_step_width = cls.prev_logwidth + def setUp(self) -> None: # gui test scripts self.script_app = os.path.join(common_test.unittest_dir, 'qt5_application.py') @@ -81,6 +96,10 @@ def setUp(self) -> None: self.textedit_quit_control = Location(65, 60) self.textedit_any_control = Location(65, 95) + # the image logger will recreate its logging destination + ImageLogger.step = 1 + ImageLogger.accumulate_logging = False + self.backends = [] if os.environ.get('DISABLE_AUTOPY', "0") == "0": self.backends += [AutoPyController()] @@ -133,6 +152,12 @@ def wait_end(self, subprocess_pipe: Any, timeout: int = 30) -> int: time.sleep(0.2) + def _verify_dumps(self, control_type: str) -> list[str]: + dumps = os.listdir(self.logpath) + self.assertEqual(len(dumps), 1) + self.assertRegex(dumps[0], rf"imglog\d\d\d\d-1control-{control_type}.png") + self.assertTrue(os.path.isfile(os.path.join(self.logpath, dumps[0]))) + def test_basic(self) -> None: """Check basic functionality for all display controller backends.""" for display in self.backends: @@ -264,6 +289,8 @@ def test_mouse_click(self) -> None: # single right button has context menu requiring extra care if button == mouse.RIGHT_BUTTON and count == 1: + # remove auxiliary dumps from first mouse click + shutil.rmtree(self.logpath) time.sleep(3) display.mouse_move(self.context_menu_close_control, smooth=False) display.mouse_click(mouse.LEFT_BUTTON) @@ -271,6 +298,9 @@ def test_mouse_click(self) -> None: self.assertEqual(0, self.wait_end(self.child_app)) self.child_app = None + self._verify_dumps("mouse") + shutil.rmtree(self.logpath) + @unittest.skipIf(os.environ.get('DISABLE_PYQT', "0") == "1", "PyQt disabled") def test_mouse_updown(self) -> None: """Check mouse up/down effect for all display controller backends.""" @@ -332,6 +362,9 @@ def test_keys_press(self) -> None: self.child_app = None + self._verify_dumps("keys") + shutil.rmtree(self.logpath) + @unittest.skipIf(os.environ.get('DISABLE_PYQT', "0") == "1", "PyQt disabled") def test_keys_type(self) -> None: """Check key type effect for all display controller backends.""" @@ -342,12 +375,17 @@ def test_keys_type(self) -> None: display.mouse_move(self.textedit_quit_control) display.mouse_click(display.mousemap.LEFT_BUTTON) + # remove auxiliary dumps from mouse click + shutil.rmtree(self.logpath) time.sleep(0.2) display.keys_type('quit', modifiers) self.assertEqual(0, self.wait_end(self.child_app)) self.child_app = None + self._verify_dumps("keys") + shutil.rmtree(self.logpath) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_finder.py b/tests/test_finder.py index 73f0c397..434345d9 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -74,9 +74,9 @@ def _get_matches_in(self, pattern: str, dumps: list[str]) -> list[str]: def _verify_and_get_dumps(self, count: int, index: int = 1, multistep: bool = False) -> list[str]: dumps = os.listdir(self.logpath) self.assertEqual(len(dumps), count) - steps = self._get_matches_in('imglog\d\d\d\d-.+', dumps) + steps = self._get_matches_in(r'imglog\d\d\d\d-.+', dumps) self.assertEqual(len(steps), len(dumps)) - first_steps = self._get_matches_in('imglog%04d-.+' % index, dumps) + first_steps = self._get_matches_in(r'imglog%04d-.+' % index, dumps) if not multistep: self.assertEqual(len(first_steps), len(steps)) else: @@ -118,7 +118,7 @@ def _verify_single_hotmap(self, dumps: list[str], backend) -> None: self.assertEqual(len(hotmaps), 1) self.assertIn('3hotmap', hotmaps[0]) # report achieved similarity in the end of the filename - self.assertRegex(hotmaps[0], ".*-\d\.\d+.*") + self.assertRegex(hotmaps[0], r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmaps[0]))) @unittest.skipIf(os.environ.get('DISABLE_OPENCV', "0") == "1", "OpenCV disabled") @@ -242,7 +242,7 @@ def test_contour_same(self) -> None: self.assertEqual(len(hotmaps), 3) self.assertIn('3hotmap', hotmaps[0]) # report achieved similarity in the end of the filename - self.assertRegex(hotmaps[0], ".*-\d\.\d+.*") + self.assertRegex(hotmaps[0], r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmaps[0]))) self.assertIn('3hotmap-1threshold', hotmaps[1]) self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmaps[1]))) @@ -276,7 +276,7 @@ def test_contour_nomatch(self) -> None: self.assertEqual(len(hotmaps), 3) self.assertIn('3hotmap', hotmaps[0]) # report achieved similarity in the end of the filename - self.assertRegex(hotmaps[0], ".*-\d\.\d+.*") + self.assertRegex(hotmaps[0], r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmaps[0]))) self.assertIn('3hotmap-1threshold', hotmaps[1]) self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmaps[1]))) @@ -318,7 +318,7 @@ def test_template_same(self) -> None: else: self.assertIn('3hotmap-1template', hotmap) # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) shutil.rmtree(self.logpath) @@ -352,7 +352,7 @@ def test_template_nomatch(self) -> None: else: self.assertIn('3hotmap-1template', hotmap) # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) shutil.rmtree(self.logpath) @@ -395,7 +395,7 @@ def test_template_multiple(self) -> None: else: self.assertIn('3hotmap-%stemplate' % i, hotmap) # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) @unittest.skipIf(os.environ.get('DISABLE_OPENCV', "0") == "1", "OpenCV disabled") @@ -434,7 +434,7 @@ def test_feature_same(self) -> None: self.assertEqual(len(hotmaps), 4) self.assertIn('3hotmap', hotmaps[0]) # report achieved similarity in the end of the filename - self.assertRegex(hotmaps[0], ".*-\d\.\d+.*") + self.assertRegex(hotmaps[0], r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmaps[0]))) self.assertIn('3hotmap-1detect', hotmaps[1]) self.assertIn('3hotmap-2match', hotmaps[2]) @@ -475,7 +475,7 @@ def test_feature_nomatch(self) -> None: self.assertEqual(len(hotmaps), 4) self.assertIn('3hotmap', hotmaps[0]) # report achieved similarity in the end of the filename - self.assertRegex(hotmaps[0], ".*-\d\.\d+.*") + self.assertRegex(hotmaps[0], r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmaps[0]))) self.assertIn('3hotmap-1detect', hotmaps[1]) self.assertIn('3hotmap-2match', hotmaps[2]) @@ -664,7 +664,7 @@ def test_text_same(self) -> None: self.assertIn('3hotmap-3ocr-%stext' % (j-2), hotmap) if j == 3 or j == 4: # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) shutil.rmtree(self.logpath) @@ -724,7 +724,7 @@ def test_text_nomatch(self) -> None: self.assertIn('3hotmap-3ocr-%stext' % (j-2), hotmap) if j == 3 or j == 4: # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) shutil.rmtree(self.logpath) @@ -830,7 +830,7 @@ def test_tempfeat_same(self) -> None: else: self.assertIn('%itemplate' % int((i - 1) / 2 + 1), hotmap) # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) shutil.rmtree(self.logpath) @@ -860,7 +860,7 @@ def test_tempfeat_nomatch(self) -> None: self.assertNotIn('template', hotmap) self.assertNotIn('feature', hotmap) # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) shutil.rmtree(self.logpath) @@ -890,7 +890,7 @@ def test_deep_same(self) -> None: if i == 0: self.assertIn('3hotmap', hotmap) # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") else: self.assertIn('%sf' % i, hotmap) self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) @@ -920,7 +920,7 @@ def test_deep_nomatch(self) -> None: if i == 0: self.assertIn('3hotmap', hotmap) # report achieved similarity in the end of the filename - self.assertRegex(hotmap, ".*-\d\.\d+.*") + self.assertRegex(hotmap, r".*-\d\.\d+.*") else: self.assertIn('%sf' % i, hotmap) self.assertTrue(os.path.isfile(os.path.join(self.logpath, hotmap))) diff --git a/tests/test_imagelogger.py b/tests/test_imagelogger.py index 2141f32a..c67af5f8 100644 --- a/tests/test_imagelogger.py +++ b/tests/test_imagelogger.py @@ -54,6 +54,11 @@ def tearDown(self) -> None: self._patch_mkdir.stop() return super().tearDown() + def test_abstract_log(self) -> None: + """Test the main log method requires implementation.""" + with self.assertRaises(NotImplementedError): + self.imglog.log(10) + def test_step_print(self) -> None: """Test the string representation of the current step.""" for i in range(1, 10):