From 6d90f72cb2132852fb348b46f107438f726e9f19 Mon Sep 17 00:00:00 2001 From: Kurt Godwin Date: Fri, 3 Mar 2023 13:07:17 -0500 Subject: [PATCH 1/5] added logger --- src/xdist/dsession.py | 14 ++++++++++- src/xdist/remote.py | 51 +++++++++++++++++++++++++++++++++++++++ src/xdist/workermanage.py | 2 ++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index a622b8bd..786989dc 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -10,7 +10,8 @@ LoadGroupScheduling, WorkStealingScheduling, ) - +import logging +import pickle from queue import Empty, Queue @@ -276,6 +277,17 @@ def worker_testreport(self, node, rep): self.config.hook.pytest_runtest_logreport(report=rep) self._handlefailures(rep) + def worker_runtest_logmessage(self, node, record): + record = pickle.loads(record) + for handler in logging.getLogger().handlers: + + if not True: #self.respect_handler_level: + process = True + else: + process = record.levelno >= handler.level + if process: + handler.handle(record) + def worker_runtest_protocol_complete(self, node, item_index, duration): """ Emitted when a node fires the 'runtest_protocol_complete' event, diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 2e83a8dc..12de3009 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -11,6 +11,9 @@ import os import time from typing import Any +import pickle +import logging +import copy import pytest from execnet.gateway_base import dumps, DumpError @@ -55,6 +58,43 @@ def worker_title(title): # changing the process name is very optional, no errors please pass +class RemoteMessageHandler(logging.Handler): + """ + This handler sends events to a queue. Typically, it would be used together + with a multiprocessing Queue to centralise logging to file in one process + (in a multi-process application), so as to avoid file write contention + between processes. + + This code is new in Python 3.2, but this class can be copy pasted into + user code for use with earlier Python versions. + """ + + def __init__(self, queue): + """ + Initialise an instance, using the passed queue. + """ + logging.Handler.__init__(self) + self.queue = queue + + def emit(self, record): + """ + Emit a record. + + Writes the LogRecord to the queue, preparing it for pickling first. + """ + try: + msg = self.format(record) + # bpo-35726: make copy of record to avoid affecting other handlers in the chain. + record = copy.copy(record) + record.message = msg + record.msg = msg + record.args = None + record.exc_info = None + record.exc_text = None + x = pickle.dumps(record) + self.queue.send_log(x) + except Exception as e: + self.handleError(record) class WorkerInteractor: SHUTDOWN_MARK = object() @@ -69,6 +109,14 @@ def __init__(self, config, channel): self.nextitem_index = None config.pluginmanager.register(self) + # pump cli messages back to master if a level is set + if config.option.log_cli_level: + rootlog = logging.getLogger() + myhandler = RemoteMessageHandler(self) + rootlog.addHandler(myhandler) + level = logging.getLevelName(config.option.log_cli_level) + myhandler.setLevel(level) + def _make_queue(self): return self.channel.gateway.execmodel.queue.Queue() @@ -76,6 +124,9 @@ def sendevent(self, name, **kwargs): self.log("sending", name, kwargs) self.channel.send((name, kwargs)) + def send_log(self, record): + self.sendevent("runtest_logmessage", record=record) + @pytest.hookimpl def pytest_internalerror(self, excrepr): formatted_error = str(excrepr) diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index fdd4109a..7e5b4b6e 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -362,6 +362,8 @@ def process_from_remote(self, eventcall): # noqa too complex self.notify_inproc(eventname, node=self, ids=kwargs["ids"]) elif eventname == "runtest_protocol_complete": self.notify_inproc(eventname, node=self, **kwargs) + elif eventname == "runtest_logmessage": + self.notify_inproc(eventname, node=self, **kwargs) elif eventname == "unscheduled": self.notify_inproc(eventname, node=self, **kwargs) elif eventname == "logwarning": From c8f5e19379487e4907c6927bf49a0dcadfa6e208 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 3 Mar 2023 18:35:50 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/xdist/dsession.py | 3 +-- src/xdist/remote.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 786989dc..12052994 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -280,8 +280,7 @@ def worker_testreport(self, node, rep): def worker_runtest_logmessage(self, node, record): record = pickle.loads(record) for handler in logging.getLogger().handlers: - - if not True: #self.respect_handler_level: + if not True: # self.respect_handler_level: process = True else: process = record.levelno >= handler.level diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 12de3009..0554bbe4 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -58,6 +58,7 @@ def worker_title(title): # changing the process name is very optional, no errors please pass + class RemoteMessageHandler(logging.Handler): """ This handler sends events to a queue. Typically, it would be used together @@ -93,9 +94,10 @@ def emit(self, record): record.exc_text = None x = pickle.dumps(record) self.queue.send_log(x) - except Exception as e: + except Exception: self.handleError(record) + class WorkerInteractor: SHUTDOWN_MARK = object() From 04c26f2bfc32f56a0f75d5ee3ccbd65dbea1fd9b Mon Sep 17 00:00:00 2001 From: Kurt Godwin Date: Thu, 28 Sep 2023 07:56:20 -0400 Subject: [PATCH 3/5] Updates based on comments fromi nicoddemus --- src/xdist/dsession.py | 5 +---- src/xdist/remote.py | 3 +++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 56cb7628..6e5725a4 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -287,10 +287,7 @@ def worker_testreport(self, node, rep): def worker_runtest_logmessage(self, node, record): record = pickle.loads(record) for handler in logging.getLogger().handlers: - if not True: # self.respect_handler_level: - process = True - else: - process = record.levelno >= handler.level + process = record.levelno >= handler.level if process: handler.handle(record) diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 7848787a..a6e83030 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -68,6 +68,9 @@ class RemoteMessageHandler(logging.Handler): This code is new in Python 3.2, but this class can be copy pasted into user code for use with earlier Python versions. + + Largely based on QueueHandler in handlers.py in cpython : + Source: https://github.com/python/cpython/blob/8f324b7ecd2df3036fab098c4c8ac185ac07b277/Lib/logging/handlers.py#L1412 """ def __init__(self, queue): From b12937cfde5b1ccd5bf8724e101c45fdd9196b52 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 11:57:16 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/xdist/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xdist/remote.py b/src/xdist/remote.py index a6e83030..8e12050e 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -69,7 +69,7 @@ class RemoteMessageHandler(logging.Handler): This code is new in Python 3.2, but this class can be copy pasted into user code for use with earlier Python versions. - Largely based on QueueHandler in handlers.py in cpython : + Largely based on QueueHandler in handlers.py in cpython : Source: https://github.com/python/cpython/blob/8f324b7ecd2df3036fab098c4c8ac185ac07b277/Lib/logging/handlers.py#L1412 """ From f4d4039efc3a4788c39b10b4da013682a2070c5c Mon Sep 17 00:00:00 2001 From: Kurt Godwin <48928663+kurt-cb@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:39:06 -0500 Subject: [PATCH 5/5] Update src/xdist/remote.py Co-authored-by: Bruno Oliveira --- src/xdist/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 8e12050e..496d0ff3 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -139,8 +139,8 @@ def sendevent(self, name, **kwargs): self.log("sending", name, kwargs) self.channel.send((name, kwargs)) - def send_log(self, record): - self.sendevent("runtest_logmessage", record=record) + def send_log(self, pickled_record): + self.sendevent("runtest_logmessage", pickled_record=pickled_record) @pytest.hookimpl def pytest_internalerror(self, excrepr):