From 2b3d0690a1c1093fafe537c51edbfc823ed96f82 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 10 May 2023 17:49:41 +0200 Subject: [PATCH 01/20] Adding stage setting schema --- pyproject.toml | 3 +- setup.cfg | 9 ++++- src/pytest_fluent/data/default.stage.json | 6 +++ src/pytest_fluent/data/schema.stage.json | 45 +++++++++++++++++++++ tests/test_schema.py | 49 +++++++++++++++++++++++ 5 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 src/pytest_fluent/data/default.stage.json create mode 100644 src/pytest_fluent/data/schema.stage.json create mode 100644 tests/test_schema.py diff --git a/pyproject.toml b/pyproject.toml index e1ce067..b88cc49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,8 @@ dependencies = [ "pytest>=7.0.0", "msgpack", "six", - "fluent-logger" + "fluent-logger", + "jsonschema", ] dynamic = ["version"] diff --git a/setup.cfg b/setup.cfg index d5524b1..dec14f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] copyright = Copyright © Rohde & Schwarz GmbH & Co. KG 2022 -platform = +platform = Unix Linux Windows @@ -9,10 +9,15 @@ platform = package_dir= =src packages=find: +include_package_data = True [options.entry_points] pytest11 = fluent-logging = pytest_fluent.plugin [options.packages.find] -where = src \ No newline at end of file +where = src + +[options.package_data] +pytest_fluent.data = + *.json \ No newline at end of file diff --git a/src/pytest_fluent/data/default.stage.json b/src/pytest_fluent/data/default.stage.json new file mode 100644 index 0000000..4fc34b5 --- /dev/null +++ b/src/pytest_fluent/data/default.stage.json @@ -0,0 +1,6 @@ +{ + "all": { + "tag": "", + "label": "" + } +} \ No newline at end of file diff --git a/src/pytest_fluent/data/schema.stage.json b/src/pytest_fluent/data/schema.stage.json new file mode 100644 index 0000000..9ee0d94 --- /dev/null +++ b/src/pytest_fluent/data/schema.stage.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "patternProperties": { + "all|pytest_runtest_logstart|pytest_runtest_logreport|pytest_runtest_logfinish": { + "$ref": "#/definitions/AdditionalProperties" + } + }, + "additionalProperties": false, + "definitions": { + "RegexString": { + "type": "string", + "regex": "(\\$)?([<\\{])?[\\w_.-]+([>\\}])?" + }, + "AdditionalProperties": { + "type": "object", + "additionalProperties": false, + "properties": { + "tag": { + "$ref": "#/definitions/RegexString", + "default": "" + }, + "label": { + "$ref": "#/definitions/RegexString", + "default": "" + }, + "replace": { + "type": "object", + "patternProperties": { + "status|stage|sessionId|testId|name|when|duration": { + "$ref": "#/definitions/RegexString" + } + } + }, + "add": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/RegexString" + } + } + }, + "required": [] + } + } +} \ No newline at end of file diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..945de6d --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,49 @@ +import json + +import jsonschema +import pytest + +try: + from importlib.resources import files +except ImportError: + from importlib_resources import files + +pytest_fluent_resources = files("pytest_fluent") +schema_file = pytest_fluent_resources / "data" / "schema.stage.json" +default_file = pytest_fluent_resources / "data" / "default.stage.json" + + +@pytest.fixture +def default() -> dict: + with open(default_file, "r") as fp: + default = json.load(fp) + return default + + +@pytest.fixture +def schema() -> dict: + with open(schema_file, "r") as fp: + schema = json.load(fp) + return schema + + +def test_default_compliance(default, schema): + jsonschema.validate(default, schema) + default["pytest_runtest_logstart"] = {"replace": {"status": "state"}} + jsonschema.validate(default, schema) + default["pytest_runtest_logstart"] = {"add": {"my_field": "${MY_FIELD}"}} + jsonschema.validate(default, schema) + default["pytest_runtest_logstart"] = {"tag": "", "label": ""} + jsonschema.validate(default, schema) + + +def test_default_compliance_fail(default, schema): + default["any"] = {"tag": "x", "label": "y", "replace": {"where": "test"}} + + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(default, schema) + + default["any"] = {"replace": {"where": "test"}} + + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(default, schema) From 1ca063378df5a55ff0427f9f036cd496667d4036 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 10 May 2023 17:50:28 +0200 Subject: [PATCH 02/20] Adding setting file loader to argument parser + test --- pyproject.toml | 1 + src/pytest_fluent/SettingFileLoaderAction.py | 37 ++++++++++++++ src/pytest_fluent/plugin.py | 18 +++++++ tests/data/default.json | 6 +++ tests/data/default.yaml | 3 ++ tests/test_parser.py | 51 ++++++++++++++++++++ 6 files changed, 116 insertions(+) create mode 100644 src/pytest_fluent/SettingFileLoaderAction.py create mode 100644 tests/data/default.json create mode 100644 tests/data/default.yaml create mode 100644 tests/test_parser.py diff --git a/pyproject.toml b/pyproject.toml index b88cc49..f5f0023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "six", "fluent-logger", "jsonschema", + "ruamel.yaml" ] dynamic = ["version"] diff --git a/src/pytest_fluent/SettingFileLoaderAction.py b/src/pytest_fluent/SettingFileLoaderAction.py new file mode 100644 index 0000000..f1bc827 --- /dev/null +++ b/src/pytest_fluent/SettingFileLoaderAction.py @@ -0,0 +1,37 @@ +"""Package in order to train and test complex compression.""" +import argparse +import json +from ruamel.yaml import YAML +import os + +import jsonschema + + +class SettingFileLoaderAction(argparse.Action): + """Custom action for loading JSON/YAML configuration.""" + + def __call__(self, parser, args, values, option_string=None): + """Implementing call.""" + parameter = values + if parameter.endswith(".json"): + pickle = json + elif parameter.endswith(".yaml"): + pickle = YAML() + + def loads(fp): + return pickle.load(fp.read()) + + pickle.__setattr__("loads", loads) + else: + raise ValueError("File type not supported.") + if os.path.exists(parameter): + with open(parameter) as fid: + parameter = pickle.load(fid) + else: + parameter = pickle.loads(parameter) + with open( + os.path.join(os.path.dirname(__file__), "data", "schema.stage.json") + ) as fid: + schema = json.load(fid) + jsonschema.validate(instance=parameter, schema=schema) + setattr(args, self.dest, parameter) diff --git a/src/pytest_fluent/plugin.py b/src/pytest_fluent/plugin.py index 2807572..b47fb48 100644 --- a/src/pytest_fluent/plugin.py +++ b/src/pytest_fluent/plugin.py @@ -4,6 +4,7 @@ import time import typing import uuid +import os from io import BytesIO import msgpack @@ -11,6 +12,8 @@ from fluent import event, sender from fluent.handler import FluentHandler, FluentRecordFormatter +from .SettingFileLoaderAction import SettingFileLoaderAction + from .additional_information import ( get_additional_session_information, get_additional_test_information, @@ -122,6 +125,14 @@ def pytest_runtest_setup(self, item: pytest.Item): if not self.config.getoption("collectonly"): pass + def pytest_runtest_teardown(self, item: pytest.Item, nextitem: pytest.Item): + """Custom hook for test teardown.""" + set_stage("testcase") + docstring = get_test_docstring(item) + item.stash[DOCSTRING_STASHKEY] = docstring + if not self.config.getoption("collectonly"): + pass + def pytest_runtest_call(self, item: pytest.Item): """Custom hook for test call.""" set_stage("testcase") @@ -241,6 +252,13 @@ def pytest_addoption(parser): action="store_true", help="Add test docstrings to the testcase call messages.", ) + group.addoption( + "--stage-settings", + type=str, + default=os.path.join(os.path.dirname(__file__), "data", "default.stage.json"), + action=SettingFileLoaderAction, + help="Stage setting description JSON or YAML file path or string object.", + ) def pytest_configure(config): diff --git a/tests/data/default.json b/tests/data/default.json new file mode 100644 index 0000000..4fc34b5 --- /dev/null +++ b/tests/data/default.json @@ -0,0 +1,6 @@ +{ + "all": { + "tag": "", + "label": "" + } +} \ No newline at end of file diff --git a/tests/data/default.yaml b/tests/data/default.yaml new file mode 100644 index 0000000..8062978 --- /dev/null +++ b/tests/data/default.yaml @@ -0,0 +1,3 @@ +all: + tag: "" + label: "" diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..8768749 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,51 @@ +import argparse +import os + +import pytest +from pytest_fluent.SettingFileLoaderAction import SettingFileLoaderAction + + +parser = argparse.ArgumentParser() +parser.add_argument( + "--stage-settings", + type=str, + dest="settings", + default=os.path.join(os.path.dirname(__file__), "data", "default.stage.json"), + action=SettingFileLoaderAction, + help="Stage setting description JSON or YAML file path or string object.", +) + + +@pytest.fixture +def default() -> dict: + return {"all": {"tag": "", "label": ""}} + + +def test_json_file(default): + args = parser.parse_args( + [ + "--stage-settings", + os.path.join(os.path.dirname(__file__), "data", "default.json"), + ] + ) + assert args.settings == default + + +def test_yaml_file(default): + args = parser.parse_args( + [ + "--stage-settings", + os.path.join(os.path.dirname(__file__), "data", "default.yaml"), + ] + ) + assert args.settings == default + + +def test_xml_error(): + with pytest.raises(ValueError): + args = parser.parse_args( + [ + "--stage-settings", + os.path.join(os.path.dirname(__file__), "data", "default.xml"), + ] + ) From 5d669bb6836533fd02377005a608fabdc00557f1 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Fri, 26 May 2023 16:13:39 +0200 Subject: [PATCH 03/20] Implementing content_patcher + user setting loader + custom fluent Event class --- pyproject.toml | 2 +- src/pytest_fluent/SettingFileLoaderAction.py | 37 --- src/pytest_fluent/content_patcher.py | 170 +++++++++++ src/pytest_fluent/data/default.stage.json | 4 +- src/pytest_fluent/data/schema.stage.json | 8 +- src/pytest_fluent/event.py | 47 +++ src/pytest_fluent/plugin.py | 138 +++++---- .../setting_file_loader_action.py | 53 ++++ tests/test_content_patcher.py | 272 ++++++++++++++++++ tests/test_parser.py | 2 +- 10 files changed, 632 insertions(+), 101 deletions(-) delete mode 100644 src/pytest_fluent/SettingFileLoaderAction.py create mode 100644 src/pytest_fluent/content_patcher.py create mode 100644 src/pytest_fluent/event.py create mode 100644 src/pytest_fluent/setting_file_loader_action.py create mode 100644 tests/test_content_patcher.py diff --git a/pyproject.toml b/pyproject.toml index f5f0023..c69cf4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "six", "fluent-logger", "jsonschema", - "ruamel.yaml" + "ruamel.yaml", ] dynamic = ["version"] diff --git a/src/pytest_fluent/SettingFileLoaderAction.py b/src/pytest_fluent/SettingFileLoaderAction.py deleted file mode 100644 index f1bc827..0000000 --- a/src/pytest_fluent/SettingFileLoaderAction.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Package in order to train and test complex compression.""" -import argparse -import json -from ruamel.yaml import YAML -import os - -import jsonschema - - -class SettingFileLoaderAction(argparse.Action): - """Custom action for loading JSON/YAML configuration.""" - - def __call__(self, parser, args, values, option_string=None): - """Implementing call.""" - parameter = values - if parameter.endswith(".json"): - pickle = json - elif parameter.endswith(".yaml"): - pickle = YAML() - - def loads(fp): - return pickle.load(fp.read()) - - pickle.__setattr__("loads", loads) - else: - raise ValueError("File type not supported.") - if os.path.exists(parameter): - with open(parameter) as fid: - parameter = pickle.load(fid) - else: - parameter = pickle.loads(parameter) - with open( - os.path.join(os.path.dirname(__file__), "data", "schema.stage.json") - ) as fid: - schema = json.load(fid) - jsonschema.validate(instance=parameter, schema=schema) - setattr(args, self.dest, parameter) diff --git a/src/pytest_fluent/content_patcher.py b/src/pytest_fluent/content_patcher.py new file mode 100644 index 0000000..f27b333 --- /dev/null +++ b/src/pytest_fluent/content_patcher.py @@ -0,0 +1,170 @@ +"""Patch content according to settings.""" +import argparse +import enum +import inspect +import os +import re +import typing + + +class _ContentType(enum.Enum): + ENV = 0 + ARGS = 1 + + +class ContentPatcher: + """Patch the transmission content according to the user settings.""" + + def __init__( + self, + user_settings: dict, + args_settings: argparse.Namespace, + stage_names: typing.List[str], + ) -> None: + """Initialize content patcher.""" + self._args_settings: argparse.Namespace = args_settings + self._user_settings: dict = self._stage_settings(user_settings, stage_names) + + def _stage_settings( + self, user_settings: dict, stage_names: typing.List[str] + ) -> dict: + """Prepare stage settings for faster online lookup. + + Args: + user_settings (dict): User settings from JSON or YAML file. + stage_names (typing.List[str]): Used stage names by Pytest-fluent plugin. + + Returns: + dict: Returns patched user settings. + """ + patched = {} + all_settings = {} + for key, value in user_settings.get("all", {}).items(): + if isinstance(value, dict): + for subkey, subvalue in value.items(): + value[subkey] = self._get_env_or_args(subvalue) + else: + value = self._get_env_or_args(value) + all_settings.update({key: value}) + for stage_name in stage_names: + patched.update({stage_name: all_settings.copy()}) + stage_info = user_settings.get(stage_name, {}) + for key, value in stage_info.items(): + if isinstance(value, dict): + for subkey, subvalue in value.items(): + value[subkey] = self._get_env_or_args(subvalue) + else: + value = self._get_env_or_args(value) + patched[stage_name].update({key: value}) + return patched + + @property + def user_settings(self) -> dict: + """Retrieve processed user settings. + + Returns: + dict: Dictionary of user settings. + """ + return self._user_settings + + def get_tag_and_label( + self, stage_name: typing.Optional[str] = None + ) -> typing.Tuple[str, str]: + """Return the tag for the corresponding stage. + + Args: + stage_name (typing.Optional[str], optional): Current stage. Defaults to None. + + Returns: + typing.Tuple[str, str]: Tag string, Label string. + """ + if stage_name is None: + stage_name = inspect.stack()[1][3] + stage_info = self._user_settings.get(stage_name, {}) + return stage_info["tag"], stage_info["label"] + + def patch(self, content: dict, stage_name: typing.Optional[str] = None) -> dict: + """Patch the content with the provided settings for each stage. + + Args: + content (dict): Structured data for transmission. + stage_name (typing.Optional[str], optional): Calling stage name. + Defaults to None. + + Returns: + str: Patch dictionary with the user provided stage settings. + """ + if stage_name is None: + stage_name = inspect.stack()[1][3] + + stage_info = self._user_settings.get(stage_name, {}) + if not stage_info: + return content + return self._patch_stage_content(content, stage_info) + + @staticmethod + def _patch_stage_content(stage_content: dict, user_settings: dict) -> dict: + stage_content_patched = stage_content.copy() + stage_content_patched["tag"] = user_settings["tag"] + stage_content_patched["label"] = user_settings["label"] + if "replace" in user_settings: + for key, value in user_settings["replace"].items(): + if key in stage_content_patched: + tmp = stage_content_patched[key] + stage_content_patched[value] = tmp + del stage_content_patched[key] + to_add = user_settings.get("add", {}) + stage_content_patched.update(to_add) + to_drop = user_settings.get("drop", []) + for key in to_drop: + del user_settings[key] + return stage_content_patched + + def _get_env_or_args(self, value: str) -> str: + reference = self._is_reference_string(value) + if reference == _ContentType.ENV: + return self._get_env_content(value) + elif reference == _ContentType.ARGS: + return self._get_args_content(value) + else: + return value + + @staticmethod + def _is_reference_string(value: str) -> typing.Optional[_ContentType]: + if re.match(r"(\$)?({)([\w_]+)(})", value): + return _ContentType.ENV + elif re.match(r"(<)([\w-]+)(>)", value): + return _ContentType.ARGS + return None + + @staticmethod + def _get_env_content(value: str) -> str: + """Check if string relates to ENV variable and return that value. + + Args: + value (str): String providing ENV reference e.g. ${USE_ENV} + + Returns: + str: String with ENV content or empty string. + """ + env_match = re.findall(r"\$({)?([\w_]+)(})?", value) + if not env_match: + return "" + env_value = os.getenv(env_match[0][1], "") + return env_value + + def _get_args_content(self, value: str) -> str: + """Check if string relates to CLI argument variable and return that value. + + Args: + value (str): String providing argument reference e.g. + + Returns: + str: String with argument content or empty string. + """ + args_match = re.findall(r"^<([\w-]+)>$", value) + if not args_match: + return "" + args_match = args_match[0].replace("-", "_") + args_value = getattr(self._args_settings, args_match, "") + return args_value diff --git a/src/pytest_fluent/data/default.stage.json b/src/pytest_fluent/data/default.stage.json index 4fc34b5..1e8f941 100644 --- a/src/pytest_fluent/data/default.stage.json +++ b/src/pytest_fluent/data/default.stage.json @@ -1,6 +1,6 @@ { "all": { - "tag": "", - "label": "" + "tag": "", + "label": "" } } \ No newline at end of file diff --git a/src/pytest_fluent/data/schema.stage.json b/src/pytest_fluent/data/schema.stage.json index 9ee0d94..44a67fa 100644 --- a/src/pytest_fluent/data/schema.stage.json +++ b/src/pytest_fluent/data/schema.stage.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "patternProperties": { - "all|pytest_runtest_logstart|pytest_runtest_logreport|pytest_runtest_logfinish": { + "all|pytest_runtest_logstart|pytest_runtest_logreport|pytest_runtest_logfinish|logging": { "$ref": "#/definitions/AdditionalProperties" } }, @@ -37,6 +37,12 @@ "additionalProperties": { "$ref": "#/definitions/RegexString" } + }, + "drop": { + "type": "array", + "items": { + "$ref": "#/definitions/RegexString" + } } }, "required": [] diff --git a/src/pytest_fluent/event.py b/src/pytest_fluent/event.py new file mode 100644 index 0000000..2a1cb9d --- /dev/null +++ b/src/pytest_fluent/event.py @@ -0,0 +1,47 @@ +"""Custom Event class.""" + +import logging +import time +import typing + +from fluent import sender + +LOGGER = logging.getLogger(__package__) + + +class Event: + """Customized Event class for sending different tags. + + Args: + host (str): Host name of the Fluent instance. Defaults to "localhost". + port (int): Port of the Fluent instance. Defaults to 24224. + """ + + def __init__( + self, + tags: typing.List[str], + host: str = "localhost", + port: int = 24224, + **kwargs, + ) -> None: + """Initialize custom event class.""" + self.senders = { + tag: sender.FluentSender(tag=tag, host=host, port=port, **kwargs) + for tag in tags + } + + def __call__(self, tag: str, label: str, data: dict, **kwargs): + """Send a new event. + + Args: + tag (str): Fluent tag. + label (str): Fluent label. + data (dict): Data to transmit as dictionary. + """ + assert isinstance(data, dict), "data must be a dict" + sender_ = self.senders.get(tag) + if not sender_: + LOGGER.warning(f"Could not retrieve fluent instance for tag {tag}") + timestamp = kwargs.get("time", int(time.time())) + if not sender_.emit_with_time(label, timestamp, data): + LOGGER.warning(f"Could not send data via fluent for tag {tag}: {data}") diff --git a/src/pytest_fluent/plugin.py b/src/pytest_fluent/plugin.py index b47fb48..da9fcf3 100644 --- a/src/pytest_fluent/plugin.py +++ b/src/pytest_fluent/plugin.py @@ -1,23 +1,26 @@ """pytest-fluent-logging plugin definition.""" import logging +import os import textwrap import time import typing import uuid -import os from io import BytesIO import msgpack import pytest -from fluent import event, sender from fluent.handler import FluentHandler, FluentRecordFormatter -from .SettingFileLoaderAction import SettingFileLoaderAction - from .additional_information import ( get_additional_session_information, get_additional_test_information, ) +from .content_patcher import ContentPatcher +from .event import Event +from .setting_file_loader_action import ( + SettingFileLoaderAction, + load_and_check_settings_file, +) from .test_report import LogReport ##################################################### @@ -41,24 +44,32 @@ def __init__(self, config): self._label = config.getoption("--fluentd-label") self._extend_logging = config.getoption("--extend-logging") self._add_docstrings = config.getoption("--add-docstrings") + stage_names = [method for method in dir(self) if method.startswith("pytest_")] + stage_names.append("logging") + self._content_patcher = ContentPatcher( + user_settings=config.getoption("--stage-settings"), + args_settings=config.option, + stage_names=stage_names, + ) + tags = [] + for value in self._content_patcher.user_settings.values(): + tag = value.get("tag") + if not tag: + continue + tags.append(tag) + tags = set(tags) + self._event = Event( + tags, self._host, self._port, buffer_overflow_handler=overflow_handler + ) self._log_reporter = LogReport(self.config) - self._setup_fluent_sender() - self._patch_logging() - - def _setup_fluent_sender(self): - if self._host is None: - sender.setup(self._tag, buffer_overflow_handler=overflow_handler) - else: - sender.setup( - self._tag, - host=self._host, - port=self._port, - buffer_overflow_handler=overflow_handler, - ) + self._patch_logging( + f"{self._content_patcher.user_settings['logging']['tag']}." + f"{self._content_patcher.user_settings['logging']['label']}" + ) - def _patch_logging(self): + def _patch_logging(self, tag: str): if self._extend_logging: - extend_loggers(self._host, self._port, self._tag) + extend_loggers(self._host, self._port, tag) def _set_session_uid( self, id: typing.Optional[typing.Union[str, uuid.UUID]] = None @@ -99,8 +110,10 @@ def pytest_sessionstart(self): "stage": "session", "sessionId": self.session_uid, } + data = self._content_patcher.patch(data) data.update(get_additional_session_information()) - event.Event(self._label, data) + tag, label = self._content_patcher.get_tag_and_label() + self._event(tag, label, data) def pytest_runtest_logstart(self, nodeid: str, location: typing.Tuple[int, str]): """Custom hook for test start.""" @@ -114,8 +127,10 @@ def pytest_runtest_logstart(self, nodeid: str, location: typing.Tuple[int, str]) "testId": self.test_uid, "name": nodeid, } + data = self._content_patcher.patch(data) data.update(get_additional_test_information()) - event.Event(self._label, data) + tag, label = self._content_patcher.get_tag_and_label() + self._event(tag, label, data) def pytest_runtest_setup(self, item: pytest.Item): """Custom hook for test setup.""" @@ -141,6 +156,7 @@ def pytest_runtest_call(self, item: pytest.Item): @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(self, item: pytest.Item, call): + """Custom hook for make report.""" report = (yield).get_result() docstring = item.stash.get(DOCSTRING_STASHKEY, None) report.stash = {} @@ -150,10 +166,10 @@ def pytest_runtest_logreport(self, report: pytest.TestReport): """Custom hook for logging results.""" set_stage("testcase") if not self.config.getoption("collectonly"): - result_data = self._log_reporter(report) - if not result_data: + data = self._log_reporter(report) + if not data: return - result_data.update( + data.update( { "stage": "testcase", "when": report.when, @@ -164,8 +180,10 @@ def pytest_runtest_logreport(self, report: pytest.TestReport): if self._add_docstrings: docstring = report.stash.get(DOCSTRING_KEY, None) if docstring: - result_data.update({"docstring": docstring}) - event.Event(self._label, result_data) + data.update({"docstring": docstring}) + data = self._content_patcher.patch(data) + tag, label = self._content_patcher.get_tag_and_label() + self._event(tag, label, data) def pytest_runtest_logfinish( self, @@ -175,16 +193,16 @@ def pytest_runtest_logfinish( """Custom hook for test end.""" set_stage("testcase") if not self.config.getoption("collectonly"): - event.Event( - self._label, - { - "status": "finish", - "stage": "testcase", - "sessionId": self.session_uid, - "testId": self.test_uid, - "name": nodeid, - }, - ) + data = { + "status": "finish", + "stage": "testcase", + "sessionId": self.session_uid, + "testId": self.test_uid, + "name": nodeid, + } + data = self._content_patcher.patch(data) + tag, label = self._content_patcher.get_tag_and_label() + self._event(tag, label, data) def pytest_sessionfinish( self, @@ -194,19 +212,19 @@ def pytest_sessionfinish( """Custom hook for session end.""" set_stage("session") if not self.config.getoption("collectonly"): - event.Event( - self._label, - { - "status": "finish", - "duration": time.time() - self._session_start_time, - "stage": "session", - "sessionId": self.session_uid, - }, - ) + data = { + "status": "finish", + "duration": time.time() - self._session_start_time, + "stage": "session", + "sessionId": self.session_uid, + } + data = self._content_patcher.patch(data) + tag, label = self._content_patcher.get_tag_and_label() + self._event(tag, label, data) -stage: str = "session" -fluent_runtime: typing.Optional[FluentLoggerRuntime] = None +STAGE: str = "session" +FLUENT_RUNTIME: typing.Optional[FluentLoggerRuntime] = None ##################################################### # Setup @@ -255,7 +273,9 @@ def pytest_addoption(parser): group.addoption( "--stage-settings", type=str, - default=os.path.join(os.path.dirname(__file__), "data", "default.stage.json"), + default=load_and_check_settings_file( + os.path.join(os.path.dirname(__file__), "data", "default.stage.json") + ), action=SettingFileLoaderAction, help="Stage setting description JSON or YAML file path or string object.", ) @@ -263,20 +283,20 @@ def pytest_addoption(parser): def pytest_configure(config): """Extend pytest configuration.""" - global fluent_runtime + global FLUENT_RUNTIME config.fluent = FluentLoggerRuntime(config) config.pluginmanager.register(config.fluent, "fluent-reporter-runtime") - fluent_runtime = config.fluent + FLUENT_RUNTIME = config.fluent def pytest_unconfigure(config): """Unregister runtime from pytest.""" - global fluent_runtime + global FLUENT_RUNTIME fluent = getattr(config, "fluent", None) if fluent: del config.fluent config.pluginmanager.unregister(fluent) - fluent_runtime = None + FLUENT_RUNTIME = None ##################################################### @@ -379,13 +399,13 @@ def overflow_handler(pendings): def set_stage(val: str) -> None: """Set the current execution stage.""" - global stage - stage = val + global STAGE + STAGE = val def get_stage() -> str: """Get the current execution stage.""" - return stage + return STAGE # Unique identifiers @@ -398,16 +418,16 @@ def create_unique_identifier(): def get_session_uid() -> typing.Optional[str]: """Get current session UID.""" - if fluent_runtime is None: + if FLUENT_RUNTIME is None: return None - return typing.cast(FluentLoggerRuntime, fluent_runtime).session_uid + return typing.cast(FluentLoggerRuntime, FLUENT_RUNTIME).session_uid def get_test_uid() -> typing.Optional[str]: """Get current test UID.""" - if fluent_runtime is None: + if FLUENT_RUNTIME is None: return None - return typing.cast(FluentLoggerRuntime, fluent_runtime).test_uid + return typing.cast(FluentLoggerRuntime, FLUENT_RUNTIME).test_uid # Docstrings diff --git a/src/pytest_fluent/setting_file_loader_action.py b/src/pytest_fluent/setting_file_loader_action.py new file mode 100644 index 0000000..c0c6b66 --- /dev/null +++ b/src/pytest_fluent/setting_file_loader_action.py @@ -0,0 +1,53 @@ +"""Load and schema check settings file.""" +import argparse +import json +import os + +import jsonschema +from ruamel.yaml import YAML + + +class SettingFileLoaderAction(argparse.Action): + """Custom action for loading JSON/YAML configuration.""" + + def __call__(self, parser, args, values, option_string=None): + """Implementing call.""" + parameter = load_and_check_settings_file(values) + setattr(args, self.dest, parameter) + + +def load_and_check_settings_file(file_name: os.PathLike): + """Load settings file and check content against schema. + + Args: + file_name (os.PathLike): Path to settings file. + + Raises: + ValueError: File type not supported. + + Returns: + _type_: User settings dictionary. + """ + if file_name.endswith(".json"): + pickle = json + elif file_name.endswith(".yaml"): + pickle = YAML() + + def loads(file_pointer): + return pickle.load(file_pointer.read()) + + setattr(pickle, "loads", loads) + else: + raise ValueError("File type not supported.") + if os.path.exists(file_name): + with open(file_name, encoding="utf-8") as fid: + content = pickle.load(fid) + else: + content = pickle.loads(file_name) + with open( + os.path.join(os.path.dirname(__file__), "data", "schema.stage.json"), + encoding="utf-8", + ) as fid: + schema = json.load(fid) + jsonschema.validate(instance=content, schema=schema) + return content diff --git a/tests/test_content_patcher.py b/tests/test_content_patcher.py new file mode 100644 index 0000000..83ecdea --- /dev/null +++ b/tests/test_content_patcher.py @@ -0,0 +1,272 @@ +"""Tests for ContentPatcher.""" +# pylint: disable=W0212, C0116, W0621 +import argparse +import os +import typing +import uuid + +import pytest + +from pytest_fluent.content_patcher import ContentPatcher, _ContentType +from pytest_fluent.plugin import FluentLoggerRuntime + +UNIQUE_IDENTIFIER = str(uuid.uuid4()) + + +@pytest.fixture +def stage_names() -> typing.List[str]: + names = [ + method for method in dir(FluentLoggerRuntime) if method.startswith("pytest_") + ] + names.append("logging") + return names + + +@pytest.fixture +def user_settings() -> dict: + return { + "all": { + "tag": "run", + "label": "pytest", + "replace": {"status": "state", "sessionId": "id"}, + }, + "pytest_sessionstart": { + "tag": "run", + "label": "test", + "add": {"start_info": "Pytest started"}, + }, + "pytest_sessionfinish": { + "tag": "result", + "label": "test", + "add": {"stop_info": "Pytest finished"}, + }, + "pytest_runtest_logstart": { + "tag": "run", + "label": "testcase", + "add": {"start_info": "Testcase started"}, + }, + "pytest_runtest_logreport": { + "tag": "result", + "label": "testcase", + "add": {"stop_info": "Testcase finished"}, + }, + } + + +@pytest.fixture +def user_settings_patched() -> dict: + return { + "pytest_runtest_call": { + "tag": "run", + "label": "pytest", + "replace": {"status": "state", "sessionId": "id"}, + }, + "pytest_runtest_logfinish": { + "tag": "run", + "label": "pytest", + "replace": {"status": "state", "sessionId": "id"}, + }, + "pytest_runtest_logreport": { + "tag": "result", + "label": "testcase", + "replace": {"status": "state", "sessionId": "id"}, + "add": {"stop_info": "Testcase finished"}, + }, + "pytest_runtest_logstart": { + "tag": "run", + "label": "testcase", + "replace": {"status": "state", "sessionId": "id"}, + "add": {"start_info": "Testcase started"}, + }, + "pytest_runtest_makereport": { + "tag": "run", + "label": "pytest", + "replace": {"status": "state", "sessionId": "id"}, + }, + "pytest_runtest_setup": { + "tag": "run", + "label": "pytest", + "replace": {"status": "state", "sessionId": "id"}, + }, + "pytest_runtest_teardown": { + "tag": "run", + "label": "pytest", + "replace": {"status": "state", "sessionId": "id"}, + }, + "pytest_sessionfinish": { + "tag": "result", + "label": "test", + "replace": {"status": "state", "sessionId": "id"}, + "add": {"stop_info": "Pytest finished"}, + }, + "pytest_sessionstart": { + "tag": "run", + "label": "test", + "replace": {"status": "state", "sessionId": "id"}, + "add": {"start_info": "Pytest started"}, + }, + "logging": { + "tag": "run", + "label": "pytest", + "replace": {"status": "state", "sessionId": "id"}, + }, + } + + +@pytest.fixture +def stage_content() -> dict: + return {} + + +@pytest.fixture +def stage_content_patched() -> dict: + return {} + + +@pytest.fixture +def namespace() -> argparse.Namespace: + return argparse.Namespace(**{"fluentd-tag": "pytest"}) + + +def test_is_reference_string(): + assert ContentPatcher._is_reference_string("${USE_ENV}") == _ContentType.ENV + assert ContentPatcher._is_reference_string("") == _ContentType.ARGS + + +def test_get_env_content__no_env_string(): + assert ContentPatcher._get_env_content("test") == "" + + +def test_get_env_content__env_string(): + result = "test" + os.environ["USE_ENV"] = result + assert ContentPatcher._get_env_content("$USE_ENV") == result + assert ContentPatcher._get_env_content("${USE_ENV}") == result + + +def test_get_env_content__env_string_no_content(): + assert ContentPatcher._get_env_content("$USE_ENV") == "" + + +def test_get_args_content__retrieve_content(stage_content, namespace, stage_names): + patcher = ContentPatcher( + user_settings=stage_content, args_settings=namespace, stage_names=stage_names + ) + assert patcher._get_args_content("") == "pytest" + + +def test_get_args_content__retrieve_no_content(stage_content, namespace, stage_names): + patcher = ContentPatcher( + user_settings=stage_content, args_settings=namespace, stage_names=stage_names + ) + assert patcher._get_args_content("") == "" + + +def test_stage_settings(user_settings, user_settings_patched, stage_names): + patched = ContentPatcher._stage_settings(user_settings, stage_names) + assert patched == user_settings_patched + + +@pytest.mark.parametrize( + "to_patch,expected,stage", + [ + ( + { + "status": "start", + "stage": "session", + "sessionId": UNIQUE_IDENTIFIER, + }, + { + "tag": "run", + "label": "test", + "state": "start", + "stage": "session", + "id": UNIQUE_IDENTIFIER, + "start_info": "Pytest started", + }, + "pytest_sessionstart", + ), + ( + { + "status": "finish", + "stage": "session", + "sessionId": UNIQUE_IDENTIFIER, + }, + { + "tag": "result", + "label": "test", + "state": "finish", + "stage": "session", + "id": UNIQUE_IDENTIFIER, + "stop_info": "Pytest finished", + }, + "pytest_sessionfinish", + ), + ( + { + "status": "start", + "stage": "testcase", + "sessionId": UNIQUE_IDENTIFIER, + "testId": UNIQUE_IDENTIFIER, + "name": "testcase", + }, + { + "tag": "run", + "label": "testcase", + "state": "start", + "stage": "testcase", + "id": UNIQUE_IDENTIFIER, + "testId": UNIQUE_IDENTIFIER, + "name": "testcase", + "start_info": "Testcase started", + }, + "pytest_runtest_logstart", + ), + ( + { + "status": "finish", + "stage": "testcase", + "sessionId": UNIQUE_IDENTIFIER, + "testId": UNIQUE_IDENTIFIER, + "name": "testcase", + }, + { + "tag": "result", + "label": "testcase", + "state": "finish", + "stage": "testcase", + "id": UNIQUE_IDENTIFIER, + "testId": UNIQUE_IDENTIFIER, + "name": "testcase", + "stop_info": "Testcase finished", + }, + "pytest_runtest_logreport", + ), + ], +) +def test_patch_content(to_patch, expected, stage, user_settings_patched): + patched = ContentPatcher._patch_stage_content( + to_patch, user_settings_patched[stage] + ) + assert patched == expected + + +def test_get_tag_and_label( + user_settings, namespace, stage_names, user_settings_patched +): + stage = "pytest_runtest_logstart" + patcher = ContentPatcher( + user_settings=user_settings, args_settings=namespace, stage_names=stage_names + ) + assert patcher.get_tag_and_label(stage) == ( + user_settings_patched[stage]["tag"], + user_settings_patched[stage]["label"], + ) + + def pytest_runtest_logstart(): + return patcher.get_tag_and_label() + + assert pytest_runtest_logstart() == ( + user_settings_patched[stage]["tag"], + user_settings_patched[stage]["label"], + ) diff --git a/tests/test_parser.py b/tests/test_parser.py index 8768749..96ad2b4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2,8 +2,8 @@ import os import pytest -from pytest_fluent.SettingFileLoaderAction import SettingFileLoaderAction +from pytest_fluent.setting_file_loader_action import SettingFileLoaderAction parser = argparse.ArgumentParser() parser.add_argument( From 8b71a19ea9c6d76ebaed8dc470c60619c27ccfc7 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Tue, 13 Jun 2023 14:54:19 +0200 Subject: [PATCH 04/20] Fixing some merge issues and fixing tests --- src/pytest_fluent/event.py | 8 +++---- src/pytest_fluent/plugin.py | 3 ++- tests/conftest.py | 31 ++++++++++++++++++++-------- tests/test_additional_information.py | 3 +-- tests/test_content_patcher.py | 15 ++++++++------ tests/test_docstrings.py | 9 +++----- tests/test_fixtures.py | 3 +-- tests/test_reporting.py | 15 +++++--------- 8 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/pytest_fluent/event.py b/src/pytest_fluent/event.py index 2a1cb9d..761f7b8 100644 --- a/src/pytest_fluent/event.py +++ b/src/pytest_fluent/event.py @@ -4,7 +4,7 @@ import time import typing -from fluent import sender +from fluent.sender import FluentSender LOGGER = logging.getLogger(__package__) @@ -26,8 +26,7 @@ def __init__( ) -> None: """Initialize custom event class.""" self.senders = { - tag: sender.FluentSender(tag=tag, host=host, port=port, **kwargs) - for tag in tags + tag: FluentSender(tag=tag, host=host, port=port, **kwargs) for tag in tags } def __call__(self, tag: str, label: str, data: dict, **kwargs): @@ -40,8 +39,9 @@ def __call__(self, tag: str, label: str, data: dict, **kwargs): """ assert isinstance(data, dict), "data must be a dict" sender_ = self.senders.get(tag) - if not sender_: + if sender_ is None or isinstance(sender_, FluentSender): LOGGER.warning(f"Could not retrieve fluent instance for tag {tag}") + return timestamp = kwargs.get("time", int(time.time())) if not sender_.emit_with_time(label, timestamp, data): LOGGER.warning(f"Could not send data via fluent for tag {tag}: {data}") diff --git a/src/pytest_fluent/plugin.py b/src/pytest_fluent/plugin.py index 64b73ec..f03c267 100644 --- a/src/pytest_fluent/plugin.py +++ b/src/pytest_fluent/plugin.py @@ -251,7 +251,8 @@ def pytest_addoption(parser): ) group.addoption( "--fluentd-host", - default=None, + default="localhost", + type=str, help="Fluentd remote host. Defaults to a local Fluentd session", ) group.addoption( diff --git a/tests/conftest.py b/tests/conftest.py index c9a9cb2..3922f9d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,23 @@ import uuid -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest +import pytest_fluent.event + plugin_name = "pytest_fluent" +def isinstance_patch( + __obj: object, + __class_or_tuple, +) -> bool: + """Patch for isinstance.""" + if isinstance(__obj, MagicMock): + return True + return isinstance(__obj, __class_or_tuple) + + @pytest.fixture(scope="session") def logging_content(): return "Logged from test_base" @@ -53,14 +65,15 @@ def test_base(): @pytest.fixture() -def run_mocked_pytest(runpytest): - """create a temporary pytest environment with FluentSender mock.""" - - with patch("fluent.sender.FluentSender") as sender: - yield runpytest, sender +def fluentd_sender(): + """Get FluentSender mock.""" + with patch("pytest_fluent.event.FluentSender") as sender: + yield sender.return_value @pytest.fixture() -def fluentd_sender(): - with patch("fluent.sender.FluentSender") as sender: - yield sender.return_value +def run_mocked_pytest(runpytest, fluentd_sender): + """Create a temporary pytest environment with FluentSender mock.""" + + with patch.object(pytest_fluent.event, "isinstance", isinstance_patch): + yield runpytest, fluentd_sender diff --git a/tests/test_additional_information.py b/tests/test_additional_information.py index 81b4096..d87b5ff 100644 --- a/tests/test_additional_information.py +++ b/tests/test_additional_information.py @@ -15,9 +15,8 @@ def test_info() -> dict: def test_additional_information(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest runpytest(f"--session-uuid={session_uuid}") - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list for idx, call_arg in enumerate(call_args): data = call_arg.args[2] diff --git a/tests/test_content_patcher.py b/tests/test_content_patcher.py index 83ecdea..ffe6264 100644 --- a/tests/test_content_patcher.py +++ b/tests/test_content_patcher.py @@ -4,6 +4,7 @@ import os import typing import uuid +from unittest.mock import patch import pytest @@ -125,7 +126,7 @@ def stage_content_patched() -> dict: @pytest.fixture def namespace() -> argparse.Namespace: - return argparse.Namespace(**{"fluentd-tag": "pytest"}) + return argparse.Namespace(**{"fluentd_tag": "pytest"}) def test_is_reference_string(): @@ -139,9 +140,9 @@ def test_get_env_content__no_env_string(): def test_get_env_content__env_string(): result = "test" - os.environ["USE_ENV"] = result - assert ContentPatcher._get_env_content("$USE_ENV") == result - assert ContentPatcher._get_env_content("${USE_ENV}") == result + with patch.dict(os.environ, {"USE_ENV": result}): + assert ContentPatcher._get_env_content("$USE_ENV") == result + assert ContentPatcher._get_env_content("${USE_ENV}") == result def test_get_env_content__env_string_no_content(): @@ -163,8 +164,10 @@ def test_get_args_content__retrieve_no_content(stage_content, namespace, stage_n def test_stage_settings(user_settings, user_settings_patched, stage_names): - patched = ContentPatcher._stage_settings(user_settings, stage_names) - assert patched == user_settings_patched + patched = ContentPatcher( + user_settings=user_settings, args_settings=namespace, stage_names=stage_names + ) + assert patched._user_settings == user_settings_patched @pytest.mark.parametrize( diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 0a3d8c6..6546a42 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -2,7 +2,7 @@ def test_add_docstrings(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest result = runpytest( f"--session-uuid={session_uuid}", "--add-docstrings", @@ -14,7 +14,6 @@ def test_base(): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list result.assert_outcomes(passed=1) assert len(call_args) > 0 @@ -24,7 +23,7 @@ def test_base(): def test_docstrings_disabled(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest result = runpytest( f"--session-uuid={session_uuid}", pyfile=f""" @@ -35,7 +34,6 @@ def test_base(): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list result.assert_outcomes(passed=1) assert len(call_args) > 0 @@ -44,7 +42,7 @@ def test_base(): def test_missing_docstring(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest result = runpytest( f"--session-uuid={session_uuid}", "--add-docstrings", @@ -53,7 +51,6 @@ def test_base(): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list result.assert_outcomes(passed=1) assert len(call_args) > 0 diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py index c9acdc3..dff3c06 100644 --- a/tests/test_fixtures.py +++ b/tests/test_fixtures.py @@ -1,5 +1,5 @@ def test_get_logger(run_mocked_pytest, session_uuid, logging_content): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest result = runpytest( f"--session-uuid={session_uuid}", "--extend-logging", @@ -10,7 +10,6 @@ def test_base(get_logger): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list result.assert_outcomes(passed=1) assert len(call_args) > 0 diff --git a/tests/test_reporting.py b/tests/test_reporting.py index 2231b98..d84f05a 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -1,5 +1,5 @@ def test_data_reporter_base_with_passed(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest result = runpytest( f"--session-uuid={session_uuid}", pyfile=""" @@ -7,7 +7,6 @@ def test_base(): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list result.assert_outcomes(passed=1) assert len(call_args) > 0 @@ -24,7 +23,7 @@ def test_base(): def test_data_reporter_xdist_passed(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest result = runpytest( "-n 2", f"--session-uuid={session_uuid}", @@ -48,7 +47,6 @@ def test_base_group_six(): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list result.assert_outcomes(passed=6) assert len(call_args) > 0 @@ -76,7 +74,7 @@ def check_for_verdict(session_uuid, report: dict): def test_data_reporter_base_with_xfail(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest _ = runpytest( f"--session-uuid={session_uuid}", pyfile=""" @@ -87,7 +85,6 @@ def test_base(): assert False """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list assert len(call_args) > 0 args = call_args[2].args[2] @@ -97,7 +94,7 @@ def test_base(): def test_data_reporter_base_with_exception(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest _ = runpytest( f"--session-uuid={session_uuid}", pyfile=""" @@ -106,7 +103,6 @@ def test_base(): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list assert len(call_args) > 0 args = call_args[2].args[2] @@ -116,7 +112,7 @@ def test_base(): def test_data_reporter_base_with_setup_exception(run_mocked_pytest, session_uuid): - runpytest, sender = run_mocked_pytest + runpytest, fluent_sender = run_mocked_pytest _ = runpytest( f"--session-uuid={session_uuid}", pyfile=""" @@ -133,7 +129,6 @@ def test_base(my_value): assert True """, ) - fluent_sender = sender.return_value call_args = fluent_sender.emit_with_time.call_args_list assert len(call_args) > 0 args = call_args[2].args[2] From 347fbb5b56e9f489964e7b29ff56b6df64c055b3 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Tue, 13 Jun 2023 15:09:48 +0200 Subject: [PATCH 05/20] Further merge fix for tests --- src/pytest_fluent/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_fluent/event.py b/src/pytest_fluent/event.py index 761f7b8..491a308 100644 --- a/src/pytest_fluent/event.py +++ b/src/pytest_fluent/event.py @@ -39,7 +39,7 @@ def __call__(self, tag: str, label: str, data: dict, **kwargs): """ assert isinstance(data, dict), "data must be a dict" sender_ = self.senders.get(tag) - if sender_ is None or isinstance(sender_, FluentSender): + if sender_ is None or not isinstance(sender_, FluentSender): LOGGER.warning(f"Could not retrieve fluent instance for tag {tag}") return timestamp = kwargs.get("time", int(time.time())) From 3525ae101c6e2db9d2174fe5d36c84d33960ad32 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 14 Jun 2023 10:27:34 +0200 Subject: [PATCH 06/20] Fixing tests --- src/pytest_fluent/content_patcher.py | 3 +- .../setting_file_loader_action.py | 4 +- tests/conftest.py | 11 ++++-- tests/test_addoptions.py | 38 ++++++++++++------- tests/test_content_patcher.py | 10 ++--- tests/test_ini_configuration.py | 12 ++++-- 6 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/pytest_fluent/content_patcher.py b/src/pytest_fluent/content_patcher.py index f27b333..92433a3 100644 --- a/src/pytest_fluent/content_patcher.py +++ b/src/pytest_fluent/content_patcher.py @@ -73,7 +73,8 @@ def get_tag_and_label( """Return the tag for the corresponding stage. Args: - stage_name (typing.Optional[str], optional): Current stage. Defaults to None. + stage_name (typing.Optional[str], optional): Current stage. + Defaults to None. Returns: typing.Tuple[str, str]: Tag string, Label string. diff --git a/src/pytest_fluent/setting_file_loader_action.py b/src/pytest_fluent/setting_file_loader_action.py index c0c6b66..47c971e 100644 --- a/src/pytest_fluent/setting_file_loader_action.py +++ b/src/pytest_fluent/setting_file_loader_action.py @@ -16,11 +16,11 @@ def __call__(self, parser, args, values, option_string=None): setattr(args, self.dest, parameter) -def load_and_check_settings_file(file_name: os.PathLike): +def load_and_check_settings_file(file_name: str): """Load settings file and check content against schema. Args: - file_name (os.PathLike): Path to settings file. + file_name (str): Path to settings file. Raises: ValueError: File type not supported. diff --git a/tests/conftest.py b/tests/conftest.py index 3922f9d..e8afce3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock, patch import pytest +from fluent import handler import pytest_fluent.event @@ -65,9 +66,12 @@ def test_base(): @pytest.fixture() -def fluentd_sender(): +def fluentd_sender(monkeypatch): """Get FluentSender mock.""" - with patch("pytest_fluent.event.FluentSender") as sender: + with patch("pytest_fluent.event.FluentSender") as sender, patch.object( + pytest_fluent.event, "isinstance", isinstance_patch + ): + monkeypatch.setattr(handler.sender, "FluentSender", sender) yield sender.return_value @@ -75,5 +79,4 @@ def fluentd_sender(): def run_mocked_pytest(runpytest, fluentd_sender): """Create a temporary pytest environment with FluentSender mock.""" - with patch.object(pytest_fluent.event, "isinstance", isinstance_patch): - yield runpytest, fluentd_sender + return runpytest, fluentd_sender diff --git a/tests/test_addoptions.py b/tests/test_addoptions.py index 08c7a41..9fca1aa 100644 --- a/tests/test_addoptions.py +++ b/tests/test_addoptions.py @@ -11,28 +11,36 @@ FAKE_TEST_UUID = "6d653fee-0c6a-4923-9216-dfc949bd05a0" -def get_data_from_call_args(call_args, fields: typing.List[str]) -> typing.Dict: - return {field: call_args.args[2].get(field) for field in fields} +@pytest.fixture +def pyfile_testcase(logging_content): + return f""" +import logging +def test_base(): + LOGGER = logging.getLogger() + LOGGER.info('{logging_content}') + LOGGER.warning('{logging_content}') + assert True +""" -@pytest.fixture(name="monkeypatched_uuid4") -def monkeypatched_uuid4_fixture(monkeypatch): - def myuuid4(): - return uuid.UUID(FAKE_TEST_UUID) - monkeypatch.setattr(uuid, "uuid4", myuuid4) +def get_data_from_call_args(call_args, fields: typing.List[str]) -> typing.Dict: + return {field: call_args.args[2].get(field) for field in fields} def test_fluentd_logged_parameters( - monkeypatched_uuid4, runpytest, fluentd_sender, session_uuid, logging_content + monkeypatch, run_mocked_pytest, session_uuid, logging_content, pyfile_testcase ): + runpytest, fluent_sender = run_mocked_pytest + monkeypatch.setattr(uuid, "uuid4", lambda: uuid.UUID(FAKE_TEST_UUID)) result = runpytest( f"--session-uuid={session_uuid}", f"--fluentd-tag={FLUENTD_TAG}", f"--fluentd-label={FLUENTD_LABEL}", "--extend-logging", + pyfile=pyfile_testcase, ) - call_args = fluentd_sender.emit_with_time.call_args_list + call_args = fluent_sender.emit_with_time.call_args_list result.assert_outcomes(passed=1) assert len(call_args) == 7 @@ -102,14 +110,16 @@ def is_pytest_message(args): def test_fluentd_with_options_and_timestamp_enabled_shows_timestamp_field_in_output( - runpytest, fluentd_sender, session_uuid + run_mocked_pytest, fluentd_sender, session_uuid, pyfile_testcase ): + runpytest, fluentd_sender = run_mocked_pytest result = runpytest( f"--session-uuid={session_uuid}", f"--fluentd-tag={FLUENTD_TAG}", f"--fluentd-label={FLUENTD_LABEL}", - f"--fluentd-timestamp=@timestamp", + "--fluentd-timestamp=@timestamp", "--extend-logging", + pyfile=pyfile_testcase, ) result.assert_outcomes(passed=1) call_args = fluentd_sender.emit_with_time.call_args_list @@ -120,11 +130,13 @@ def test_fluentd_with_options_and_timestamp_enabled_shows_timestamp_field_in_out def test_fluentd_with_timestamp_enabled_shows_timestamp_field_in_output( - runpytest, fluentd_sender, session_uuid + run_mocked_pytest, session_uuid, pyfile_testcase ): + runpytest, fluentd_sender = run_mocked_pytest result = runpytest( f"--session-uuid={session_uuid}", - f"--fluentd-timestamp=@timestamp", + "--fluentd-timestamp=@timestamp", + pyfile=pyfile_testcase, ) result.assert_outcomes(passed=1) call_args = fluentd_sender.emit_with_time.call_args_list diff --git a/tests/test_content_patcher.py b/tests/test_content_patcher.py index ffe6264..0fcd8ff 100644 --- a/tests/test_content_patcher.py +++ b/tests/test_content_patcher.py @@ -1,10 +1,8 @@ """Tests for ContentPatcher.""" # pylint: disable=W0212, C0116, W0621 import argparse -import os import typing import uuid -from unittest.mock import patch import pytest @@ -138,11 +136,11 @@ def test_get_env_content__no_env_string(): assert ContentPatcher._get_env_content("test") == "" -def test_get_env_content__env_string(): +def test_get_env_content__env_string(monkeypatch): result = "test" - with patch.dict(os.environ, {"USE_ENV": result}): - assert ContentPatcher._get_env_content("$USE_ENV") == result - assert ContentPatcher._get_env_content("${USE_ENV}") == result + monkeypatch.setenv("USE_ENV", result) + assert ContentPatcher._get_env_content("$USE_ENV") == result + assert ContentPatcher._get_env_content("${USE_ENV}") == result def test_get_env_content__env_string_no_content(): diff --git a/tests/test_ini_configuration.py b/tests/test_ini_configuration.py index b51ec46..334a364 100644 --- a/tests/test_ini_configuration.py +++ b/tests/test_ini_configuration.py @@ -15,14 +15,18 @@ @pytest.fixture def tox_ini(pytester, session_uuid): return pytester.makeini( - f"[pytest]\naddopts = --session-uuid='{session_uuid}' --fluentd-port={PORT} --fluentd-host='{HOSTNAME}' --fluentd-tag='{TAG}' --fluentd-label='{LABEL}' --extend-logging" + f'[pytest]\naddopts = --session-uuid="{session_uuid}" --fluentd-port={PORT} ' + f'--fluentd-host="{HOSTNAME}" --fluentd-tag="{TAG}" --fluentd-label="{LABEL}" ' + f"--extend-logging" ) @pytest.fixture def pyprojtoml_ini(pytester, session_uuid): return pytester.makepyprojecttoml( - f"[tool.pytest.ini_options]\naddopts = \"--session-uuid='{session_uuid}' --fluentd-port={PORT} --fluentd-host='{HOSTNAME}' --fluentd-tag='{TAG}' --extend-logging\"" + f'[tool.pytest.ini_options]\naddopts = --session-uuid="{session_uuid}" ' + f'--fluentd-port={PORT} --fluentd-host="{HOSTNAME}" --fluentd-tag="{TAG}" ' + f"--extend-logging" ) @@ -30,7 +34,9 @@ def pyprojtoml_ini(pytester, session_uuid): def pytest_ini(pytester, session_uuid): return pytester.makefile( ".ini", - pytest=f"[pytest]\naddopts = --session-uuid='{session_uuid}' --fluentd-port={PORT} --fluentd-host='{HOSTNAME}' --fluentd-tag='{TAG}' --fluentd-label='{LABEL}' --extend-logging", + pytest=f'[pytest]\naddopts = --session-uuid="{session_uuid}" ' + f'--fluentd-port={PORT} --fluentd-host="{HOSTNAME}" --fluentd-tag="{TAG}" ' + f'--fluentd-label="{LABEL}" --extend-logging', ) From f44d76c735ec78c4daaf3e4e6f2098c889c366cf Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 14 Jun 2023 10:36:38 +0200 Subject: [PATCH 07/20] Don't continue on linting error and extended linting checks --- .github/workflows/linting.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 445cd87..1648446 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -9,7 +9,7 @@ on: jobs: linting: strategy: - fail-fast: false + fail-fast: true matrix: python: ["3.8", "3.9", "3.10", "3.11"] runs-on: ubuntu-latest @@ -29,11 +29,15 @@ jobs: - uses: isort/isort-action@master with: configuration: "--check-only --diff" + - name: Analysing the code style + run: | + tox -e codestyle + - name: Analysing the docstyle + run: | + tox -e docstyle - name: Analysing the code with mypy - continue-on-error: true run: | tox -e types - name: Analysing the code with pylint/flake - continue-on-error: true run: | tox -e errors From a40c2f1a7067b92593b62277e7f9f76b8949668e Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 14 Jun 2023 10:58:32 +0200 Subject: [PATCH 08/20] Fixing wrong f-string replacement --- tests/test_ini_configuration.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_ini_configuration.py b/tests/test_ini_configuration.py index 334a364..d7968aa 100644 --- a/tests/test_ini_configuration.py +++ b/tests/test_ini_configuration.py @@ -3,7 +3,8 @@ import pytest -# set log_level="DEBUG" and log_cli = true in pyproject.toml configuration when debug info is needed +# set log_level="DEBUG" and log_cli = true in pyproject.toml configuration when +# debug info is needed logger = logging.getLogger("debug-log") TAG = "unittest" @@ -24,9 +25,9 @@ def tox_ini(pytester, session_uuid): @pytest.fixture def pyprojtoml_ini(pytester, session_uuid): return pytester.makepyprojecttoml( - f'[tool.pytest.ini_options]\naddopts = --session-uuid="{session_uuid}" ' - f'--fluentd-port={PORT} --fluentd-host="{HOSTNAME}" --fluentd-tag="{TAG}" ' - f"--extend-logging" + f'[tool.pytest.ini_options]\naddopts = "--session-uuid={session_uuid} ' + f"--fluentd-port={PORT} --fluentd-host={HOSTNAME} --fluentd-tag={TAG} " + f'--extend-logging"' ) From 567169878bd4817bb32269761dd60c2b51b4cd20 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 14 Jun 2023 11:11:37 +0200 Subject: [PATCH 09/20] Fixing tox -e types --- src/pytest_fluent/content_patcher.py | 4 ++-- src/pytest_fluent/plugin.py | 8 +++++--- tests/test_parser.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/pytest_fluent/content_patcher.py b/src/pytest_fluent/content_patcher.py index 92433a3..89029a8 100644 --- a/src/pytest_fluent/content_patcher.py +++ b/src/pytest_fluent/content_patcher.py @@ -166,6 +166,6 @@ def _get_args_content(self, value: str) -> str: args_match = re.findall(r"^<([\w-]+)>$", value) if not args_match: return "" - args_match = args_match[0].replace("-", "_") - args_value = getattr(self._args_settings, args_match, "") + args_string = args_match[0].replace("-", "_") + args_value = getattr(self._args_settings, args_string, "") return args_value diff --git a/src/pytest_fluent/plugin.py b/src/pytest_fluent/plugin.py index f03c267..3499862 100644 --- a/src/pytest_fluent/plugin.py +++ b/src/pytest_fluent/plugin.py @@ -53,13 +53,13 @@ def __init__(self, config): args_settings=config.option, stage_names=stage_names, ) - tags = [] + tags: typing.List[str] = [] for value in self._content_patcher.user_settings.values(): tag = value.get("tag") if not tag: continue tags.append(tag) - tags = set(tags) + tags = list(set(tags)) self._event = Event( tags, self._host, self._port, buffer_overflow_handler=overflow_handler ) @@ -223,7 +223,9 @@ def pytest_sessionfinish( if not self.config.getoption("collectonly"): data = { "status": "finish", - "duration": time.time() - self._session_start_time, + "duration": time.time() - 0 + if self._session_start_time is None + else self._session_start_time, "stage": "session", "sessionId": self.session_uid, } diff --git a/tests/test_parser.py b/tests/test_parser.py index 96ad2b4..2ec5619 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -43,7 +43,7 @@ def test_yaml_file(default): def test_xml_error(): with pytest.raises(ValueError): - args = parser.parse_args( + parser.parse_args( [ "--stage-settings", os.path.join(os.path.dirname(__file__), "data", "default.xml"), From 949b6f20e50048166dc85aca6a6812822d8a386a Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 14 Jun 2023 11:19:00 +0200 Subject: [PATCH 10/20] Fixing tox -e docstyle --- src/pytest_fluent/content_patcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pytest_fluent/content_patcher.py b/src/pytest_fluent/content_patcher.py index 89029a8..2464a14 100644 --- a/src/pytest_fluent/content_patcher.py +++ b/src/pytest_fluent/content_patcher.py @@ -93,8 +93,8 @@ def patch(self, content: dict, stage_name: typing.Optional[str] = None) -> dict: Defaults to None. Returns: - str: Patch dictionary with the user provided stage settings. - """ + dict: Patched dictionary with the user provided stage settings. + """ # noqa if stage_name is None: stage_name = inspect.stack()[1][3] From 2d539d652b332e744b4fe4823c065c9e7d2a7f19 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Wed, 14 Jun 2023 13:44:08 +0200 Subject: [PATCH 11/20] Refactoring function name --- src/pytest_fluent/plugin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pytest_fluent/plugin.py b/src/pytest_fluent/plugin.py index 3499862..862579e 100644 --- a/src/pytest_fluent/plugin.py +++ b/src/pytest_fluent/plugin.py @@ -86,7 +86,7 @@ def _set_session_uid( else: raise ValueError("Unique identifier is not in a valid format.") - def set_timestamp_information(self, data: dict): + def _set_timestamp_information(self, data: dict): if self._timestamp is not None: data.update({self._timestamp: f"{datetime.datetime.utcnow().isoformat()}"}) @@ -118,7 +118,7 @@ def pytest_sessionstart(self): } data = self._content_patcher.patch(data) data.update(get_additional_session_information()) - self.set_timestamp_information(data=data) + self._set_timestamp_information(data=data) tag, label = self._content_patcher.get_tag_and_label() self._event(tag, label, data) @@ -136,7 +136,7 @@ def pytest_runtest_logstart(self, nodeid: str, location: typing.Tuple[int, str]) } data = self._content_patcher.patch(data) data.update(get_additional_test_information()) - self.set_timestamp_information(data=data) + self._set_timestamp_information(data=data) tag, label = self._content_patcher.get_tag_and_label() self._event(tag, label, data) @@ -188,7 +188,7 @@ def pytest_runtest_logreport(self, report: pytest.TestReport): docstring = report.stash.get(DOCSTRING_KEY, None) if docstring: data.update({"docstring": docstring}) - self.set_timestamp_information(data=data) + self._set_timestamp_information(data=data) data = self._content_patcher.patch(data) tag, label = self._content_patcher.get_tag_and_label() self._event(tag, label, data) @@ -208,7 +208,7 @@ def pytest_runtest_logfinish( "testId": self.test_uid, "name": nodeid, } - self.set_timestamp_information(data=data) + self._set_timestamp_information(data=data) data = self._content_patcher.patch(data) tag, label = self._content_patcher.get_tag_and_label() self._event(tag, label, data) @@ -229,7 +229,7 @@ def pytest_sessionfinish( "stage": "session", "sessionId": self.session_uid, } - self.set_timestamp_information(data=data) + self._set_timestamp_information(data=data) data = self._content_patcher.patch(data) tag, label = self._content_patcher.get_tag_and_label() self._event(tag, label, data) From 73402279680805df202dfdd88fa1e010d6161b97 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 08:15:29 +0200 Subject: [PATCH 12/20] Improving logging patching and new testcase --- src/pytest_fluent/content_patcher.py | 16 ++++++-- src/pytest_fluent/plugin.py | 54 ++++++++++++++++++-------- tests/test_content_patcher.py | 58 +++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 22 deletions(-) diff --git a/src/pytest_fluent/content_patcher.py b/src/pytest_fluent/content_patcher.py index 2464a14..0881695 100644 --- a/src/pytest_fluent/content_patcher.py +++ b/src/pytest_fluent/content_patcher.py @@ -84,7 +84,12 @@ def get_tag_and_label( stage_info = self._user_settings.get(stage_name, {}) return stage_info["tag"], stage_info["label"] - def patch(self, content: dict, stage_name: typing.Optional[str] = None) -> dict: + def patch( + self, + content: dict, + stage_name: typing.Optional[str] = None, + ignore_entries: typing.List[str] = [], + ) -> dict: """Patch the content with the provided settings for each stage. Args: @@ -99,6 +104,7 @@ def patch(self, content: dict, stage_name: typing.Optional[str] = None) -> dict: stage_name = inspect.stack()[1][3] stage_info = self._user_settings.get(stage_name, {}) + stage_info = {k: v for k, v in stage_info.items() if k not in ignore_entries} if not stage_info: return content return self._patch_stage_content(content, stage_info) @@ -106,10 +112,12 @@ def patch(self, content: dict, stage_name: typing.Optional[str] = None) -> dict: @staticmethod def _patch_stage_content(stage_content: dict, user_settings: dict) -> dict: stage_content_patched = stage_content.copy() - stage_content_patched["tag"] = user_settings["tag"] - stage_content_patched["label"] = user_settings["label"] + if "tag" in user_settings: + stage_content_patched["tag"] = user_settings["tag"] + if "label" in user_settings: + stage_content_patched["label"] = user_settings["label"] if "replace" in user_settings: - for key, value in user_settings["replace"].items(): + for key, value in user_settings.get("replace", {}).items(): if key in stage_content_patched: tmp = stage_content_patched[key] stage_content_patched[value] = tmp diff --git a/src/pytest_fluent/plugin.py b/src/pytest_fluent/plugin.py index 862579e..2572e33 100644 --- a/src/pytest_fluent/plugin.py +++ b/src/pytest_fluent/plugin.py @@ -64,15 +64,27 @@ def __init__(self, config): tags, self._host, self._port, buffer_overflow_handler=overflow_handler ) self._log_reporter = LogReport(self.config) - self._patch_logging( - f"{self._content_patcher.user_settings['logging']['tag']}." - f"{self._content_patcher.user_settings['logging']['label']}" + self._patch_logging() + + def _patch_logging(self): + if not self._extend_logging: + return + tag = self._content_patcher.user_settings.get("logging", {}).get("tag") + if not tag: + raise ValueError( + "Tag for logging was not set. Please set either specific tag value for \ + key 'logging' or use the 'all' object in stage settings file." + ) + label = self._content_patcher.user_settings.get("logging", {}).get("label") + if label: + tag = f"{tag}.{label}" + extend_loggers( + self._host, + self._port, + tag, + self._content_patcher, ) - def _patch_logging(self, tag: str): - if self._extend_logging: - extend_loggers(self._host, self._port, tag) - def _set_session_uid( self, id: typing.Optional[typing.Union[str, uuid.UUID]] = None ) -> None: @@ -330,14 +342,14 @@ def get_logger(request): port = config.getoption("--fluentd-port") tag = config.getoption("--fluentd-tag") - def get_logger(name=None): + def get_logger_wrapper(name=None): logger = logging.getLogger(name) if name is None: return logger add_handler(host, port, tag, logger) return logger - return get_logger + return get_logger_wrapper @pytest.fixture @@ -363,9 +375,10 @@ def test_uid() -> typing.Optional[str]: class RecordFormatter(FluentRecordFormatter): """Extension of FluentRecordFormatter in order to add unique ID's""" - def __init__(self, *args, **kwargs): + def __init__(self, patcher: typing.Optional[ContentPatcher], *args, **kwargs): """Specific initilization.""" super(RecordFormatter, self).__init__(*args, **kwargs) + self.content_patcher = patcher def format(self, record): """Extend formatting for Fluentd handler.""" @@ -375,34 +388,41 @@ def format(self, record): data["sessionId"] = get_session_uid() data["testId"] = get_test_uid() data["stage"] = get_stage() + if self.content_patcher: + data = self.content_patcher.patch(data, "logging", ["tag", "label"]) return data -def extend_loggers(host, port, tag) -> None: +def extend_loggers(host, port, tag, patcher: ContentPatcher) -> None: """Extend Python logging with a Fluentd handler.""" - modify_logger(host, port, tag, None) - modify_logger(host, port, tag, "fluent") + modify_logger(host, port, tag, None, patcher) + modify_logger(host, port, tag, "fluent", patcher) -def modify_logger(host, port, tag, name=None) -> None: +def modify_logger( + host, port, tag, name=None, patcher: typing.Optional[ContentPatcher] = None +) -> None: """Extend Python logging with a Fluentd handler.""" logger = logging.getLogger(name) - add_handler(host, port, tag, logger) + add_handler(host, port, tag, logger, patcher) -def add_handler(host, port, tag, logger): +def add_handler( + host, port, tag, logger, patcher: typing.Optional[ContentPatcher] = None +): """Add handler to a specific logger.""" handler = FluentHandler( tag, host=host, port=port, buffer_overflow_handler=overflow_handler ) formatter = RecordFormatter( + patcher, { "type": "logging", "host": "%(hostname)s", "where": "%(module)s.%(funcName)s", "level": "%(levelname)s", "stack_trace": "%(exc_text)s", - } + }, ) handler.setFormatter(formatter) logger.addHandler(handler) diff --git a/tests/test_content_patcher.py b/tests/test_content_patcher.py index 0fcd8ff..90e0ca6 100644 --- a/tests/test_content_patcher.py +++ b/tests/test_content_patcher.py @@ -49,6 +49,9 @@ def user_settings() -> dict: "label": "testcase", "add": {"stop_info": "Testcase finished"}, }, + "logging": { + "replace": {"message": "msg", "sessionId": "id"}, + }, } @@ -107,7 +110,7 @@ def user_settings_patched() -> dict: "logging": { "tag": "run", "label": "pytest", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"message": "msg", "sessionId": "id"}, }, } @@ -252,6 +255,59 @@ def test_patch_content(to_patch, expected, stage, user_settings_patched): assert patched == expected +@pytest.mark.parametrize( + "to_patch,expected,stage,ignore", + [ + ( + { + "status": "start", + "stage": "session", + "sessionId": UNIQUE_IDENTIFIER, + }, + { + "tag": "run", + "label": "test", + "state": "start", + "stage": "session", + "id": UNIQUE_IDENTIFIER, + "start_info": "Pytest started", + }, + "pytest_sessionstart", + [], + ), + ( + { + "type": "logging", + "stage": "testcase", + "message": "Logged from test_base", + "sessionId": UNIQUE_IDENTIFIER, + }, + { + "type": "logging", + "stage": "testcase", + "id": UNIQUE_IDENTIFIER, + "msg": "Logged from test_base", + }, + "logging", + ["tag", "label"], + ), + ], +) +def test_patch( + to_patch, + expected, + stage, + ignore, + user_settings, + namespace, + stage_names, +): + patched = ContentPatcher(user_settings, namespace, stage_names).patch( + to_patch, stage, ignore + ) + assert patched == expected + + def test_get_tag_and_label( user_settings, namespace, stage_names, user_settings_patched ): From 62ca5ed2056ce54e1e66f52911d1f5cd4e46ffc7 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 08:26:35 +0200 Subject: [PATCH 13/20] Fixing some default values --- src/pytest_fluent/data/schema.stage.json | 4 ++-- tests/data/default.json | 4 ++-- tests/data/default.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pytest_fluent/data/schema.stage.json b/src/pytest_fluent/data/schema.stage.json index 44a67fa..8ca2f1a 100644 --- a/src/pytest_fluent/data/schema.stage.json +++ b/src/pytest_fluent/data/schema.stage.json @@ -18,11 +18,11 @@ "properties": { "tag": { "$ref": "#/definitions/RegexString", - "default": "" + "default": "" }, "label": { "$ref": "#/definitions/RegexString", - "default": "" + "default": "" }, "replace": { "type": "object", diff --git a/tests/data/default.json b/tests/data/default.json index 4fc34b5..1e8f941 100644 --- a/tests/data/default.json +++ b/tests/data/default.json @@ -1,6 +1,6 @@ { "all": { - "tag": "", - "label": "" + "tag": "", + "label": "" } } \ No newline at end of file diff --git a/tests/data/default.yaml b/tests/data/default.yaml index 8062978..1700368 100644 --- a/tests/data/default.yaml +++ b/tests/data/default.yaml @@ -1,3 +1,3 @@ all: - tag: "" - label: "" + tag: "" + label: "" From dcfab783d386e5604cc860453ed6ead57026716d Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 11:09:21 +0200 Subject: [PATCH 14/20] Adding also replace values functionality --- src/pytest_fluent/content_patcher.py | 50 +++++-- src/pytest_fluent/data/schema.stage.json | 48 ++++++- tests/conftest.py | 3 +- tests/test_content_patcher.py | 30 ++-- tests/test_parser.py | 2 +- tests/test_reporting_patching.py | 170 +++++++++++++++++++++++ 6 files changed, 268 insertions(+), 35 deletions(-) create mode 100644 tests/test_reporting_patching.py diff --git a/src/pytest_fluent/content_patcher.py b/src/pytest_fluent/content_patcher.py index 0881695..dbe53c2 100644 --- a/src/pytest_fluent/content_patcher.py +++ b/src/pytest_fluent/content_patcher.py @@ -40,24 +40,37 @@ def _stage_settings( patched = {} all_settings = {} for key, value in user_settings.get("all", {}).items(): - if isinstance(value, dict): - for subkey, subvalue in value.items(): - value[subkey] = self._get_env_or_args(subvalue) - else: - value = self._get_env_or_args(value) + value = self._patch_value(key, value) all_settings.update({key: value}) for stage_name in stage_names: patched.update({stage_name: all_settings.copy()}) stage_info = user_settings.get(stage_name, {}) for key, value in stage_info.items(): + value = self._patch_value(key, value) if isinstance(value, dict): - for subkey, subvalue in value.items(): - value[subkey] = self._get_env_or_args(subvalue) - else: - value = self._get_env_or_args(value) + value = self._merge_patched_values( + patched[stage_name].get(key, {}), value + ) patched[stage_name].update({key: value}) return patched + def _patch_value(self, key: str, value: typing.Any) -> typing.Any: + if key == "replace": + value = {key: self._patch_value(key, v) for key, v in value.items()} + else: + if isinstance(value, dict): + for subkey, subvalue in value.items(): + value[subkey] = self._get_env_or_args(subvalue) + else: + value = self._get_env_or_args(value) + return value + + def _merge_patched_values(self, old: dict, new: dict) -> dict: + merged = old.copy() + for key, value in new.items(): + merged[key] = value + return merged + @property def user_settings(self) -> dict: """Retrieve processed user settings. @@ -117,11 +130,20 @@ def _patch_stage_content(stage_content: dict, user_settings: dict) -> dict: if "label" in user_settings: stage_content_patched["label"] = user_settings["label"] if "replace" in user_settings: - for key, value in user_settings.get("replace", {}).items(): - if key in stage_content_patched: - tmp = stage_content_patched[key] - stage_content_patched[value] = tmp - del stage_content_patched[key] + replace_it = user_settings["replace"] + if "keys" in replace_it: + keys_settings = replace_it["keys"] + for key, value in keys_settings.items(): + if key in stage_content_patched: + tmp = stage_content_patched[key] + stage_content_patched[value] = tmp + del stage_content_patched[key] + if "values" in replace_it: + value_settings = replace_it["values"] + new_value_keys = value_settings.keys() + for key, value in stage_content_patched.items(): + if not isinstance(value, dict) and value in new_value_keys: + stage_content_patched[key] = value_settings[value] to_add = user_settings.get("add", {}) stage_content_patched.update(to_add) to_drop = user_settings.get("drop", []) diff --git a/src/pytest_fluent/data/schema.stage.json b/src/pytest_fluent/data/schema.stage.json index 8ca2f1a..5466faf 100644 --- a/src/pytest_fluent/data/schema.stage.json +++ b/src/pytest_fluent/data/schema.stage.json @@ -6,6 +6,28 @@ "$ref": "#/definitions/AdditionalProperties" } }, + "anyOf": [ + { + "type": "object", + "patternProperties": { + "all": { + "$ref": "#/definitions/AdditionalProperties", + "required": [ + "tag", + "label" + ] + } + } + }, + { + "type": "object", + "patternProperties": { + "pytest_runtest_logstart|pytest_runtest_logreport|pytest_runtest_logfinish|logging": { + "$ref": "#/definitions/AdditionalProperties" + } + } + } + ], "additionalProperties": false, "definitions": { "RegexString": { @@ -25,10 +47,20 @@ "default": "" }, "replace": { - "type": "object", - "patternProperties": { - "status|stage|sessionId|testId|name|when|duration": { - "$ref": "#/definitions/RegexString" + "keys": { + "type": "object", + "patternProperties": { + "status|stage|sessionId|testId|name|when|duration": { + "$ref": "#/definitions/RegexString" + } + } + }, + "values": { + "type": "object", + "patternProperties": { + "passed|failed|skipped|error|start|finish|session|testcase": { + "type": "string" + } } } }, @@ -44,8 +76,10 @@ "$ref": "#/definitions/RegexString" } } - }, - "required": [] + } } - } + }, + "required": [ + "all" + ] } \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index e8afce3..c172d4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ import pytest_fluent.event plugin_name = "pytest_fluent" +SESSION_UUID = uuid.uuid4() def isinstance_patch( @@ -26,7 +27,7 @@ def logging_content(): @pytest.fixture(scope="session") def session_uuid(): - return uuid.uuid4() + return SESSION_UUID @pytest.fixture() diff --git a/tests/test_content_patcher.py b/tests/test_content_patcher.py index 90e0ca6..ee1568f 100644 --- a/tests/test_content_patcher.py +++ b/tests/test_content_patcher.py @@ -27,7 +27,7 @@ def user_settings() -> dict: "all": { "tag": "run", "label": "pytest", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, }, "pytest_sessionstart": { "tag": "run", @@ -47,10 +47,13 @@ def user_settings() -> dict: "pytest_runtest_logreport": { "tag": "result", "label": "testcase", + "replace": { + "values": {"passed": "pass", "failed": "fail"}, + }, "add": {"stop_info": "Testcase finished"}, }, "logging": { - "replace": {"message": "msg", "sessionId": "id"}, + "replace": {"keys": {"message": "msg", "sessionId": "id"}}, }, } @@ -61,56 +64,59 @@ def user_settings_patched() -> dict: "pytest_runtest_call": { "tag": "run", "label": "pytest", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, }, "pytest_runtest_logfinish": { "tag": "run", "label": "pytest", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, }, "pytest_runtest_logreport": { "tag": "result", "label": "testcase", - "replace": {"status": "state", "sessionId": "id"}, + "replace": { + "keys": {"status": "state", "sessionId": "id"}, + "values": {"passed": "pass", "failed": "fail"}, + }, "add": {"stop_info": "Testcase finished"}, }, "pytest_runtest_logstart": { "tag": "run", "label": "testcase", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, "add": {"start_info": "Testcase started"}, }, "pytest_runtest_makereport": { "tag": "run", "label": "pytest", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, }, "pytest_runtest_setup": { "tag": "run", "label": "pytest", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, }, "pytest_runtest_teardown": { "tag": "run", "label": "pytest", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, }, "pytest_sessionfinish": { "tag": "result", "label": "test", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, "add": {"stop_info": "Pytest finished"}, }, "pytest_sessionstart": { "tag": "run", "label": "test", - "replace": {"status": "state", "sessionId": "id"}, + "replace": {"keys": {"status": "state", "sessionId": "id"}}, "add": {"start_info": "Pytest started"}, }, "logging": { "tag": "run", "label": "pytest", - "replace": {"message": "msg", "sessionId": "id"}, + "replace": {"keys": {"message": "msg", "sessionId": "id"}}, }, } diff --git a/tests/test_parser.py b/tests/test_parser.py index 2ec5619..84238aa 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -18,7 +18,7 @@ @pytest.fixture def default() -> dict: - return {"all": {"tag": "", "label": ""}} + return {"all": {"tag": "", "label": ""}} def test_json_file(default): diff --git a/tests/test_reporting_patching.py b/tests/test_reporting_patching.py new file mode 100644 index 0000000..92fe437 --- /dev/null +++ b/tests/test_reporting_patching.py @@ -0,0 +1,170 @@ +import json + +import pytest + +from .conftest import SESSION_UUID + + +@pytest.mark.parametrize( + "patch_file_content,expected_result", + [ + ( + {"all": {"tag": "", "label": ""}}, + [ + { + "status": "start", + "stage": "session", + "sessionId": str(SESSION_UUID), + "tag": "test", + "label": "pytest", + }, + { + "status": "start", + "stage": "testcase", + "sessionId": str(SESSION_UUID), + "name": "test_data_reporter_with_patched_values.py::test_base", + "tag": "test", + "label": "pytest", + }, + { + "type": "logging", + "where": "test_data_reporter_with_patched_values.test_base", + "level": "INFO", + "stack_trace": "None", + "message": "Test running", + "sessionId": str(SESSION_UUID), + "stage": "testcase", + }, + { + "name": "test_data_reporter_with_patched_values.py::test_base", + "outcome": "passed", + "markers": { + "test_base": 1, + "test_data_reporter_with_patched_values.py": 1, + "test_data_reporter_with_patched_values0": 1, + }, + "stage": "testcase", + "when": "call", + "sessionId": str(SESSION_UUID), + "tag": "test", + }, + { + "status": "finish", + "stage": "testcase", + "sessionId": str(SESSION_UUID), + "name": "test_data_reporter_with_patched_values.py::test_base", + "tag": "test", + "label": "pytest", + }, + { + "status": "finish", + "stage": "session", + "sessionId": str(SESSION_UUID), + "tag": "test", + "label": "pytest", + }, + ], + ), + ( + { + "all": { + "tag": "", + "label": "", + "replace": { + "keys": {"status": "state", "sessionId": "id"}, + }, + }, + "pytest_runtest_logreport": { + "replace": { + "values": {"passed": "pass", "failed": "fail"}, + }, + "add": {"stop_info": "Testcase finished"}, + }, + }, + [ + { + "state": "start", + "stage": "session", + "id": str(SESSION_UUID), + "tag": "test", + "label": "pytest", + }, + { + "state": "start", + "stage": "testcase", + "id": str(SESSION_UUID), + "name": "test_data_reporter_with_patched_values.py::test_base", + "tag": "test", + "label": "pytest", + }, + { + "type": "logging", + "where": "test_data_reporter_with_patched_values.test_base", + "level": "INFO", + "stack_trace": "None", + "message": "Test running", + "id": str(SESSION_UUID), + "stage": "testcase", + }, + { + "name": "test_data_reporter_with_patched_values.py::test_base", + "outcome": "pass", + "markers": { + "test_base": 1, + "test_data_reporter_with_patched_values.py": 1, + "test_data_reporter_with_patched_values0": 1, + }, + "stage": "testcase", + "when": "call", + "id": str(SESSION_UUID), + "tag": "test", + "stop_info": "Testcase finished", + }, + { + "state": "finish", + "stage": "testcase", + "id": str(SESSION_UUID), + "name": "test_data_reporter_with_patched_values.py::test_base", + "tag": "test", + "label": "pytest", + }, + { + "state": "finish", + "stage": "session", + "id": str(SESSION_UUID), + "tag": "test", + "label": "pytest", + }, + ], + ), + ], +) +def test_data_reporter_with_patched_values( + pytester, run_mocked_pytest, session_uuid, patch_file_content, expected_result +): + runpytest, fluent_sender = run_mocked_pytest + pytester.makefile(".json", patch_file=json.dumps(patch_file_content)) + log_content = "Test running" + result = runpytest( + f"--session-uuid={session_uuid}", + "--stage-settings=patch_file.json", + "--extend-logging", + pyfile=f""" + import logging + + def test_base(): + logger = logging.getLogger() + logger.info("{log_content}") + assert True + """, + ) + call_args = fluent_sender.emit_with_time.call_args_list + call_args = [x[0][2] for x in call_args] + result.assert_outcomes(passed=1) + assert len(call_args) == len(expected_result) + for report, expected in zip(call_args, expected_result): + for key in expected.keys(): + assert key in report + if key in ["duration", "testId", "host"]: + continue + assert report[key] == expected[key] From 5b1c5394fcd1688916a5d7343adce4d05e890f2d Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 11:13:26 +0200 Subject: [PATCH 15/20] Fixing test --- tests/test_reporting_patching.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/test_reporting_patching.py b/tests/test_reporting_patching.py index 92fe437..7eea9f2 100644 --- a/tests/test_reporting_patching.py +++ b/tests/test_reporting_patching.py @@ -38,11 +38,6 @@ { "name": "test_data_reporter_with_patched_values.py::test_base", "outcome": "passed", - "markers": { - "test_base": 1, - "test_data_reporter_with_patched_values.py": 1, - "test_data_reporter_with_patched_values0": 1, - }, "stage": "testcase", "when": "call", "sessionId": str(SESSION_UUID), @@ -109,11 +104,6 @@ { "name": "test_data_reporter_with_patched_values.py::test_base", "outcome": "pass", - "markers": { - "test_base": 1, - "test_data_reporter_with_patched_values.py": 1, - "test_data_reporter_with_patched_values0": 1, - }, "stage": "testcase", "when": "call", "id": str(SESSION_UUID), @@ -165,6 +155,6 @@ def test_base(): for report, expected in zip(call_args, expected_result): for key in expected.keys(): assert key in report - if key in ["duration", "testId", "host"]: + if key in ["duration", "testId", "host", "markers"]: continue assert report[key] == expected[key] From 9551628252d0733fcea27a44d6317e778f8f9bf2 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 11:43:28 +0200 Subject: [PATCH 16/20] Fix wrong duration value for session finished --- src/pytest_fluent/plugin.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/pytest_fluent/plugin.py b/src/pytest_fluent/plugin.py index 2572e33..3595f5b 100644 --- a/src/pytest_fluent/plugin.py +++ b/src/pytest_fluent/plugin.py @@ -235,9 +235,14 @@ def pytest_sessionfinish( if not self.config.getoption("collectonly"): data = { "status": "finish", - "duration": time.time() - 0 - if self._session_start_time is None - else self._session_start_time, + "duration": ( + time.time() + - ( + 0 + if self._session_start_time is None + else self._session_start_time + ) + ), "stage": "session", "sessionId": self.session_uid, } From 6e607c29f6456c61da17d2b90595b102b98b99e9 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 11:57:26 +0200 Subject: [PATCH 17/20] Preparation for stage settings description in docs --- README.md | 34 +++++----- docs/conf.py | 2 +- docs/usage.md | 3 + docs/usage/stage_settings.md | 121 +++++++++++++++++++++++++++++++++++ tox.ini | 11 ++-- 5 files changed, 147 insertions(+), 24 deletions(-) create mode 100644 docs/usage/stage_settings.md diff --git a/README.md b/README.md index 1618dd2..7ae153f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ pip install pytest-fluent ## Usage -pytest-fluent-logging forwards meta data from pytest to Fluentd for further processing. The meta data are +pytest-fluent-logging forwards meta data from pytest to Fluentd for further processing. The meta data are + * unique session ID * unique test ID * status of the session respectively test case @@ -31,7 +32,7 @@ pytest-fluent-logging forwards meta data from pytest to Fluentd for further proc * `record_property` entries * custom testcase information * custom session information - + Furthermore, the Python logging instance can be extended in order to forward test case runtime logging. ```python @@ -43,7 +44,7 @@ def test_my_runtime_log(): assert value == 1 ``` -or +or ```python from logging import getLogger @@ -104,22 +105,22 @@ def provide_more_test_information() -> dict: The pytest CLI can be called with the following arguments in order to configure fluent-logging. -| argument | description | default | -|---------------------|------------------------------------------------------------------------------------|----------| -| --session-uuid | Use a custom externally created UUID, e.g. link a CI job with the pytest session. | | -| --fluentd-host | Fluentd host address. If not provided, a local Fluentd instance will be called. | | -| --fluentd-port | Fluent host port | 24224 | -| --fluentd-tag | Set a custom Fluentd tag | 'test' | -| --fluentd-label | Set a custom Fluentd label | 'pytest' | -| --fluentd-timestamp | Specify a Fluentd timestamp | None | -| --extend-logging | Extend the Python logging with a Fluent handler | False | -| --add-docstrings | Add test docstrings to testcase call messages | | +| argument | description | default | +| ------------------- | --------------------------------------------------------------------------------- | -------- | +| --session-uuid | Use a custom externally created UUID, e.g. link a CI job with the pytest session. | | +| --fluentd-host | Fluentd host address. If not provided, a local Fluentd instance will be called. | | +| --fluentd-port | Fluent host port | 24224 | +| --fluentd-tag | Set a custom Fluentd tag | 'test' | +| --fluentd-label | Set a custom Fluentd label | 'pytest' | +| --fluentd-timestamp | Specify a Fluentd timestamp | None | +| --extend-logging | Extend the Python logging with a Fluent handler | False | +| --add-docstrings | Add test docstrings to testcase call messages | | -### Ini Configuration Support +### Ini Configuration Support Default values of the CLI arguments for a project could also be defined in one of the following ini configuration files: -1. pytest.ini: Arguments are defined under pytest section in the file. This file takes precedence over all other configuration files even if empty. +1. pytest.ini: Arguments are defined under pytest section in the file. This file takes precedence over all other configuration files even if empty. ```python [pytest] @@ -224,7 +225,7 @@ Timestamps are added to the information if the ``--fluentd-timestamp`` option is addopts= --session-uuid="ac2f7600-a079-46cf-a7e0-6408b166364c" --fluentd-port=24224 --fluentd-host=localhost --fluentd-tag='dummytest' --fluentd-label='pytest' --fluentd-timestamp='@timestamp' --extend-logging ``` -The timestamp is added to each message. The value is in ISO 8601 format. A sample +The timestamp is added to each message. The value is in ISO 8601 format. A sample of the data collection from `test_addoptions.py` (as above) would look as below: ```json @@ -253,4 +254,3 @@ The changelog. ## Contributing We welcome any contributions, enhancements, and bug-fixes. Open an [issue](https://github.com/Rohde-Schwarz/pytest-fluent/issues) on [Github](https://github.com) and [submit a pull request](https://github.com/Rohde-Schwarz/pytest-fluent/pulls). - diff --git a/docs/conf.py b/docs/conf.py index 5bfad10..17b9f8a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,4 +19,4 @@ ] suppress_warnings = ["myst.header"] html_theme = "sphinx_rtd_theme" -html_static_path = ["images"] +# html_static_path = ["image"] diff --git a/docs/usage.md b/docs/usage.md index 28dbeda..a173413 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -4,3 +4,6 @@ :start-after: "## Usage" :end-before: "## Changelog" ``` + +```{include} ./usage/stage_settings.md +``` diff --git a/docs/usage/stage_settings.md b/docs/usage/stage_settings.md new file mode 100644 index 0000000..5501251 --- /dev/null +++ b/docs/usage/stage_settings.md @@ -0,0 +1,121 @@ +### Custom stage settings + +Sometimes, the default settings are not enough in order to forward the test information as needed. Thus, you can set custom stage settings +in order to fit your needs. + +You can set specific values for `all` stages or specific values for any used stage. In order to do so, call your test run with the `--stage-settings=YourFileName.json` parameter. The following example stage settings JSON file content + +```json +{ + "all": { + "tag": "run", + "label": "pytest", + "replace": {"keys": {"status": "state", "sessionId": "id"}}, + }, + "pytest_sessionstart": { + "tag": "run", + "label": "test", + "add": {"start_info": "Pytest started"}, + }, + "pytest_sessionfinish": { + "tag": "result", + "label": "test", + "add": {"stop_info": "Pytest finished"}, + }, + "pytest_runtest_logstart": { + "tag": "run", + "label": "testcase", + "add": {"start_info": "Testcase started"}, + }, + "pytest_runtest_logreport": { + "tag": "result", + "label": "testcase", + "replace": { + "values": {"passed": "pass", "failed": "fail"}, + }, + "add": {"stop_info": "Testcase finished"}, + }, + "logging": { + "replace": {"keys": {"message": "msg", "sessionId": "id"}}, + }, +} +``` + +will result in the following output + +```json +[ + { + "stage": "session", + "tag": "test", + "label": "pytest", + "state": "start", + "id": "3d82b514-60e2-4580-96ab-3daf5a5446c8" + }, + { + "stage": "testcase", + "testId": "6b5092ad-c905-4879-a70c-cb5b2a7df90d", + "name": "test_data_reporter_with_patched_values.py::test_base", + "tag": "test", + "label": "pytest", + "state": "start", + "id": "3d82b514-60e2-4580-96ab-3daf5a5446c8" + }, + { + "type": "logging", + "host": "MU763729", + "where": "test_data_reporter_with_patched_values.test_base", + "level": "INFO", + "stack_trace": "None", + "message": "Test running", + "testId": "6b5092ad-c905-4879-a70c-cb5b2a7df90d", + "stage": "testcase", + "id": "3d82b514-60e2-4580-96ab-3daf5a5446c8" + }, + { + "name": "test_data_reporter_with_patched_values.py::test_base", + "outcome": "pass", + "duration": 0.0034263000000001043, + "markers": { + "test_base": 1, + "test_data_reporter_with_patched_values.py": 1, + "test_data_reporter_with_patched_values0": 1 + }, + "stage": "testcase", + "when": "call", + "testId": "6b5092ad-c905-4879-a70c-cb5b2a7df90d", + "tag": "test", + "label": "pytest", + "id": "3d82b514-60e2-4580-96ab-3daf5a5446c8", + "stop_info": "Testcase finished" + }, + { + "stage": "testcase", + "testId": "6b5092ad-c905-4879-a70c-cb5b2a7df90d", + "name": "test_data_reporter_with_patched_values.py::test_base", + "tag": "test", + "label": "pytest", + "state": "finish", + "id": "3d82b514-60e2-4580-96ab-3daf5a5446c8" + }, + { + "duration": 1.3674933910369873, + "stage": "session", + "tag": "test", + "label": "pytest", + "state": "finish", + "id": "3d82b514-60e2-4580-96ab-3daf5a5446c8" + } +] +``` + +for this test case + +```python +import logging + +def test_base(): + logger = logging.getLogger() + logger.info("Test running") + assert True +``` diff --git a/tox.ini b/tox.ini index 78db77a..05871f2 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ description = Check code and tests for PEP 8 compliance and code complexity. skip_install = true envdir = {toxworkdir}/lint deps = - flake8 + flake8 flake8-colors isort >= 5.0 commands = @@ -51,7 +51,7 @@ description = Check docstrings for PEP 257 compliance (reST style). skip_install = true envdir = {toxworkdir}/lint deps = - flake8 + flake8 flake8-colors flake8-rst-docstrings commands = flake8 --select RST src/ @@ -62,7 +62,7 @@ envdir = {toxworkdir}/lint deps = flake8 flake8-colors -commands = +commands = flake8 --select F src/ tests/ [testenv:pylint] @@ -70,7 +70,7 @@ description = Find errors with static code analysis. envdir = {toxworkdir}/lint deps = pylint -commands = +commands = pylint --output-format=colorized --errors-only src/pytest_fluent [testenv:errors] @@ -137,10 +137,9 @@ commands = description = Generate API documentation. changedir = docs extras = docs -setenv = +setenv = WORKSPACE_ROOT_PATH = {toxinidir} commands = - {envbindir}/python patch_readme.py sphinx-build -W -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:version] From 296cc9405b9b7854cc269d4c3d1b6f8503464f5d Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 13:55:54 +0200 Subject: [PATCH 18/20] Adding more detailed documentation for stage settings --- docs/usage/stage_settings.md | 83 ++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/docs/usage/stage_settings.md b/docs/usage/stage_settings.md index 5501251..f8a6070 100644 --- a/docs/usage/stage_settings.md +++ b/docs/usage/stage_settings.md @@ -119,3 +119,86 @@ def test_base(): logger.info("Test running") assert True ``` + +#### Stage setting file + +Custom settings for each supported stage can be easily setup. You have to create a file with +a `.json` or `.yaml` extension and call pytest with this additional parameter `--stage-settings`. + +The file will be validated against a schema of supported values and in case of an error, a `jsonschema.ValidationError` +will be thrown. + +#### Stage settings + +##### Number of supported stage + +The following stages can be modified. + +* `pytest_sessionstart` +* `pytest_runtest_logstart` +* `pytest_runtest_logreport` +* `pytest_runtest_logfinish` +* `pytest_sessionfinish` +* `logging` + +These values are the keys for the dictionary object. Additionally, you can set also +a `all` key for convenience reasons to patch all keys at once. + +#### Patch events + +Probably, your stage setting would look like + +```json +{ + "pytest_sessionstart": { + "tag": "run", + "label": "pytest", + "replace": { + "keys": { + "status": "state", "sessionId": "id" + }, + "values": { + "passed": "pass" + } + }, + "add": {"start_info": "Pytest started"}, + } +} +``` + +The following values are supported + +| Key name | action | type | +| --------- | -------------------------------------------------------------------------------------- | ------ | +| `tag` | Set a specifc Fluent tag for this stage | `str` | +| `label` | Set a specifc Fluent label for this stage | `str` | +| `replace` | Replace key values from a dictionary and also replace some preset pytest result values | `dict` | +| `add` | Add new values to the result dictionary | `dict` | +| `drop` | Drop specific values from the result dictionary | `dict` | + +##### replace dictionary + +The `replace` patching action has two keys `keys` and `values` in order to replace either a key value or a result value. +See the following default values in order to get an idea about the content. + +At the moment, the following values can be changed + +* `passed` +* `failed` +* `skipped` +* `error` +* `start` +* `finish` +* `session` +* `testcase` + +#### Default values + +| stage | value | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `pytest_sessionstart` | 
{
    "status": "start",
    "stage": "session",
    "sessionId": "8d0d165d-5581-478c-ba0f-f7ec7d5bcbcf",
    "tag": "test",
    "label": "pytest"
}
| +| `pytest_runtest_logstart` | 
{
    "status": "start",
    "stage": "testcase",
    "sessionId": "8d0d165d-5581-478c-ba0f-f7ec7d5bcbcf",
    "testId": "9f0363fa-ef99-49c7-8a2d-6261e90acb00",
    "name": "test_data_reporter_with_patched_values.py::test_base",
    "tag": "test",
    "label": "pytest"
}
| +| `pytest_runtest_logreport` | 
{
    "name": "test_data_reporter_with_patched_values.py::test_base",
    "outcome": "passed",
    "duration": 0.0035069000000005346,
    "markers": {
      "test_base": 1,
      "test_data_reporter_with_patched_values.py": 1,
      "test_data_reporter_with_patched_values0": 1
    },
    "stage": "testcase",
    "when": "call",
    "sessionId": "8d0d165d-5581-478c-ba0f-f7ec7d5bcbcf",
    "testId": "9f0363fa-ef99-49c7-8a2d-6261e90acb00",
    "tag": "test",
    "label": "pytest"
}
| +| `pytest_runtest_logfinish` | 
{
    "status": "finish",
    "stage": "testcase",
    "sessionId": "8d0d165d-5581-478c-ba0f-f7ec7d5bcbcf",
    "testId": "9f0363fa-ef99-49c7-8a2d-6261e90acb00",
    "name": "test_data_reporter_with_patched_values.py::test_base",
    "tag": "test",
    "label": "pytest"
}
| +| `pytest_sessionfinish` | 
{
    "status": "finish",
    "duration": 1.5651893615722656,
    "stage": "session",
    "sessionId": "8d0d165d-5581-478c-ba0f-f7ec7d5bcbcf",
    "tag": "test",
    "label": "pytest"
}
| +| `logging` | 
{
    "type": "logging",
    "host": "hostname",
    "where": "test_data_reporter_with_patched_values.test_base",
    "level": "INFO",
    "stack_trace": "None",
    "message": "Test running",
    "sessionId": "8d0d165d-5581-478c-ba0f-f7ec7d5bcbcf",
    "testId": "9f0363fa-ef99-49c7-8a2d-6261e90acb00",
    "stage": "testcase"
}
| From 200ff4731640ff511b827a05f51e88fec1ef596f Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Thu, 15 Jun 2023 15:58:08 +0200 Subject: [PATCH 19/20] Editorial change --- docs/usage/stage_settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/stage_settings.md b/docs/usage/stage_settings.md index f8a6070..5945e91 100644 --- a/docs/usage/stage_settings.md +++ b/docs/usage/stage_settings.md @@ -63,7 +63,7 @@ will result in the following output }, { "type": "logging", - "host": "MU763729", + "host": "hostname", "where": "test_data_reporter_with_patched_values.test_base", "level": "INFO", "stack_trace": "None", From d7c80bc6babac02ccd61182dcd329674dae5b6b2 Mon Sep 17 00:00:00 2001 From: Carsten Sauerbrey Date: Fri, 16 Jun 2023 10:00:45 +0200 Subject: [PATCH 20/20] Extending stage settings docs --- docs/usage/stage_settings.md | 41 +++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/usage/stage_settings.md b/docs/usage/stage_settings.md index 5945e91..5612d4f 100644 --- a/docs/usage/stage_settings.md +++ b/docs/usage/stage_settings.md @@ -155,13 +155,16 @@ Probably, your stage setting would look like "label": "pytest", "replace": { "keys": { - "status": "state", "sessionId": "id" + "status": "state", + "sessionId": "id" }, "values": { "passed": "pass" } }, - "add": {"start_info": "Pytest started"}, + "add": { + "start_info": "Pytest started" + }, } } ``` @@ -176,7 +179,7 @@ The following values are supported | `add` | Add new values to the result dictionary | `dict` | | `drop` | Drop specific values from the result dictionary | `dict` | -##### replace dictionary +##### Replace dictionary The `replace` patching action has two keys `keys` and `values` in order to replace either a key value or a result value. See the following default values in order to get an idea about the content. @@ -192,6 +195,38 @@ At the moment, the following values can be changed * `session` * `testcase` +##### Use values from ARGV and ENV + +If you want to use data provided by the command line arguments or directly from environment variables, +use the following syntax for value strings. + +| Type | Syntax | +| ---- | ------------------------------ | +| ARGV | `""` | +| ENV | `"${USE_ENV}"` or `"$USE_ENV"` | + +Here is a simple example using both variants + +```json +{ + "pytest_sessionstart": { + "tag": "run", + "label": "pytest", + "replace": { + "keys": { + "tag": "", + "sessionId": "${ID}" + }, + "values": { + "passed": "$OUTCOME_PASSED" + } + } + } +} +``` + +The data will be mapped after starting the pytest session. + #### Default values | stage | value |