Improve F6401: cannot-enumerate-pytest-fixtures:

* Capture and return `stdout` and `stderr` (instead of trashing them)
* Fix the 'duplicate-path' error by `relative`-izing the paths before `union`ing them
* Update tests to test for both conditions

Additionally:
* `pylint-pytest` does not support `pylint>=3`
* Fix two `used-before-assignment` pylint issues (`stdout`, `stderr`) 🤪

Signed-off-by: Stavros Ntentos <[email protected]>

pylint_pytest/checkers/fixture.py

@@ -1,3 +1,4 @@
+import io
 import os
 import sys
 from pathlib import Path
@@ -61,7 +62,7 @@ class FixtureChecker(BasePytestChecker):
         'F6401': (
             (
                 'pylint-pytest plugin cannot enumerate and collect pytest fixtures. '
-                'Please run `pytest --fixtures --collect-only %s` and resolve any potential syntax error or package dependency issues'
+                'Please run `pytest --fixtures --collect-only %s` and resolve any potential syntax error or package dependency issues. stdout: %s. stderr: %s.'
             ),
             'cannot-enumerate-pytest-fixtures',
             'Used when pylint-pytest has been unable to enumerate and collect pytest fixtures.',
@@ -110,11 +111,12 @@ def visit_module(self, node):
                 is_test_module = True
                 break
 
+        stdout, stderr = sys.stdout, sys.stderr
         try:
-            with open(os.devnull, 'w') as devnull:
+            with (io.StringIO() as captured_stdout, io.StringIO() as captured_stderr):
                 # suppress any future output from pytest
-                stdout, stderr = sys.stdout, sys.stderr
-                sys.stderr = sys.stdout = devnull
+                sys.stderr = captured_stderr
+                sys.stdout = captured_stdout
 
                 # run pytest session with customized plugin to collect fixtures
                 fixture_collector = FixtureCollector()
@@ -146,9 +148,17 @@ def visit_module(self, node):
                     )
                 )
                 if (ret != pytest.ExitCode.OK or legitimate_failure_paths) and is_test_module:
+                    files_to_report = {
+                        str(Path(x).absolute().relative_to(Path.cwd())) for x in legitimate_failure_paths | {node.file}
+                    }
+
                     self.add_message(
                         'cannot-enumerate-pytest-fixtures',
-                        args=' '.join(legitimate_failure_paths | {node.file}),
+                        args=(
+                            ' '.join(files_to_report),
+                            captured_stdout.getvalue(),
+                            captured_stderr.getvalue(),
+                        ),
                         node=node,
                     )
         finally:

setup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 from os import path
@@ -24,7 +24,7 @@
     long_description_content_type='text/markdown',
     packages=find_packages(exclude=['tests', 'sandbox']),
     install_requires=[
-        'pylint',
+        'pylint<3',
         'pytest>=4.6',
     ],
     python_requires='>=3.6',

tests/test_cannot_enumerate_fixtures.py

@@ -1,5 +1,7 @@
+import re
+
 import pytest
-from pylint.checkers.variables import VariablesChecker
+
 from base_tester import BasePytestTester
 from pylint_pytest.checkers.fixture import FixtureChecker
 
@@ -13,7 +15,39 @@ def test_no_such_package(self, enable_plugin):
         self.run_linter(enable_plugin)
         self.verify_messages(1 if enable_plugin else 0)
 
+        if enable_plugin:
+            msg = self.MESSAGES[0]
+
+            # Asserts/Fixes duplicate filenames in output:
+            # https://github.com/reverbc/pylint-pytest/pull/22/files#r698204470
+            filename_arg = msg.args[0]
+            assert len(re.findall(r"\.py", filename_arg)) == 1
+
+            # Asserts that path is relative (usually to the root of the repository).
+            assert filename_arg[0] != "/"
+
+            # Assert `stdout` is non-empty.
+            assert msg.args[1]
+            # Assert `stderr` is empty (pytest runs stably, even though fixture collection fails).
+            assert not msg.args[2]
+
     @pytest.mark.parametrize('enable_plugin', [True, False])
     def test_import_corrupted_module(self, enable_plugin):
         self.run_linter(enable_plugin)
         self.verify_messages(1 if enable_plugin else 0)
+
+        if enable_plugin:
+            msg = self.MESSAGES[0]
+
+            # ... somehow, since `import_corrupted_module.py` imports `no_such_package.py`
+            # both of their names are returned in the message.
+            filename_arg = msg.args[0]
+            assert len(re.findall(r"\.py", filename_arg)) == 2
+
+            # Asserts that paths are relative (usually to the root of the repository).
+            assert not [x for x in filename_arg.split(' ') if x[0] == "/"]
+
+            # Assert `stdout` is non-empty.
+            assert msg.args[1]
+            # Assert `stderr` is empty (pytest runs stably, even though fixture collection fails).
+            assert not msg.args[2]