Merge pull request #132 from sbidoul/normalize-reqs-sbi

Additional normalization of requirements

news/132.feature

@@ -0,0 +1,3 @@
+Normalize distribution names in the generated lock files. This change, which will cause
+some changes in generated ``requirements*.txt`` files, was made following the change in
+setuptools 69 that started preserving underscores in distribution names.

pyproject.toml

@@ -33,7 +33,7 @@ dependencies=[
 dynamic = ["version"]
 
 [project.optional-dependencies]
-"test" = ["pytest", "pytest-cov", "pytest-xdist", "virtualenv"]
+"test" = ["pytest", "pytest-cov", "pytest-xdist", "virtualenv", "setuptools", "wheel"]
 "mypy" = ["mypy", "types-toml", "types-setuptools"]
 
 [project.scripts]

src/pip_deepfreeze/sync.py

@@ -22,6 +22,7 @@
     make_project_name_with_extras,
     make_requirements_path,
     make_requirements_paths,
+    normalize_req_line,
     open_with_rollback,
     run_commands,
 )
@@ -79,7 +80,7 @@ def sync(
                         print(parsed_req_line.raw_line, file=f)
             # output frozen dependencies of project
             for req_line in frozen_reqs:
-                print(req_line, file=f)
+                print(normalize_req_line(req_line), file=f)
     # uninstall unneeded dependencies, if asked to do so
     unneeded_req_names = sorted(
         set(str(s) for s in get_req_names(unneeded_reqs))

src/pip_deepfreeze/utils.py

@@ -10,6 +10,7 @@
 
 import httpx
 import typer
+from packaging.utils import canonicalize_name
 
 
 @contextlib.contextmanager
@@ -122,7 +123,7 @@ def make_project_name_with_extras(
 
 
 _NORMALIZE_REQ_LINE_RE = re.compile(
-    r"^(?P<name>[a-zA-Z0-9-_.\[\]]+)(?P<arobas>\s*@\s*)(?P<rest>.*)$"
+    r"^(?P<name>[a-zA-Z0-9-_.\[\]]+)(?P<arobas>\s*@\s*)?(?P<rest>.*)$"
 )
 
 
@@ -132,11 +133,18 @@ def normalize_req_line(req_line: str) -> str:
     This is a little hack because some requirements.txt generator such
     as pip-requirements-parser dont always generate the exact same
     output as pip freeze.
+
+    Moreover, setuptools 69 stopped normalizing the name in the wheels it generates,
+    so users of different setuptools version can see different pip freeze results.
     """
     mo = _NORMALIZE_REQ_LINE_RE.match(req_line)
     if not mo:
         return req_line
-    return mo.group("name") + " @ " + mo.group("rest")
+    return (
+        canonicalize_name(mo.group("name"))
+        + (" @ " if mo.group("arobas") else "")
+        + mo.group("rest")
+    )
 
 
 def get_temp_path_in_dir(dir: Path, prefix: str, suffix: str) -> Path:

tests/conftest.py

@@ -65,6 +65,7 @@ def testpkgs(tmp_path_factory):
             extras_require={"c": ["pkgc<0.0.3"], "b": ["pkgb"]},
         ),
         dict(name="pkge", install_requires=["pkgd[b,c]"]),
+        dict(name="pkg-F.G_H"),  # to test name normalization
     ]
 
     testpkgs_dir = tmp_path_factory.mktemp("testpkgs")
@@ -81,6 +82,7 @@ def testpkgs(tmp_path_factory):
                 "-m",
                 "pip",
                 "wheel",
+                "--no-build-isolation",
                 "--no-deps",
                 str(setup_py_dir),
                 "--wheel-dir",

tests/test_sync.py

@@ -47,6 +47,39 @@ def test_sync(virtualenv_python, testpkgs, tmp_path):
     )
 
 
+def test_sync_normalization(virtualenv_python, testpkgs, tmp_path):
+    (tmp_path / "requirements.txt.in").write_text(
+        textwrap.dedent(
+            f"""\
+            --no-index
+            --find-links {testpkgs}
+            """
+        )
+    )
+    (tmp_path / "pyproject.toml").write_text(
+        textwrap.dedent(
+            """\
+            [project]
+            name = "theproject"
+            version = "1.0"
+            dependencies = ["pkg-F.G_H"]
+            """
+        )
+    )
+    subprocess.check_call(
+        [sys.executable, "-m", "pip_deepfreeze", "--python", virtualenv_python, "sync"],
+        cwd=tmp_path,
+    )
+    assert (tmp_path / "requirements.txt").read_text() == textwrap.dedent(
+        f"""\
+        # frozen requirements generated by pip-deepfreeze
+        --no-index
+        --find-links {testpkgs}
+        pkg-f-g-h==0.0.0
+        """
+    )
+
+
 def test_sync_no_in_req(virtualenv_python, tmp_path):
     (tmp_path / "setup.py").write_text(
         textwrap.dedent(

tests/test_utils.py

@@ -140,6 +140,9 @@ def test_make_project_name_with_extras(project_name, extras, expected):
     [
         ("prj", "prj"),
         ("prj==1.0", "prj==1.0"),
+        ("prj.X-Y_Z", "prj-x-y-z"),
+        ("prj.X-Y_Z==1.0", "prj-x-y-z==1.0"),
+        ("prj.X-Y_Z@http://g.c/o/p", "prj-x-y-z @ http://g.c/o/p"),
         ("name @https://g.c/o/p@branch", "name @ https://g.c/o/p@branch"),
         ("name@https://g.c/o/p@branch", "name @ https://g.c/o/p@branch"),
         ("name[extra] @https://g.c/o/p@branch", "name[extra] @ https://g.c/o/p@branch"),