From 0ff1c15b38bb69877b2e4181339dcdf855c806f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 2 Jan 2024 18:58:54 +0100 Subject: [PATCH 1/5] Further normalize requirement lines --- src/pip_deepfreeze/utils.py | 12 ++++++++++-- tests/test_utils.py | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/pip_deepfreeze/utils.py b/src/pip_deepfreeze/utils.py index 361a257..dc53c2c 100644 --- a/src/pip_deepfreeze/utils.py +++ b/src/pip_deepfreeze/utils.py @@ -10,6 +10,7 @@ import httpx import typer +from packaging.utils import canonicalize_name @contextlib.contextmanager @@ -122,7 +123,7 @@ def make_project_name_with_extras( _NORMALIZE_REQ_LINE_RE = re.compile( - r"^(?P[a-zA-Z0-9-_.\[\]]+)(?P\s*@\s*)(?P.*)$" + r"^(?P[a-zA-Z0-9-_.\[\]]+)(?P\s*@\s*)?(?P.*)$" ) @@ -132,11 +133,18 @@ def normalize_req_line(req_line: str) -> str: This is a little hack because some requirements.txt generator such as pip-requirements-parser dont always generate the exact same output as pip freeze. + + Moreover, setuptools 69 stopped normalizing the name in the wheels it generates, + so users of different setuptools version can see different pip freeze results. """ mo = _NORMALIZE_REQ_LINE_RE.match(req_line) if not mo: return req_line - return mo.group("name") + " @ " + mo.group("rest") + return ( + canonicalize_name(mo.group("name")) + + (" @ " if mo.group("arobas") else "") + + mo.group("rest") + ) def get_temp_path_in_dir(dir: Path, prefix: str, suffix: str) -> Path: diff --git a/tests/test_utils.py b/tests/test_utils.py index d6190ce..8898043 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -140,6 +140,9 @@ def test_make_project_name_with_extras(project_name, extras, expected): [ ("prj", "prj"), ("prj==1.0", "prj==1.0"), + ("prj.X-Y_Z", "prj-x-y-z"), + ("prj.X-Y_Z==1.0", "prj-x-y-z==1.0"), + ("prj.X-Y_Z@http://g.c/o/p", "prj-x-y-z @ http://g.c/o/p"), ("name @https://g.c/o/p@branch", "name @ https://g.c/o/p@branch"), ("name@https://g.c/o/p@branch", "name @ https://g.c/o/p@branch"), ("name[extra] @https://g.c/o/p@branch", "name[extra] @ https://g.c/o/p@branch"), From bcc2405f7a59ad9cbab64f9a55e936f216fc97ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 5 Jan 2024 18:43:45 +0100 Subject: [PATCH 2/5] Test requirements.txt name normalization --- tests/conftest.py | 1 + tests/test_sync.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 60a2cfd..b1e6a12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,6 +65,7 @@ def testpkgs(tmp_path_factory): extras_require={"c": ["pkgc<0.0.3"], "b": ["pkgb"]}, ), dict(name="pkge", install_requires=["pkgd[b,c]"]), + dict(name="pkg-F.G_H"), # to test name normalization ] testpkgs_dir = tmp_path_factory.mktemp("testpkgs") diff --git a/tests/test_sync.py b/tests/test_sync.py index 97599e2..94240d2 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -47,6 +47,39 @@ def test_sync(virtualenv_python, testpkgs, tmp_path): ) +def test_sync_normalization(virtualenv_python, testpkgs, tmp_path): + (tmp_path / "requirements.txt.in").write_text( + textwrap.dedent( + f"""\ + --no-index + --find-links {testpkgs} + """ + ) + ) + (tmp_path / "pyproject.toml").write_text( + textwrap.dedent( + """\ + [project] + name = "theproject" + version = "1.0" + dependencies = ["pkg-F.G_H"] + """ + ) + ) + subprocess.check_call( + [sys.executable, "-m", "pip_deepfreeze", "--python", virtualenv_python, "sync"], + cwd=tmp_path, + ) + assert (tmp_path / "requirements.txt").read_text() == textwrap.dedent( + f"""\ + # frozen requirements generated by pip-deepfreeze + --no-index + --find-links {testpkgs} + pkg-f-g-h==0.0.0 + """ + ) + + def test_sync_no_in_req(virtualenv_python, tmp_path): (tmp_path / "setup.py").write_text( textwrap.dedent( From 68811e58ff153176a1694e26b4f03c05e3af1760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 5 Jan 2024 18:44:10 +0100 Subject: [PATCH 3/5] Normalize names in requirements.txt --- src/pip_deepfreeze/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pip_deepfreeze/sync.py b/src/pip_deepfreeze/sync.py index af094ff..0f8b991 100644 --- a/src/pip_deepfreeze/sync.py +++ b/src/pip_deepfreeze/sync.py @@ -22,6 +22,7 @@ make_project_name_with_extras, make_requirements_path, make_requirements_paths, + normalize_req_line, open_with_rollback, run_commands, ) @@ -79,7 +80,7 @@ def sync( print(parsed_req_line.raw_line, file=f) # output frozen dependencies of project for req_line in frozen_reqs: - print(req_line, file=f) + print(normalize_req_line(req_line), file=f) # uninstall unneeded dependencies, if asked to do so unneeded_req_names = sorted( set(str(s) for s in get_req_names(unneeded_reqs)) From ecbb8e80fa8943778d01d8ac473ccd341572bf22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 5 Jan 2024 18:44:32 +0100 Subject: [PATCH 4/5] Don't use build isolation when building test wheels For better test performance. --- pyproject.toml | 2 +- tests/conftest.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3dfd90f..b445009 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies=[ dynamic = ["version"] [project.optional-dependencies] -"test" = ["pytest", "pytest-cov", "pytest-xdist", "virtualenv"] +"test" = ["pytest", "pytest-cov", "pytest-xdist", "virtualenv", "setuptools", "wheel"] "mypy" = ["mypy", "types-toml", "types-setuptools"] [project.scripts] diff --git a/tests/conftest.py b/tests/conftest.py index b1e6a12..342abff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -82,6 +82,7 @@ def testpkgs(tmp_path_factory): "-m", "pip", "wheel", + "--no-build-isolation", "--no-deps", str(setup_py_dir), "--wheel-dir", From c8310c7b2ed340ec77e0f45dc1f30cf4530316dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Fri, 5 Jan 2024 19:03:59 +0100 Subject: [PATCH 5/5] Add news --- news/132.feature | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/132.feature diff --git a/news/132.feature b/news/132.feature new file mode 100644 index 0000000..1c61eae --- /dev/null +++ b/news/132.feature @@ -0,0 +1,3 @@ +Normalize distribution names in the generated lock files. This change, which will cause +some changes in generated ``requirements*.txt`` files, was made following the change in +setuptools 69 that started preserving underscores in distribution names.