From c49fa00d873aa78eecea4a892d01dce2755c289b Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Tue, 3 Oct 2023 11:54:20 -0500 Subject: [PATCH 1/2] Add support for PEP658 core metadata --- README.md | 13 ++++++++++++- dumb_pypi/main.py | 3 +++ dumb_pypi/templates/package.html | 3 +++ testing.py | 16 +++++++++++++++- tests/conftest.py | 2 +- tests/integration_test.py | 19 +++++++++++++++++++ tests/main_test.py | 2 ++ 7 files changed, 55 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8d7051ba0bd..770ec807daa 100644 --- a/README.md +++ b/README.md @@ -108,9 +108,20 @@ format listed above of one filename per line, format your file with one JSON object per line, like this: ```json -{"filename": "dumb-init-1.1.2.tar.gz", "hash": "md5=", "requires_dist": ["cfgv"], "requires_python": ">=3.6", "uploaded_by": "ckuehl", "upload_timestamp": 1512539924, "yanked_reason": null} +{"filename": "dumb-init-1.1.2.tar.gz", "hash": "sha256=", "requires_python": ">=3.6", "uploaded_by": "ckuehl", "upload_timestamp": 1512539924, "yanked_reason": null, "core_metadata": "sha256="} ``` +| Key | Required? | Description | +| -------------------- | --------- | ----------- | +| `filename` | Yes | Name of the file | +| `hash` | No | Hash of the file in the format `=` | +| `requires_python` | No | Python requirement string for the package ([PEP345](https://peps.python.org/pep-0345/#requires-python)) | +| `core_metadata` | No | Either string `"true"` or a string in the format `=` to indicate metadata is available for this file by appending `.metadata` to the file URL ([PEP658](https://peps.python.org/pep-0658/), [PEP714](https://peps.python.org/pep-0714/)) | +| `uploaded_by` | No | Freeform text to indicate an uploader of the package; only shown on web UI | +| `upload_timestamp` | No | UNIX timestamp to indicate upload time of the package | +| `yanked_reason` | No | Freeform text to indicate the package is yanked for the given reason ([PEP592](https://peps.python.org/pep-0592/)) | +| `requires_dist` | No | _(Deprecated)_ Array of requires_dist dependencies ([PEP345](https://peps.python.org/pep-0345/#requires-python)), used only in the JSON API; consider using `dist_info_metadata` instead | + The `filename` key is required. All other keys are optional and will be used to provide additional information in your generated repository. This extended information can be useful to determine, for example, who uploaded a package. diff --git a/dumb_pypi/main.py b/dumb_pypi/main.py index 02766a82ca1..c086c46a025 100644 --- a/dumb_pypi/main.py +++ b/dumb_pypi/main.py @@ -106,6 +106,7 @@ class Package(NamedTuple): hash: str | None requires_dist: tuple[str, ...] | None requires_python: str | None + core_metadata: str | None upload_timestamp: int | None uploaded_by: str | None yanked_reason: str | None @@ -197,6 +198,7 @@ def create( upload_timestamp: int | None = None, uploaded_by: str | None = None, yanked_reason: str | None = None, + core_metadata: str | None = None, ) -> Package: if not re.match(r'[a-zA-Z0-9_\-\.\+]+$', filename) or '..' in filename: raise ValueError(f'Unsafe package name: {filename}') @@ -210,6 +212,7 @@ def create( hash=hash, requires_dist=tuple(requires_dist) if requires_dist is not None else None, requires_python=requires_python, + core_metadata=core_metadata, upload_timestamp=upload_timestamp, uploaded_by=uploaded_by, yanked_reason=yanked_reason, diff --git a/dumb_pypi/templates/package.html b/dumb_pypi/templates/package.html index f9c479fa8af..d1920fcc240 100644 --- a/dumb_pypi/templates/package.html +++ b/dumb_pypi/templates/package.html @@ -26,6 +26,9 @@

{{package_name}}

{%- if file.requires_python %} data-requires-python="{{file.requires_python}}" {%- endif %} + {%- if file.core_metadata %} + data-core-metadata="{{file.core_metadata}}" + {%- endif %} {%- if file.yanked_reason %} data-yanked="{{file.yanked_reason}}" {%- endif %} diff --git a/testing.py b/testing.py index 8fd55206f26..d9be4ae1ac2 100644 --- a/testing.py +++ b/testing.py @@ -6,6 +6,7 @@ import subprocess import sys import tempfile +import zipfile from typing import NamedTuple from dumb_pypi import main @@ -26,6 +27,7 @@ def as_json(self): return json.dumps({ 'filename': self.filename, 'requires_python': self.requires_python, + 'core_metadata': 'true' if self.filename.endswith('.whl') else None, }) @@ -50,4 +52,16 @@ def make_package(package: FakePackage, path: str) -> None: subprocess.check_call((sys.executable, setup_py) + args, cwd=td) created, = os.listdir(os.path.join(td, 'dist')) - shutil.move(os.path.join(td, 'dist', created), os.path.join(path, package.filename)) + dest = os.path.join(path, package.filename) + shutil.move(os.path.join(td, 'dist', created), dest) + + # Extract PEP658 metadata. + if dest.endswith('.whl'): + with zipfile.ZipFile(dest) as zf: + metadata_path, = ( + name + for name in zf.namelist() + if name.endswith('.dist-info/METADATA') + ) + with open(f'{dest}.metadata', 'wb') as f: + f.write(zf.read(metadata_path)) diff --git a/tests/conftest.py b/tests/conftest.py index 370b3fffbef..6404c8b242e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ import requests -PIP_TEST_VERSION = '23.1.2' +PIP_TEST_VERSION = '23.2.1' UrlAndPath = collections.namedtuple('UrlAndPath', ('url', 'path')) diff --git a/tests/integration_test.py b/tests/integration_test.py index 604d7d5365b..9aaa974ee7e 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -136,3 +136,22 @@ def test_pip_respects_requires_python(tmpdir, tmpweb, pip): ) downloaded_package, = tmpdir.listdir(fil=os.path.isfile) assert downloaded_package.basename == 'foo-2.tar.gz' + + +def test_pip_uses_core_metadata(capfd, tmpdir, tmpweb, pip): + install_packages( + tmpweb.path, + (FakePackage('foo-1-py2.py3-none-any.whl'),) + ) + pip_download( + pip, + tmpweb.url + '/simple', + tmpdir.strpath, + 'foo', + ) + downloaded_package, = tmpdir.listdir(fil=os.path.isfile) + assert downloaded_package.basename == 'foo-1-py2.py3-none-any.whl' + assert ( + f'Obtaining dependency information for foo from {tmpweb.url}/pool/foo-1-py2.py3-none-any.whl.metadata' + in capfd.readouterr().out + ) diff --git a/tests/main_test.py b/tests/main_test.py index 1a300fa6a6e..40dab9ed8c1 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -167,6 +167,7 @@ def test_input_json_all_info(): hash='sha256=deadbeef', requires_dist=['aspy.yaml'], requires_python='>=3.6', + core_metadata="sha256=badc0ffee", uploaded_by='asottile', upload_timestamp=1528586805, yanked_reason='Wrong Python Pinning', @@ -177,6 +178,7 @@ def test_input_json_all_info(): 'hash': 'sha256=deadbeef', 'requires_dist': ('aspy.yaml',), 'requires_python': '>=3.6', + 'core_metadata': 'sha256=badc0ffee', 'uploaded_by': 'asottile', 'upload_timestamp': 1528586805, 'yanked_reason': 'Wrong Python Pinning', From 1a2868beda59cfd79478d4cf31782f296aa5167e Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Tue, 3 Oct 2023 12:05:35 -0500 Subject: [PATCH 2/2] Fix reference to old key name dist_info_metadata --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 770ec807daa..3f003ea01b2 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ object per line, like this: | `uploaded_by` | No | Freeform text to indicate an uploader of the package; only shown on web UI | | `upload_timestamp` | No | UNIX timestamp to indicate upload time of the package | | `yanked_reason` | No | Freeform text to indicate the package is yanked for the given reason ([PEP592](https://peps.python.org/pep-0592/)) | -| `requires_dist` | No | _(Deprecated)_ Array of requires_dist dependencies ([PEP345](https://peps.python.org/pep-0345/#requires-python)), used only in the JSON API; consider using `dist_info_metadata` instead | +| `requires_dist` | No | _(Deprecated)_ Array of requires_dist dependencies ([PEP345](https://peps.python.org/pep-0345/#requires-python)), used only in the JSON API; consider using `core_metadata` instead | The `filename` key is required. All other keys are optional and will be used to provide additional information in your generated repository. This extended