Parse additional fields from metadata.

Author: thatch

README.md

@@ -7,6 +7,50 @@ things, with as minimal dependencies as possible:
 1. Support just enough metadata to be able to look up deps.
 2. Do "the thing that pip does" when deciding what dist-info dir to look at.
 
+# Usage
+
+Example snippet to show how to get the metadata from a wheel. 
+
+```python
+from zipfile import ZipFile
+from metadata_please import basic_metadata_from_wheel
+
+zf = ZipFile('somepkg.whl')
+print(basic_metadata_from_wheel(zf, "somepkg"))
+```
+
+### Output
+
+```
+BasicMetadata(
+    reqs=[
+        'build',
+        'setuptools',
+        'pip',
+        'imperfect<1',
+        'tomlkit<1',
+        'click~=8.0',
+        'GitPython~=3.1.18',
+        'metatron==0.60.0',
+        'pkginfo~=1.9',
+        'pyyaml~=6.0',
+        'runez~=5.2',
+        'pathspec<1',
+        'virtualenv<20.21',
+        'tox~=3.28',
+        'requests~=2.27',
+        'urllib3~=1.26'
+    ],
+    provides_extra=frozenset(),
+    name='pynt',
+    requires_python='>=3.6',
+    url='https://stash.corp.netflix.com/projects/NFPY/repos/pynt/browse',
+    project_urls={}
+)
+```
+
+The metadata can be extracted from a `wheel`, `sdist` (zip or tarball). Check [`__init__.py`](metadata_please/__init__.py) file for all available functions.
+
 # Version Compat
 
 Usage of this library should work back to 3.7, but development (and mypy

metadata_please/sdist.py

@@ -31,7 +31,7 @@ def basic_metadata_from_zip_sdist(zf: ZipFile) -> BasicMetadata:
     requires = [f for f in zf.namelist() if f.endswith("/requires.txt")]
     requires.sort(key=len)
     if not requires:
-        return BasicMetadata((), frozenset())
+        return BasicMetadata((), frozenset(), "-")
 
     data = zf.read(requires[0])
     assert data is not None

metadata_please/source_checkout.py

@@ -11,8 +11,10 @@
 
 Notably, does not read nontrivial setup.py or attempt to emulate anything that can't be read staticly.
 """
+
 import ast
 import re
+from dataclasses import asdict
 from pathlib import Path
 
 try:
@@ -81,6 +83,54 @@ def from_pep621_checkout(path: Path) -> bytes:
         for i in v:
             buf.append("Requires-Dist: " + merge_extra_marker(extra_name, i) + "\n")
 
+    name = doc.get("project", {}).get("name")
+    if name:
+        buf.append(f"Name: {name}\n")
+
+    # Version
+    version = doc.get("project", {}).get("version")
+    if version:
+        buf.append(f"Version: {version}\n")
+
+    # Requires-Python
+    requires_python = doc.get("project", {}).get("requires-python")
+    if requires_python:
+        buf.append(f"Requires-Python: {requires_python}\n")
+
+    # Project-URL
+    urls = doc.get("project", {}).get("urls")
+    if urls:
+        for k, v in urls.items():
+            buf.append(f"Project-URL: {k}={v}\n")
+
+    # Author
+    authors = doc.get("project", {}).get("authors")
+    if authors:
+        for author in authors:
+            try:
+                buf.append(f"Author: {author.get('name')}\n")
+            except AttributeError:
+                pass
+            try:
+                buf.append(f"Author-Email: {author.get('email')}\n")
+            except AttributeError:
+                pass
+
+    # Summary
+    summary = doc.get("project", {}).get("description")
+    if summary:
+        buf.append(f"Summary: {summary}\n")
+
+    # Description
+    description = doc.get("project", {}).get("readme")
+    if description:
+        buf.append(f"Description: {description}\n")
+
+    # Keywords
+    keywords = doc.get("project", {}).get("keywords")
+    if keywords:
+        buf.append(f"Keywords: {keywords}\n")
+
     return "".join(buf).encode("utf-8")
 
 
@@ -193,6 +243,45 @@ def from_poetry_checkout(path: Path) -> bytes:
                 f"Requires-Dist: {vi}{constraints}{merge_extra_marker(k, markers)}"
             )
 
+    name = doc.get("tool", {}).get("poetry", {}).get("name")
+    if name:
+        buf.append(f"Name: {name}\n")
+
+    # Version
+    version = doc.get("tool", {}).get("poetry", {}).get("version")
+    if version:
+        buf.append(f"Version: {version}\n")
+
+    # Requires-Python
+    requires_python = doc.get("tool", {}).get("poetry", {}).get("requires-python")
+    if requires_python:
+        buf.append(f"Requires-Python: {requires_python}\n")
+
+    # Project-URL
+    url = doc.get("tool", {}).get("poetry", {}).get("homepage")
+    if url:
+        buf.append(f"Home-Page: {url}\n")
+
+    # Author
+    authors = doc.get("tool", {}).get("poetry", {}).get("authors")
+    if authors:
+        buf.append(f"Author: {authors}\n")
+
+    # Summary
+    summary = doc.get("tool", {}).get("poetry", {}).get("description")
+    if summary:
+        buf.append(f"Summary: {summary}\n")
+
+    # Description
+    description = doc.get("tool", {}).get("poetry", {}).get("readme")
+    if description:
+        buf.append(f"Description: {description}\n")
+
+    # Keywords
+    keywords = doc.get("tool", {}).get("poetry", {}).get("keywords")
+    if keywords:
+        buf.append(f"Keywords: {keywords}\n")
+
     return "".join(buf).encode("utf-8")
 
 
@@ -206,6 +295,55 @@ def from_setup_cfg_checkout(path: Path) -> bytes:
     rc.read_string(data)
 
     buf: list[str] = []
+    try:
+        buf.append(f"Name: {rc.get('metadata', 'name')}\n")
+    except (NoOptionError, NoSectionError):
+        pass
+
+    # Requires-Python
+    try:
+        buf.append(f"Requires-Python: {rc.get('options', 'python_requires')}\n")
+    except (NoOptionError, NoSectionError):
+        pass
+
+    # Home-Page
+    try:
+        buf.append(f"Home-Page: {rc.get('metadata', 'url')}\n")
+    except (NoOptionError, NoSectionError):
+        pass
+
+    # Author
+    try:
+        buf.append(f"Author: {rc.get('metadata', 'author')}\n")
+    except (NoOptionError, NoSectionError):
+        pass
+
+    # Author-Email
+    try:
+        buf.append(f"Author-Email: {rc.get('metadata', 'author_email')}\n")
+    except (NoOptionError, NoSectionError):
+        pass
+
+    # Summary
+    try:
+        buf.append(f"Summary: {rc.get('metadata', 'description')}\n")
+    except (NoOptionError, NoSectionError):
+        pass
+
+    # Description
+    try:
+        buf.append(f"Description: {rc.get('metadata', 'long_description')}\n")
+    except (NoOptionError, NoSectionError):
+        pass
+
+    # Description-Content-Type
+    try:
+        buf.append(
+            f"Description-Content-Type: {rc.get('metadata', 'long_description_content_type')}\n"
+        )
+    except (NoOptionError, NoSectionError):
+        pass
+
     try:
         for dep in rc.get("options", "install_requires").splitlines():
             dep = dep.strip()
@@ -229,6 +367,8 @@ def from_setup_cfg_checkout(path: Path) -> bytes:
                         "Requires-Dist: " + merge_extra_marker(extra_name, i) + "\n"
                     )
 
+    # TODO name requires_python url project_urls
+
     return "".join(buf).encode("utf-8")
 
 
@@ -252,6 +392,7 @@ def from_setup_py_checkout(path: Path) -> bytes:
             raise ValueError("Complex setup call can't extract reqs")
         for dep in r:
             buf.append(f"Requires-Dist: {dep}\n")
+
     er = v.setup_call_args.get("extras_require")
     if er:
         if er is UNKNOWN:
@@ -262,6 +403,31 @@ def from_setup_py_checkout(path: Path) -> bytes:
             for i in deps:
                 buf.append("Requires-Dist: " + merge_extra_marker(extra_name, i) + "\n")
 
+    n = v.setup_call_args.get("name")
+    if n:
+        if n is UNKNOWN:
+            raise ValueError("Complex setup call can't extract name")
+        buf.append(f"Name: {n}\n")
+
+    n = v.setup_call_args.get("python_requires")
+    if n:
+        if n is UNKNOWN:
+            raise ValueError("Complex setup call can't extract python_requires")
+        buf.append(f"Requires-Python: {n}\n")
+
+    n = v.setup_call_args.get("url")
+    if n:
+        if n is UNKNOWN:
+            raise ValueError("Complex setup call can't extract url")
+        buf.append(f"Home-Page: {n}\n")
+
+    n = v.setup_call_args.get("project_urls")
+    if n:
+        if n is UNKNOWN:
+            raise ValueError("Complex setup call can't extract project_urls")
+        for k, v in n.items():
+            buf.append(f"Project-URL: {k}={v}\n")
+
     return "".join(buf).encode("utf-8")
 
 
@@ -270,6 +436,11 @@ def basic_metadata_from_source_checkout(path: Path) -> BasicMetadata:
 
 
 if __name__ == "__main__":  # pragma: no cover
+    import json
     import sys
 
-    print(basic_metadata_from_source_checkout(Path(sys.argv[1])))
+    md = basic_metadata_from_source_checkout(Path(sys.argv[1]))
+    if md.reqs or md.name:
+        print(json.dumps(asdict(md), default=list))
+    else:
+        sys.exit(1)

metadata_please/source_checkout_ast.py

@@ -11,7 +11,7 @@
 """
 
 import ast
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 
 # Copied from orig-index
@@ -93,12 +93,30 @@ def __init__(self) -> None:
         super().__init__()
         self.setup_call_args: Optional[Dict[str, Any]] = None
         self.setup_call_kwargs: Optional[bool] = None
+        self.stack: List[ast.AST] = []
+
+    def locate_assignment_value(self, body: List[ast.AST], name: ast.Name) -> Any:
+        for node in body:
+            if isinstance(node, ast.Assign):
+                if node.targets == [name]:
+                    return node.value
+        return UNKNOWN
+
+    def visit(self, node: ast.AST) -> Any:
+        self.stack.append(node)
+        try:
+            return super().visit(node)
+        finally:
+            self.stack.pop()
 
     def visit_Call(self, node: ast.Call) -> None:
         # .func (expr, can just be name)
         # .args
         # .keywords
-        qn = self.qualified_name(node.func)
+        try:
+            qn = self.qualified_name(node.func)
+        except ValueError:
+            return
         if qn in ("setuptools.setup", "distutils.setup"):
             self.setup_call_args = d = {}
             self.setup_call_kwargs = False
@@ -108,7 +126,18 @@ def visit_Call(self, node: ast.Call) -> None:
                     self.setup_call_kwargs = True
                 else:
                     try:
-                        d[k.arg] = ast.literal_eval(k.value)
+                        if isinstance(k.value, ast.Name):
+                            print(self.stack)
+                            for p in self.stack[::-1]:
+                                if hasattr(p, "body"):
+                                    v = self.locate_assignment_value(p.body, k.value)
+                                    if v is not UNKNOWN:
+                                        d[k.arg] = ast.literal_eval(v)
+                                        break
+                            else:
+                                raise ValueError("XXX")
+                        else:
+                            d[k.arg] = ast.literal_eval(k.value)
                     except ValueError:  # malformed node or string...
                         d[k.arg] = UNKNOWN
 

metadata_please/tests/_zip.py

@@ -1,17 +1,16 @@
 from __future__ import annotations
 
-from typing import Sequence
+from typing import Mapping, Sequence
 
 
 class MemoryZipFile:
-    def __init__(self, names: Sequence[str], read_value: bytes = b"foo") -> None:
-        self.names = names
-        self.read_value = read_value
+    def __init__(self, mock_files: Mapping[str, bytes] = {}) -> None:
+        self.mock_files = mock_files
         self.files_read: list[str] = []
 
     def namelist(self) -> Sequence[str]:
-        return self.names[:]
+        return list(self.mock_files.keys())
 
     def read(self, filename: str) -> bytes:
         self.files_read.append(filename)
-        return self.read_value
+        return self.mock_files[filename]