A hermetic pip end-to-end gate, and the #1329 review follow-ups

.github/workflows/pyre-ci.yml

@@ -439,6 +439,21 @@ jobs:
       if: ${{ !cancelled() && runner.os == 'Linux' }}
       shell: bash
       run: cargo build --release -p pyrex --bin pyre-dynasm --no-default-features --features dynasm
+    - name: Run pyre/extra_tests/pip (hermetic pip end-to-end)
+      # Drives the binary built above through venv, ensurepip, a wheel
+      # install, a PEP 517 build under real isolation, and an uninstall.
+      # Everything it resolves is a wheel already in the checkout, so it never
+      # reaches an index — and one of its checks asserts that, by requiring a
+      # plain `pip download` to fail.
+      #
+      # It rides this job's Linux copy for the same reason the suite below
+      # does: the release binary and the interpreter to drive it are already
+      # here. Ahead of the suite so its verdict lands early rather than after
+      # the suite's wall time, and `--dynasm-only` because the restored target
+      # directory can hold a `pyre-cranelift` this job never built.
+      if: ${{ !cancelled() && runner.os == 'Linux' }}
+      shell: bash
+      run: ${{ steps.cpython.outputs.python-path }} pyre/extra_tests/pip/run.py --dynasm-only
     - name: Run CPython suite (gate regressions, JIT on)
       # This is the only place CI runs the suite — `pyre/check.py` keeps the
       # stage behind `--cpython-suite`, which no job passes, so its wall time is

pyre/extra_tests/README.md

@@ -44,14 +44,48 @@ works.  `testutils.py` is the helper module shipped with the snippets
   rejects missing header fields. Each script cites the upstream file:line it guards;
   passing requires `exit 0` AND the final stdout line being `OK`.  Runner:
   `pyre/extra_tests/parity_tests/run.py`.
+- `pip/` — one stateful end-to-end sequence rather than a corpus: a release
+  binary is driven through `-m venv`, `ensurepip`, a wheel install, a PEP 517
+  build under real isolation, the console script and metadata that install
+  produced, and an uninstall.  Everything it resolves is a wheel the checkout
+  already carries (`lib-python/3/ensurepip/_bundled`, `lib-python/3/test/wheeldata`),
+  so it never reaches an index — and one of its checks asserts that by
+  requiring a plain `pip download` to fail.  It sits apart from `snippets/`
+  because it is stateful, because it needs a per-check timeout an order of
+  magnitude larger, and because the reference interpreter is not a comparand
+  here: pip succeeding under CPython says nothing about pyre, so CPython is
+  used only as a control, and only after something has already failed, to
+  separate a runtime defect from a rotted fixture.  Runner:
+  `pyre/extra_tests/pip/run.py`.
 - `upstream/` — no tests of its own: a runner plus a driver for the
   vendored PyPy tree at the repository **root** `extra_tests/`.  Those
   files stay where upstream put them and run in place, so anything they
   already cover does not get rewritten under `parity_tests/`.
   Runner: `pyre/extra_tests/upstream/run.py`.
 
-All three runners share the same backend discovery (cpython +
-pyre-dynasm + pyre-cranelift) and exit code semantics.
+The runners share the same backend discovery (pyre-dynasm + pyre-cranelift,
+plus cpython where a reference comparison is the point) and exit code
+semantics.
+
+## Running the pip gate
+
+```sh
+python3 pyre/extra_tests/pip/run.py                # every backend present
+python3 pyre/extra_tests/pip/run.py --dynasm-only  # what CI runs
+python3 pyre/extra_tests/pip/run.py --keep         # keep the working tree
+python3 pyre/extra_tests/pip/run.py --with-network # also resolve from a real index
+```
+
+Each backend gets its own temporary tree, kept and named on failure.  The
+fixtures are copied into it before anything is built, because installing from
+a source directory writes build artefacts beside it.  `--with-network` adds
+the one thing the gate cannot assert offline — that an index answers over TLS
+— and is never what CI runs, so a package server being down cannot turn a
+merge red.
+
+No version is written down: the pip and setuptools versions come from the
+filenames of the wheels in the checkout, so a stdlib sync that bumps either
+needs no edit here.
 
 ## The vendored root `extra_tests/`
 

pyre/extra_tests/pip/fixtures/stpkg/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["setuptools>=40.8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "stpkg"
+version = "0.2.0"
+
+[project.scripts]
+stpkg-hi = "stpkg:main"
+
+[tool.setuptools]
+py-modules = ["stpkg"]

pyre/extra_tests/pip/fixtures/stpkg/stpkg.py

@@ -0,0 +1,6 @@
+def hi():
+    return "hi from stpkg"
+
+
+def main():
+    print(hi())

pyre/extra_tests/pip/fixtures/tinypkg/pyproject.toml

@@ -0,0 +1,13 @@
+# `build-backend` is spelled out even though `tinybackend` is the only backend
+# here: a `[build-system]` table without it means the legacy setuptools backend
+# and `requires = ["setuptools>=40.8.0"]`, which would turn this fixture --
+# whose whole point is a PEP 517 build that needs nothing installed -- into a
+# second copy of the `stpkg` one.
+[build-system]
+requires = []
+build-backend = "tinybackend"
+backend-path = ["."]
+
+[project]
+name = "tinypkg"
+version = "0.1.0"

pyre/extra_tests/pip/fixtures/tinypkg/tinybackend.py

@@ -0,0 +1,68 @@
+"""A PEP 517 backend with no build dependencies, written on the stdlib alone.
+
+Installing this fixture exercises the hook protocol itself -- the isolated
+environment, `get_requires_for_build_wheel`, `build_wheel`, and the unpacking
+of what it returns -- with nothing to resolve and nothing to download.  When it
+passes and the `stpkg` fixture beside it does not, the defect is in what the
+build environment installs rather than in the protocol.
+"""
+
+import base64
+import hashlib
+import os
+import zipfile
+
+NAME = "tinypkg"
+VERSION = "0.1.0"
+DIST = f"{NAME}-{VERSION}"
+METADATA = f"Metadata-Version: 2.1\nName: {NAME}\nVersion: {VERSION}\n"
+WHEEL = (
+    "Wheel-Version: 1.0\n"
+    "Generator: tinybackend\n"
+    "Root-Is-Purelib: true\n"
+    "Tag: py3-none-any\n"
+)
+
+
+def get_requires_for_build_wheel(config_settings=None):
+    return []
+
+
+def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
+    info = os.path.join(metadata_directory, f"{DIST}.dist-info")
+    os.makedirs(info, exist_ok=True)
+    with open(os.path.join(info, "METADATA"), "w", encoding="utf-8") as out:
+        out.write(METADATA)
+    with open(os.path.join(info, "WHEEL"), "w", encoding="utf-8") as out:
+        out.write(WHEEL)
+    return f"{DIST}.dist-info"
+
+
+def _record_line(name, payload):
+    digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest())
+    return f"{name},sha256={digest.rstrip(b'=').decode()},{len(payload)}\n"
+
+
+def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
+    filename = f"{DIST}-py3-none-any.whl"
+    with open(os.path.join(os.path.dirname(__file__), "tinypkg.py"), "rb") as source:
+        module = source.read()
+    info = f"{DIST}.dist-info"
+    entries = [
+        ("tinypkg.py", module),
+        (f"{info}/METADATA", METADATA.encode()),
+        (f"{info}/WHEEL", WHEEL.encode()),
+    ]
+    # `RECORD` names itself with an empty hash, which is the one entry whose
+    # digest cannot be taken before the file exists.
+    record = "".join(_record_line(name, data) for name, data in entries)
+    record += f"{info}/RECORD,,\n"
+    with zipfile.ZipFile(os.path.join(wheel_directory, filename), "w") as wheel:
+        for name, data in entries:
+            wheel.writestr(name, data)
+        wheel.writestr(f"{info}/RECORD", record)
+    return filename
+
+
+def build_sdist(sdist_directory, config_settings=None):
+    raise NotImplementedError("tinypkg is installed from its directory")

pyre/extra_tests/pip/fixtures/tinypkg/tinypkg.py

@@ -0,0 +1,2 @@
+def hello():
+    return "hello from tinypkg"

pyre/extra_tests/pip/mksdist.py

@@ -0,0 +1,35 @@
+"""Write the `stpkg` source distribution.
+
+Run by the interpreter under test rather than by the driver, so the archive is
+one the runtime produced: it goes through `tarfile` over `gzip` over `zlib`,
+and the driver reads the result back with its own `tarfile` before pip is
+allowed near it.  A tarball that only the writer can open would otherwise
+surface as an unrelated failure inside the build.
+"""
+
+import io
+import os
+import sys
+import tarfile
+import time
+
+DIST = "stpkg-0.2.0"
+PKG_INFO = b"Metadata-Version: 2.1\nName: stpkg\nVersion: 0.2.0\n"
+
+
+def main(source, destination):
+    parent = os.path.dirname(destination)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+    with tarfile.open(destination, "w:gz") as archive:
+        for name in ("pyproject.toml", "stpkg.py"):
+            archive.add(os.path.join(source, name), arcname=f"{DIST}/{name}")
+        info = tarfile.TarInfo(f"{DIST}/PKG-INFO")
+        info.size = len(PKG_INFO)
+        info.mtime = int(time.time())
+        archive.addfile(info, io.BytesIO(PKG_INFO))
+    print(destination)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1], sys.argv[2])