Fix editable requirement parsing. (#2464)

Previously, editable requirements in requirements files were not parsed
properly by Pex. Although they did not trigger parse errors, PEXes
created from editable requirements would fail to import those
requirements at runtime despite the editable project distribution being
embedded in the PEX file.

Fixes #2410

Author: jsirois

CHANGES.md

@@ -1,5 +1,14 @@
 # Release Notes
 
+## 2.10.1
+
+This release fixes a long-standing bug in Pex parsing of editable
+requirements. This bug caused PEXes containing local editable project
+requirements to fail to import those local editable projects despite
+the fact the PEX itself contained them.
+
+* Fix editable requirement parsing. (#2464)
+
 ## 2.10.0
 
 This release adds support for injecting requirements into the isolated

docs-requirements.txt

@@ -2,4 +2,8 @@ furo
 httpx
 myst-parser[linkify]
 sphinx
-sphinx-simplepdf
+sphinx-simplepdf
+
+# The 0.11.0 release removes deprecated API parameters which breaks weasyprint (62.3 depends on
+# `pydyf>=0.10.0`) which is a dependency of sphinx-simplepdf.
+pydyf<0.11.0

pex/requirements.py

@@ -418,7 +418,9 @@ def _try_parse_pip_local_formats(
     directory_name, requirement_parts = match.groups()
     stripped_path = os.path.join(os.path.dirname(path), directory_name)
     abs_stripped_path = (
-        os.path.join(basepath, stripped_path) if basepath else os.path.abspath(stripped_path)
+        os.path.join(basepath, stripped_path)
+        if basepath and not os.path.isabs(stripped_path)
+        else os.path.abspath(stripped_path)
     )
     if not os.path.exists(abs_stripped_path):
         return None
@@ -650,8 +652,11 @@ def parse_requirements(
                         yield requirement
                 continue
 
-            # Skip empty lines, comment lines and all other Pip options.
-            if not processed_text or processed_text.startswith("-"):
+            # Skip empty lines, comment lines and all Pip global options.
+            if not processed_text or (
+                processed_text.startswith("-")
+                and not re.match(r"^(?:-e|--editable)\s.*", processed_text)
+            ):
                 continue
 
             # Only requirement lines remain.

pex/version.py

@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.10.0"
+__version__ = "2.10.1"

tests/integration/test_issue_2410.py

@@ -0,0 +1,77 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import os.path
+import subprocess
+from textwrap import dedent
+
+from colors import colors
+
+from pex.common import safe_open
+from pex.typing import TYPE_CHECKING
+from testing import run_pex_command
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+def test_pex_with_editable(tmpdir):
+    # type: (Any) -> None
+
+    project_dir = os.path.join(str(tmpdir), "project")
+    with safe_open(os.path.join(project_dir, "example.py"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                import sys
+
+                import colors
+
+
+                def colorize(*messages):
+                    return colors.green(" ".join(messages))
+
+
+                if __name__ == "__main__":
+                    print(colorize(*sys.argv[1:]))
+                    sys.exit(0)
+                """
+            )
+        )
+    with safe_open(os.path.join(project_dir, "setup.py"), "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                from setuptools import setup
+
+
+                setup(
+                    name="example",
+                    version="0.1.0",
+                    py_modules=["example"],
+                )
+                """
+            )
+        )
+
+    requirements = os.path.join(project_dir, "requirements.txt")
+    with safe_open(requirements, "w") as fp:
+        fp.write(
+            dedent(
+                """\
+                ansicolors==1.1.8
+                -e file://{project_dir}
+                """
+            ).format(project_dir=project_dir)
+        )
+
+    pex = os.path.join(str(tmpdir), "pex")
+    run_pex_command(args=["-r", requirements, "-m", "example", "-o", pex]).assert_success()
+    output = (
+        subprocess.check_output(args=[pex, "A", "wet", "duck", "flies", "at", "night!"])
+        .decode("utf-8")
+        .strip()
+    )
+    assert colors.green("A wet duck flies at night!") == output, output

tests/test_requirements.py

@@ -51,7 +51,7 @@ def chroot():
         curdir = os.getcwd()
         try:
             os.chdir(chroot)
-            yield chroot
+            yield os.path.realpath(chroot)
         finally:
             os.chdir(curdir)
 
@@ -278,8 +278,8 @@ def test_parse_requirements_stress(chroot):
                 hg+http://hg.example.com/MyProject@da39a3ee5e6b\\
                     #egg=AnotherProject[extra,more] ; python_version=="3.9.*"&subdirectory=foo/bar
 
-                ftp://a/${PROJECT_NAME}-1.0.tar.gz
-                http://a/${PROJECT_NAME}-1.0.zip
+                ftp://a/${{PROJECT_NAME}}-1.0.tar.gz
+                http://a/${{PROJECT_NAME}}-1.0.zip
                 https://a/numpy-1.9.2-cp34-none-win32.whl
                 https://a/numpy-1.9.2-cp34-none-win32.whl;\\
                     python_version=="3.4.*" and sys_platform=='win32'
@@ -295,8 +295,14 @@ def test_parse_requirements_stress(chroot):
 
                 # Wheel with local version
                 http://download.pytorch.org/whl/cpu/torch-1.12.1%2Bcpu-cp310-cp310-linux_x86_64.whl
+
+                # Editable
+                -e file://{chroot}/extra/a/local/project
+                --editable file://{chroot}/extra/a/local/project/
+                -e ./another/local/project
+                --editable ./another/local/project/
                 """
-            )
+            ).format(chroot=chroot)
         )
     touch("extra/pyproject.toml")
     touch("extra/a/local/project/pyproject.toml")
@@ -470,6 +476,10 @@ def test_parse_requirements_stress(chroot):
             url="http://download.pytorch.org/whl/cpu/torch-1.12.1%2Bcpu-cp310-cp310-linux_x86_64.whl",
             specifier="==1.12.1+cpu",
         ),
+        local_req(path=os.path.join(chroot, "extra/a/local/project"), editable=True),
+        local_req(path=os.path.join(chroot, "extra/a/local/project"), editable=True),
+        local_req(path=os.path.join(chroot, "extra/another/local/project"), editable=True),
+        local_req(path=os.path.join(chroot, "extra/another/local/project"), editable=True),
         url_req(
             project_name="numpy",
             url=os.path.realpath("./downloads/numpy-1.9.2-cp34-none-win32.whl"),