feat(pip): support specifying requirements per (os, arch)

CHANGELOG.md

@@ -25,14 +25,12 @@ A brief description of the categories of changes:
 * (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and
   `py_library` now depend on the `//python:exec_tools_toolchain_type` for build
   tools.
-
 * (deps): Bumped `bazel_skylib` to 1.6.1.
 * (bzlmod): The `python` and internal `rules_python` extensions have been
   marked as `reproducible` and will not include any lock file entries from now
   on.
 
 ### Fixed
-
 * (gazelle) Remove `visibility` from `NonEmptyAttr`.
   Now empty(have no `deps/main/srcs/imports` attr) `py_library/test/binary` rules will
   be automatically deleted correctly. For example, if `python_generation_mode`
@@ -64,9 +62,16 @@ A brief description of the categories of changes:
   `transitive_pyc_files`, which tell the pyc files a target makes available
   directly and transitively, respectively.
 * `//python:features.bzl` added to allow easy feature-detection in the future.
+* (pip) Allow specifying the requirements by (os, arch) and add extra
+  validations when parsing the inputs. This is a non-breaking change for most
+  users unless they have been passing multiple `requirements_*` files together
+  with `extra_pip_args = ["--platform=manylinux_2_4_x86_64"]`, that was an
+  invalid usage previously but we were not failing the build. From now on this
+  is explicitly disallowed.
 
 [precompile-docs]: /precompiling
 
+
 ## [0.32.2] - 2024-05-14
 
 [0.32.2]: https://github.com/bazelbuild/rules_python/releases/tag/0.32.2

MODULE.bazel

@@ -61,9 +61,11 @@ pip.parse(
     experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
-    requirements_darwin = "//tools/publish:requirements_darwin.txt",
-    requirements_lock = "//tools/publish:requirements.txt",
-    requirements_windows = "//tools/publish:requirements_windows.txt",
+    requirements_by_platform = {
+        "//tools/publish:requirements.txt": "linux_*",
+        "//tools/publish:requirements_darwin.txt": "osx_*",
+        "//tools/publish:requirements_windows.txt": "windows_*",
+    },
 )
 use_repo(pip, "rules_python_publish_deps")
 

docs/sphinx/pip.md

@@ -19,11 +19,41 @@ load("@pip_deps//:requirements.bzl", "install_deps")
 install_deps()
 ```
 
+For `bzlmod` an equivalent `MODULE.bazel` would look like:
+```starlark
+pip = use_extension("//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "pip_deps",
+    requirements_lock = ":requirements.txt",
+)
+use_repo(pip, "pip_deps")
+```
+
 You can then reference installed dependencies from a `BUILD` file with:
 
 ```starlark
 load("@pip_deps//:requirements.bzl", "requirement")
 
+py_library(
+    name = "bar",
+    ...
+    deps = [
+        "//my/other:dep",
+        "@pip_deps//requests",
+        "@pip_deps//numpy",
+    ],
+)
+```
+
+The rules also provide a convenience macro for translating the entries in the
+`requirements.txt` file (e.g. `opencv-python`) to the right bazel label (e.g.
+`@pip_deps//opencv_python`). The convention of bazel labels is lowercase
+`snake_case`, but you can use the helper to avoid depending on this convention
+as follows:
+
+```starlark
+load("@pip_deps//:requirements.bzl", "requirement")
+
 py_library(
     name = "bar",
     ...
@@ -35,33 +65,39 @@ py_library(
 )
 ```
 
-In addition to the `requirement` macro, which is used to access the generated `py_library`
-target generated from a package's wheel, The generated `requirements.bzl` file contains
-functionality for exposing [entry points][whl_ep] as `py_binary` targets as well.
+If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation.
 
 [whl_ep]: https://packaging.python.org/specifications/entry-points/
 
+(per-os-arch-requirements)=
+## Requirements for a specific OS/Architecture
+
+In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples.
+
+For example:
 ```starlark
-load("@pip_deps//:requirements.bzl", "entry_point")
-
-alias(
-    name = "pip-compile",
-    actual = entry_point(
-        pkg = "pip-tools",
-        script = "pip-compile",
-    ),
-)
+    # ...
+    requirements_by_platform = {
+        "requirements_linux_x86_64.txt": "linux_x86_64",
+        "requirements_osx.txt": "osx_*",
+        "requirements_linux_exotic.txt": "linux_exotic",
+        "requirements_some_platforms.txt": "linux_aarch64,windows_*",
+    },
+    # For the list of standard platforms that the rules_python has toolchains for, default to
+    # the following requirements file.
+    requirements_lock = "requirements_lock.txt",
 ```
 
-Note that for packages whose name and script are the same, only the name of the package
-is needed when calling the `entry_point` macro.
+In case of duplicate platforms, `rules_python` will raise an error as there has
+to be unambiguous mapping of the requirement files to the (os, arch) tuples.
 
+An alternative way is to use per-OS requirement attributes.
 ```starlark
-load("@pip_deps//:requirements.bzl", "entry_point")
-
-alias(
-    name = "flake8",
-    actual = entry_point("flake8"),
+    # ...
+    requirements_windows = "requirements_windows.txt",
+    requirements_darwin = "requirements_darwin.txt",
+    # For the remaining platforms (which is basically only linux OS), use this file.
+    requirements_lock = "requirements_lock.txt",
 )
 ```
 

examples/bzlmod/MODULE.bazel

@@ -128,8 +128,17 @@ pip.parse(
     ],
     hub_name = "pip",
     python_version = "3.9",
-    requirements_lock = "//:requirements_lock_3_9.txt",
-    requirements_windows = "//:requirements_windows_3_9.txt",
+    # The requirements files for each platform that we want to support.
+    requirements_by_platform = {
+        # Default requirements file for needs to explicitly provide the platforms
+        "//:requirements_lock_3_9.txt": "linux_*,osx_*",
+        # This API allows one to specify additional platforms that the users
+        # configure the toolchains for themselves. In this example we add
+        # `windows_aarch64` to illustrate that `rules_python` won't fail to
+        # process the value, but it does not mean that this example will work
+        # on Windows ARM.
+        "//:requirements_windows_3_9.txt": "windows_x86_64,windows_aarch64",
+    },
     # These modifications were created above and we
     # are providing pip.parse with the label of the mod
     # and the name of the wheel.
@@ -193,14 +202,3 @@ local_path_override(
     module_name = "other_module",
     path = "other_module",
 )
-
-# =====
-# Config for testing duplicate packages in requirements
-# =====
-#
-pip.parse(
-    hub_name = "dupe_requirements",
-    python_version = "3.9",  # Must match whatever is marked is_default=True
-    requirements_lock = "//tests/dupe_requirements:requirements.txt",
-)
-use_repo(pip, "dupe_requirements")

examples/pip_parse/MODULE.bazel

@@ -21,6 +21,7 @@ use_repo(
 
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
+    download_only = True,
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",

examples/pip_parse_vendored/requirements.bzl

@@ -1,7 +1,6 @@
 """Starlark representation of locked requirements.
 
-@generated by rules_python pip_parse repository rule
-from @//:requirements.txt
+@generated by rules_python pip_parse repository rule.
 """
 
 load("@rules_python//python:pip.bzl", "pip_utils")

python/pip_install/BUILD.bazel

@@ -23,7 +23,6 @@ bzl_library(
     srcs = ["pip_repository.bzl"],
     deps = [
         ":repositories_bzl",
-        ":requirements_parser_bzl",
         "//python:repositories_bzl",
         "//python:versions_bzl",
         "//python/pip_install/private:generate_group_library_build_bazel_bzl",
@@ -32,6 +31,7 @@ bzl_library(
         "//python/private:bzlmod_enabled_bzl",
         "//python/private:envsubst_bzl",
         "//python/private:normalize_name_bzl",
+        "//python/private:parse_requirements_bzl",
         "//python/private:parse_whl_name_bzl",
         "//python/private:patch_whl_bzl",
         "//python/private:render_pkg_aliases_bzl",