From a6cb620f0c0c34082040b0560d4dd2e11e39715e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 19 May 2024 12:38:03 +0900 Subject: [PATCH] feat(pip): support specifying requirements per (os, arch) (#1885) This PR implements a better way of specifying the requirements files for different (os, cpu) tuples. It allows for more granular specification than what is available today and allows for future extension to have all of the sources in the select statements in the hub repository. This is replacing the previous selection of the requirements and there are a few differences in behaviour that should not be visible to the external user. Instead of selecting the right file which we should then use to create `whl_library` instances we parse all of the provided requirements files and merge them based on the contents. The merging is done based on the blocks within the requirement file and this allows the starlark code to understand if we are working with different versions of the same package on different target platforms. Fixes #1868 Work towards #1643, #735 --- CHANGELOG.md | 9 +- MODULE.bazel | 8 +- docs/sphinx/pip.md | 74 +++- examples/bzlmod/MODULE.bazel | 24 +- .../tests/dupe_requirements/BUILD.bazel | 19 - .../dupe_requirements_test.py | 4 - .../tests/dupe_requirements/requirements.in | 2 - .../tests/dupe_requirements/requirements.txt | 97 ----- examples/pip_parse/MODULE.bazel | 1 + examples/pip_parse_vendored/requirements.bzl | 3 +- python/pip_install/BUILD.bazel | 2 +- python/pip_install/pip_repository.bzl | 78 ++-- .../pip_repository_requirements.bzl.tmpl | 3 +- python/private/BUILD.bazel | 16 + python/private/bzlmod/BUILD.bazel | 2 +- python/private/bzlmod/pip.bzl | 76 ++-- python/private/normalize_platform.bzl | 13 + python/private/parse_requirements.bzl | 374 ++++++++++++++++++ python/private/pypi_index.bzl | 60 +-- python/private/pypi_index_sources.bzl | 53 +++ python/private/whl_target_platforms.bzl | 43 +- tests/private/parse_requirements/BUILD.bazel | 3 + .../parse_requirements_tests.bzl | 374 ++++++++++++++++++ tests/private/pypi_index/pypi_index_tests.bzl | 34 +- tests/private/pypi_index_sources/BUILD.bazel | 3 + .../pypi_index_sources_tests.bzl | 60 +++ .../whl_target_platforms/select_whl_tests.bzl | 18 +- 27 files changed, 1071 insertions(+), 382 deletions(-) delete mode 100644 examples/bzlmod/tests/dupe_requirements/BUILD.bazel delete mode 100644 examples/bzlmod/tests/dupe_requirements/dupe_requirements_test.py delete mode 100644 examples/bzlmod/tests/dupe_requirements/requirements.in delete mode 100644 examples/bzlmod/tests/dupe_requirements/requirements.txt create mode 100644 python/private/normalize_platform.bzl create mode 100644 python/private/parse_requirements.bzl create mode 100644 python/private/pypi_index_sources.bzl create mode 100644 tests/private/parse_requirements/BUILD.bazel create mode 100644 tests/private/parse_requirements/parse_requirements_tests.bzl create mode 100644 tests/private/pypi_index_sources/BUILD.bazel create mode 100644 tests/private/pypi_index_sources/pypi_index_sources_tests.bzl diff --git a/CHANGELOG.md b/CHANGELOG.md index 09b9aaf7ee..430c5c84f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,14 +25,12 @@ A brief description of the categories of changes: * (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and `py_library` now depend on the `//python:exec_tools_toolchain_type` for build tools. - * (deps): Bumped `bazel_skylib` to 1.6.1. * (bzlmod): The `python` and internal `rules_python` extensions have been marked as `reproducible` and will not include any lock file entries from now on. ### Fixed - * (gazelle) Remove `visibility` from `NonEmptyAttr`. Now empty(have no `deps/main/srcs/imports` attr) `py_library/test/binary` rules will be automatically deleted correctly. For example, if `python_generation_mode` @@ -66,9 +64,16 @@ A brief description of the categories of changes: `transitive_pyc_files`, which tell the pyc files a target makes available directly and transitively, respectively. * `//python:features.bzl` added to allow easy feature-detection in the future. +* (pip) Allow specifying the requirements by (os, arch) and add extra + validations when parsing the inputs. This is a non-breaking change for most + users unless they have been passing multiple `requirements_*` files together + with `extra_pip_args = ["--platform=manylinux_2_4_x86_64"]`, that was an + invalid usage previously but we were not failing the build. From now on this + is explicitly disallowed. [precompile-docs]: /precompiling + ## [0.32.2] - 2024-05-14 [0.32.2]: https://github.com/bazelbuild/rules_python/releases/tag/0.32.2 diff --git a/MODULE.bazel b/MODULE.bazel index 8acde16c33..7064dfc84f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -61,9 +61,11 @@ pip.parse( experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", python_version = "3.11", - requirements_darwin = "//tools/publish:requirements_darwin.txt", - requirements_lock = "//tools/publish:requirements.txt", - requirements_windows = "//tools/publish:requirements_windows.txt", + requirements_by_platform = { + "//tools/publish:requirements.txt": "linux_*", + "//tools/publish:requirements_darwin.txt": "osx_*", + "//tools/publish:requirements_windows.txt": "windows_*", + }, ) use_repo(pip, "rules_python_publish_deps") diff --git a/docs/sphinx/pip.md b/docs/sphinx/pip.md index e73c0c6a56..e1c8e343f0 100644 --- a/docs/sphinx/pip.md +++ b/docs/sphinx/pip.md @@ -19,11 +19,41 @@ load("@pip_deps//:requirements.bzl", "install_deps") install_deps() ``` +For `bzlmod` an equivalent `MODULE.bazel` would look like: +```starlark +pip = use_extension("//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "pip_deps", + requirements_lock = ":requirements.txt", +) +use_repo(pip, "pip_deps") +``` + You can then reference installed dependencies from a `BUILD` file with: ```starlark load("@pip_deps//:requirements.bzl", "requirement") +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + "@pip_deps//requests", + "@pip_deps//numpy", + ], +) +``` + +The rules also provide a convenience macro for translating the entries in the +`requirements.txt` file (e.g. `opencv-python`) to the right bazel label (e.g. +`@pip_deps//opencv_python`). The convention of bazel labels is lowercase +`snake_case`, but you can use the helper to avoid depending on this convention +as follows: + +```starlark +load("@pip_deps//:requirements.bzl", "requirement") + py_library( name = "bar", ... @@ -35,33 +65,39 @@ py_library( ) ``` -In addition to the `requirement` macro, which is used to access the generated `py_library` -target generated from a package's wheel, The generated `requirements.bzl` file contains -functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. +If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation. [whl_ep]: https://packaging.python.org/specifications/entry-points/ +(per-os-arch-requirements)= +## Requirements for a specific OS/Architecture + +In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples. + +For example: ```starlark -load("@pip_deps//:requirements.bzl", "entry_point") - -alias( - name = "pip-compile", - actual = entry_point( - pkg = "pip-tools", - script = "pip-compile", - ), -) + # ... + requirements_by_platform = { + "requirements_linux_x86_64.txt": "linux_x86_64", + "requirements_osx.txt": "osx_*", + "requirements_linux_exotic.txt": "linux_exotic", + "requirements_some_platforms.txt": "linux_aarch64,windows_*", + }, + # For the list of standard platforms that the rules_python has toolchains for, default to + # the following requirements file. + requirements_lock = "requirements_lock.txt", ``` -Note that for packages whose name and script are the same, only the name of the package -is needed when calling the `entry_point` macro. +In case of duplicate platforms, `rules_python` will raise an error as there has +to be unambiguous mapping of the requirement files to the (os, arch) tuples. +An alternative way is to use per-OS requirement attributes. ```starlark -load("@pip_deps//:requirements.bzl", "entry_point") - -alias( - name = "flake8", - actual = entry_point("flake8"), + # ... + requirements_windows = "requirements_windows.txt", + requirements_darwin = "requirements_darwin.txt", + # For the remaining platforms (which is basically only linux OS), use this file. + requirements_lock = "requirements_lock.txt", ) ``` diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 1134487145..0d30161147 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -128,8 +128,17 @@ pip.parse( ], hub_name = "pip", python_version = "3.9", - requirements_lock = "//:requirements_lock_3_9.txt", - requirements_windows = "//:requirements_windows_3_9.txt", + # The requirements files for each platform that we want to support. + requirements_by_platform = { + # Default requirements file for needs to explicitly provide the platforms + "//:requirements_lock_3_9.txt": "linux_*,osx_*", + # This API allows one to specify additional platforms that the users + # configure the toolchains for themselves. In this example we add + # `windows_aarch64` to illustrate that `rules_python` won't fail to + # process the value, but it does not mean that this example will work + # on Windows ARM. + "//:requirements_windows_3_9.txt": "windows_x86_64,windows_aarch64", + }, # These modifications were created above and we # are providing pip.parse with the label of the mod # and the name of the wheel. @@ -193,14 +202,3 @@ local_path_override( module_name = "other_module", path = "other_module", ) - -# ===== -# Config for testing duplicate packages in requirements -# ===== -# -pip.parse( - hub_name = "dupe_requirements", - python_version = "3.9", # Must match whatever is marked is_default=True - requirements_lock = "//tests/dupe_requirements:requirements.txt", -) -use_repo(pip, "dupe_requirements") diff --git a/examples/bzlmod/tests/dupe_requirements/BUILD.bazel b/examples/bzlmod/tests/dupe_requirements/BUILD.bazel deleted file mode 100644 index 47eb7ca0fb..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_python//python:pip.bzl", "compile_pip_requirements") -load("@rules_python//python:py_test.bzl", "py_test") - -py_test( - name = "dupe_requirements_test", - srcs = ["dupe_requirements_test.py"], - deps = [ - "@dupe_requirements//pyjwt", - ], -) - -compile_pip_requirements( - name = "requirements", - src = "requirements.in", - requirements_txt = "requirements.txt", - # This is to make the requirements diff test not run on CI. The content we - # need in requirements.txt isn't exactly what will be generated. - tags = ["manual"], -) diff --git a/examples/bzlmod/tests/dupe_requirements/dupe_requirements_test.py b/examples/bzlmod/tests/dupe_requirements/dupe_requirements_test.py deleted file mode 100644 index 1139dc5252..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/dupe_requirements_test.py +++ /dev/null @@ -1,4 +0,0 @@ -# There's nothing to test at runtime. Building indicates success. -# Just import the relevant modules as a basic check. -import cryptography -import jwt diff --git a/examples/bzlmod/tests/dupe_requirements/requirements.in b/examples/bzlmod/tests/dupe_requirements/requirements.in deleted file mode 100644 index b1f623395a..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -pyjwt -pyjwt[crypto] diff --git a/examples/bzlmod/tests/dupe_requirements/requirements.txt b/examples/bzlmod/tests/dupe_requirements/requirements.txt deleted file mode 100644 index 785f556624..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/requirements.txt +++ /dev/null @@ -1,97 +0,0 @@ -# -# This file is manually tweaked output from the automatic generation. -# To generate: -# 1. bazel run //tests/dupe_requirements:requirements.update -# 2. Then copy/paste the pyjtw lines so there are duplicates -# -pyjwt==2.8.0 \ - --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ - --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 - # via -r tests/dupe_requirements/requirements.in -pyjwt[crypto]==2.8.0 \ - --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ - --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 - # via -r tests/dupe_requirements/requirements.in -cffi==1.16.0 \ - --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ - --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ - --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ - --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ - --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ - --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ - --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ - --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ - --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ - --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ - --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ - --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ - --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ - --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ - --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ - --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ - --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ - --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ - --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ - --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ - --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ - --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ - --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ - --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ - --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ - --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ - --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ - --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ - --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ - --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ - --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ - --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ - --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ - --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ - --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ - --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ - --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ - --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ - --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ - --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ - --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ - --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ - --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ - --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ - --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ - --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ - --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ - --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ - --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ - --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ - --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ - --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 - # via cryptography -cryptography==41.0.7 \ - --hash=sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960 \ - --hash=sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a \ - --hash=sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc \ - --hash=sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a \ - --hash=sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf \ - --hash=sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1 \ - --hash=sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39 \ - --hash=sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406 \ - --hash=sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a \ - --hash=sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a \ - --hash=sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c \ - --hash=sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be \ - --hash=sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15 \ - --hash=sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2 \ - --hash=sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d \ - --hash=sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157 \ - --hash=sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003 \ - --hash=sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248 \ - --hash=sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a \ - --hash=sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec \ - --hash=sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309 \ - --hash=sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7 \ - --hash=sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d - # via pyjwt -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 - # via cffi diff --git a/examples/pip_parse/MODULE.bazel b/examples/pip_parse/MODULE.bazel index b0e38f2218..f9ca90833f 100644 --- a/examples/pip_parse/MODULE.bazel +++ b/examples/pip_parse/MODULE.bazel @@ -21,6 +21,7 @@ use_repo( pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( + download_only = True, experimental_requirement_cycles = { "sphinx": [ "sphinx", diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index de5d187262..5c2391bd4c 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -1,7 +1,6 @@ """Starlark representation of locked requirements. -@generated by rules_python pip_parse repository rule -from @//:requirements.txt +@generated by rules_python pip_parse repository rule. """ load("@rules_python//python:pip.bzl", "pip_utils") diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index e794075af0..91f2ec7b59 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -23,7 +23,6 @@ bzl_library( srcs = ["pip_repository.bzl"], deps = [ ":repositories_bzl", - ":requirements_parser_bzl", "//python:repositories_bzl", "//python:versions_bzl", "//python/pip_install/private:generate_group_library_build_bazel_bzl", @@ -32,6 +31,7 @@ bzl_library( "//python/private:bzlmod_enabled_bzl", "//python/private:envsubst_bzl", "//python/private:normalize_name_bzl", + "//python/private:parse_requirements_bzl", "//python/private:parse_whl_name_bzl", "//python/private:patch_whl_bzl", "//python/private:render_pkg_aliases_bzl", diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index db6736836f..17d80838e0 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -18,13 +18,13 @@ load("@bazel_skylib//lib:sets.bzl", "sets") load("//python:repositories.bzl", "is_standalone_interpreter") load("//python:versions.bzl", "WINDOWS_NAME") load("//python/pip_install:repositories.bzl", "all_requirements") -load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load("//python/private:parse_whl_name.bzl", "parse_whl_name") load("//python/private:patch_whl.bzl", "patch_whl") load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") @@ -272,38 +272,30 @@ package(default_visibility = ["//visibility:public"]) exports_files(["requirements.bzl"]) """ -def locked_requirements_label(ctx, attr): - """Get the preferred label for a locked requirements file based on platform. - - Args: - ctx: repository or module context - attr: attributes for the repo rule or tag extension - - Returns: - Label - """ - os = ctx.os.name.lower() - requirements_txt = attr.requirements_lock - if os.startswith("mac os") and attr.requirements_darwin != None: - requirements_txt = attr.requirements_darwin - elif os.startswith("linux") and attr.requirements_linux != None: - requirements_txt = attr.requirements_linux - elif "win" in os and attr.requirements_windows != None: - requirements_txt = attr.requirements_windows - if not requirements_txt: - fail("""\ -A requirements_lock attribute must be specified, or a platform-specific lockfile using one of the requirements_* attributes. -""") - return requirements_txt - def _pip_repository_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) - - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] + requirements_by_platform = parse_requirements( + rctx, + requirements_by_platform = rctx.attr.requirements_by_platform, + requirements_linux = rctx.attr.requirements_linux, + requirements_lock = rctx.attr.requirements_lock, + requirements_osx = rctx.attr.requirements_darwin, + requirements_windows = rctx.attr.requirements_windows, + extra_pip_args = rctx.attr.extra_pip_args, + ) + selected_requirements = {} + options = None + repository_platform = host_platform(rctx.os) + for name, requirements in requirements_by_platform.items(): + r = select_requirement( + requirements, + platform = repository_platform, + ) + if not r: + continue + options = options or r.extra_pip_args + selected_requirements[name] = r.requirement_line - bzl_packages = sorted([normalize_name(name) for name, _ in parsed_requirements_txt.requirements]) + bzl_packages = sorted(selected_requirements.keys()) # Normalize cycles first requirement_cycles = { @@ -347,13 +339,6 @@ def _pip_repository_impl(rctx): rctx.file(filename, json.encode_indent(json.decode(annotation))) annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) - tokenized_options = [] - for opt in parsed_requirements_txt.options: - for p in opt.split(" "): - tokenized_options.append(p) - - options = tokenized_options + rctx.attr.extra_pip_args - config = { "download_only": rctx.attr.download_only, "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, @@ -419,10 +404,9 @@ def _pip_repository_impl(rctx): "%%PACKAGES%%": _format_repr_list( [ ("{}_{}".format(rctx.attr.name, p), r) - for p, r in packages + for p, r in sorted(selected_requirements.items()) ], ), - "%%REQUIREMENTS_LOCK%%": str(requirements_txt), }) return @@ -625,6 +609,15 @@ pip_repository_attrs = { "annotations": attr.string_dict( doc = "Optional annotations to apply to packages", ), + "requirements_by_platform": attr.label_keyed_string_dict( + doc = """\ +The requirements files and the comma delimited list of target platforms as values. + +The keys are the requirement files and the values are comma-separated platform +identifiers. For now we only support `_` values that are present in +`@platforms//os` and `@platforms//cpu` packages respectively. +""", + ), "requirements_darwin": attr.label( allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Mac OS", @@ -643,6 +636,11 @@ individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. Note that if your lockfile is platform-dependent, you can use the `requirements_[platform]` attributes. + +Note, that in general requirements files are compiled for a specific platform, +but sometimes they can work for multiple platforms. `rules_python` right now +supports requirements files that are created for a particular platform without +platform markers. """, ), "requirements_windows": attr.label( diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/pip_install/pip_repository_requirements.bzl.tmpl index 8e17720374..07b4b08148 100644 --- a/python/pip_install/pip_repository_requirements.bzl.tmpl +++ b/python/pip_install/pip_repository_requirements.bzl.tmpl @@ -1,7 +1,6 @@ """Starlark representation of locked requirements. -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%% +@generated by rules_python pip_parse repository rule. """ %%IMPORTS%% diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 181175679a..45f50effb0 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -128,6 +128,17 @@ bzl_library( deps = [":parse_whl_name_bzl"], ) +bzl_library( + name = "parse_requirements_bzl", + srcs = ["parse_requirements.bzl"], + deps = [ + ":normalize_name_bzl", + ":pypi_index_sources_bzl", + ":whl_target_platforms_bzl", + "//python/pip_install:requirements_parser_bzl", + ], +) + bzl_library( name = "parse_whl_name_bzl", srcs = ["parse_whl_name.bzl"], @@ -145,6 +156,11 @@ bzl_library( ], ) +bzl_library( + name = "pypi_index_sources_bzl", + srcs = ["pypi_index_sources.bzl"], +) + bzl_library( name = "py_cc_toolchain_bzl", srcs = [ diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel index 9edd3380bb..2eab575726 100644 --- a/python/private/bzlmod/BUILD.bazel +++ b/python/private/bzlmod/BUILD.bazel @@ -31,10 +31,10 @@ bzl_library( deps = [ ":pip_repository_bzl", "//python/pip_install:pip_repository_bzl", - "//python/pip_install:requirements_parser_bzl", "//python/private:pypi_index_bzl", "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", + "//python/private:parse_requirements_bzl", "//python/private:parse_whl_name_bzl", "//python/private:version_label_bzl", ":bazel_features_bzl", diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index ce681259ed..80ee852573 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -18,16 +18,15 @@ load("@bazel_features//:features.bzl", "bazel_features") load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") load( "//python/pip_install:pip_repository.bzl", - "locked_requirements_label", "pip_repository_attrs", "use_isolated", "whl_library", ) -load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load("//python/private:parse_whl_name.bzl", "parse_whl_name") -load("//python/private:pypi_index.bzl", "get_simpleapi_sources", "simpleapi_download") +load("//python/private:pypi_index.bzl", "simpleapi_download") load("//python/private:render_pkg_aliases.bzl", "whl_alias") load("//python/private:version_label.bzl", "version_label") load("//python/private:whl_target_platforms.bzl", "select_whl") @@ -130,27 +129,6 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ) major_minor = _major_minor_version(pip_attr.python_version) - requirements_lock = locked_requirements_label(module_ctx, pip_attr) - - # Parse the requirements file directly in starlark to get the information - # needed for the whl_libary declarations below. - requirements_lock_content = module_ctx.read(requirements_lock) - parse_result = parse_requirements(requirements_lock_content) - - # Replicate a surprising behavior that WORKSPACE builds allowed: - # Defining a repo with the same name multiple times, but only the last - # definition is respected. - # The requirement lines might have duplicate names because lines for extras - # are returned as just the base package name. e.g., `foo[bar]` results - # in an entry like `("foo", "foo[bar] == 1.0 ...")`. - requirements = { - normalize_name(entry[0]): entry - # The WORKSPACE pip_parse sorted entries, so mimic that ordering. - for entry in sorted(parse_result.requirements) - }.values() - - extra_pip_args = pip_attr.extra_pip_args + parse_result.options - if hub_name not in whl_map: whl_map[hub_name] = {} @@ -180,6 +158,18 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s whl_group_mapping = {} requirement_cycles = {} + # Create a new wheel library for each of the different whls + + requirements_by_platform = parse_requirements( + module_ctx, + requirements_by_platform = pip_attr.requirements_by_platform, + requirements_linux = pip_attr.requirements_linux, + requirements_lock = pip_attr.requirements_lock, + requirements_osx = pip_attr.requirements_darwin, + requirements_windows = pip_attr.requirements_windows, + extra_pip_args = pip_attr.extra_pip_args, + ) + index_urls = {} if pip_attr.experimental_index_url: if pip_attr.download_only: @@ -191,7 +181,11 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s index_url = pip_attr.experimental_index_url, extra_index_urls = pip_attr.experimental_extra_index_urls or [], index_url_overrides = pip_attr.experimental_index_url_overrides or {}, - sources = [requirements_lock_content], + sources = list({ + req.distribution: None + for reqs in requirements_by_platform.values() + for req in reqs + }), envsubst = pip_attr.envsubst, # Auth related info netrc = pip_attr.netrc, @@ -201,8 +195,21 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s parallel_download = pip_attr.parallel_download, ) - # Create a new wheel library for each of the different whls - for whl_name, requirement_line in requirements: + repository_platform = host_platform(module_ctx.os) + for whl_name, requirements in requirements_by_platform.items(): + requirement = select_requirement( + requirements, + platform = repository_platform, + ) + if not requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + continue + # We are not using the "sanitized name" because the user # would need to guess what name we modified the whl name # to. @@ -218,7 +225,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s whl_library_args = dict( repo = pip_name, dep_template = "@{}//{{name}}:{{target}}".format(hub_name), - requirement = requirement_line, + requirement = requirement.requirement_line, ) maybe_args = dict( # The following values are safe to omit if they have false like values @@ -228,7 +235,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s environment = pip_attr.environment, envsubst = pip_attr.envsubst, experimental_target_platforms = pip_attr.experimental_target_platforms, - extra_pip_args = extra_pip_args, + extra_pip_args = requirement.extra_pip_args, group_deps = group_deps, group_name = group_name, pip_data_exclude = pip_attr.pip_data_exclude, @@ -249,11 +256,9 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default}) if index_urls: - srcs = get_simpleapi_sources(requirement_line) - whls = [] sdist = None - for sha256 in srcs.shas: + for sha256 in requirement.srcs.shas: # For now if the artifact is marked as yanked we just ignore it. # # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api @@ -279,12 +284,11 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # Older python versions have wheels for the `*m` ABI. "cp" + major_minor.replace(".", "") + "m", ], - want_os = module_ctx.os.name, - want_cpu = module_ctx.os.arch, + want_platform = repository_platform, ) or sdist if distribution: - whl_library_args["requirement"] = srcs.requirement + whl_library_args["requirement"] = requirement.srcs.requirement whl_library_args["urls"] = [distribution.url] whl_library_args["sha256"] = distribution.sha256 whl_library_args["filename"] = distribution.filename @@ -299,7 +303,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # This is no-op because pip is not used to download the wheel. whl_library_args.pop("download_only", None) else: - print("WARNING: falling back to pip for installing the right file for {}".format(requirement_line)) # buildifier: disable=print + print("WARNING: falling back to pip for installing the right file for {}".format(requirement.requirement_line)) # buildifier: disable=print # We sort so that the lock-file remains the same no matter the order of how the # args are manipulated in the code going before. diff --git a/python/private/normalize_platform.bzl b/python/private/normalize_platform.bzl new file mode 100644 index 0000000000..633062f399 --- /dev/null +++ b/python/private/normalize_platform.bzl @@ -0,0 +1,13 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl new file mode 100644 index 0000000000..f9d7a05386 --- /dev/null +++ b/python/private/parse_requirements.bzl @@ -0,0 +1,374 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Requirements parsing for whl_library creation. + +Use cases that the code needs to cover: +* A single requirements_lock file that is used for the host platform. +* Per-OS requirements_lock files that are used for the host platform. +* A target platform specific requirements_lock that is used with extra + pip arguments with --platform, etc and download_only = True. + +In the last case only a single `requirements_lock` file is allowed, in all +other cases we assume that there may be a desire to resolve the requirements +file for the host platform to be backwards compatible with the legacy +behavior. +""" + +load("//python/pip_install:requirements_parser.bzl", "parse") +load(":normalize_name.bzl", "normalize_name") +load(":pypi_index_sources.bzl", "get_simpleapi_sources") +load(":whl_target_platforms.bzl", "whl_target_platforms") + +# This includes the vendored _translate_cpu and _translate_os from +# @platforms//host:extension.bzl at version 0.0.9 so that we don't +# force the users to depend on it. + +def _translate_cpu(arch): + if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]: + return "x86_32" + if arch in ["amd64", "x86_64", "x64"]: + return "x86_64" + if arch in ["ppc", "ppc64", "ppc64le"]: + return "ppc" + if arch in ["arm", "armv7l"]: + return "arm" + if arch in ["aarch64"]: + return "aarch64" + if arch in ["s390x", "s390"]: + return "s390x" + if arch in ["mips64el", "mips64"]: + return "mips64" + if arch in ["riscv64"]: + return "riscv64" + return arch + +def _translate_os(os): + if os.startswith("mac os"): + return "osx" + if os.startswith("freebsd"): + return "freebsd" + if os.startswith("openbsd"): + return "openbsd" + if os.startswith("linux"): + return "linux" + if os.startswith("windows"): + return "windows" + return os + +# TODO @aignas 2024-05-13: consider using the same platform tags as are used in +# the //python:versions.bzl +DEFAULT_PLATFORMS = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", +] + +def _default_platforms(*, filter): + if not filter: + fail("Must specific a filter string, got: {}".format(filter)) + + sanitized = filter.replace("*", "").replace("_", "") + if sanitized and not sanitized.isalnum(): + fail("The platform filter can only contain '*', '_' and alphanumerics") + + if "*" in filter: + prefix = filter.rstrip("*") + if "*" in prefix: + fail("The filter can only contain '*' at the end of it") + + if not prefix: + return DEFAULT_PLATFORMS + + return [p for p in DEFAULT_PLATFORMS if p.startswith(prefix)] + else: + return [p for p in DEFAULT_PLATFORMS if filter in p] + +def _platforms_from_args(extra_pip_args): + platform_values = [] + + for arg in extra_pip_args: + if platform_values and platform_values[-1] == "": + platform_values[-1] = arg + continue + + if arg == "--platform": + platform_values.append("") + continue + + if not arg.startswith("--platform"): + continue + + _, _, plat = arg.partition("=") + if not plat: + _, _, plat = arg.partition(" ") + if plat: + platform_values.append(plat) + else: + platform_values.append("") + + if not platform_values: + return [] + + platforms = { + p.target_platform: None + for arg in platform_values + for p in whl_target_platforms(arg) + } + return list(platforms.keys()) + +def parse_requirements( + ctx, + *, + requirements_by_platform = {}, + requirements_osx = None, + requirements_linux = None, + requirements_lock = None, + requirements_windows = None, + extra_pip_args = [], + fail_fn = fail): + """Get the requirements with platforms that the requirements apply to. + + Args: + ctx: A context that has .read function that would read contents from a label. + requirements_by_platform (label_keyed_string_dict): a way to have + different package versions (or different packages) for different + os, arch combinations. + requirements_osx (label): The requirements file for the osx OS. + requirements_linux (label): The requirements file for the linux OS. + requirements_lock (label): The requirements file for all OSes, or used as a fallback. + requirements_windows (label): The requirements file for windows OS. + extra_pip_args (string list): Extra pip arguments to perform extra validations and to + be joined with args fined in files. + fail_fn (Callable[[str], None]): A failure function used in testing failure cases. + + Returns: + A tuple where the first element a dict of dicts where the first key is + the normalized distribution name (with underscores) and the second key + is the requirement_line, then value and the keys are structs with the + following attributes: + * distribution: The non-normalized distribution name. + * srcs: The Simple API downloadable source list. + * requirement_line: The original requirement line. + * target_platforms: The list of target platforms that this package is for. + + The second element is extra_pip_args should be passed to `whl_library`. + """ + if not ( + requirements_lock or + requirements_linux or + requirements_osx or + requirements_windows or + requirements_by_platform + ): + fail_fn( + "A 'requirements_lock' attribute must be specified, a platform-specific lockfiles " + + "via 'requirements_by_platform' or an os-specific lockfiles must be specified " + + "via 'requirements_*' attributes", + ) + return None + + platforms = _platforms_from_args(extra_pip_args) + + if platforms: + lock_files = [ + f + for f in [ + requirements_lock, + requirements_linux, + requirements_osx, + requirements_windows, + ] + list(requirements_by_platform.keys()) + if f + ] + + if len(lock_files) > 1: + # If the --platform argument is used, check that we are using + # a single `requirements_lock` file instead of the OS specific ones as that is + # the only correct way to use the API. + fail_fn("only a single 'requirements_lock' file can be used when using '--platform' pip argument, consider specifying it via 'requirements_lock' attribute") + return None + + files_by_platform = [ + (lock_files[0], platforms), + ] + else: + files_by_platform = { + file: [ + platform + for filter_or_platform in specifier.split(",") + for platform in (_default_platforms(filter = filter_or_platform) if filter_or_platform.endswith("*") else [filter_or_platform]) + ] + for file, specifier in requirements_by_platform.items() + }.items() + + for f in [ + # If the users need a greater span of the platforms, they should consider + # using the 'requirements_by_platform' attribute. + (requirements_linux, _default_platforms(filter = "linux_*")), + (requirements_osx, _default_platforms(filter = "osx_*")), + (requirements_windows, _default_platforms(filter = "windows_*")), + (requirements_lock, None), + ]: + if f[0]: + files_by_platform.append(f) + + configured_platforms = {} + + options = {} + requirements = {} + for file, plats in files_by_platform: + if plats: + for p in plats: + if p in configured_platforms: + fail_fn( + "Expected the platform '{}' to be map only to a single requirements file, but got multiple: '{}', '{}'".format( + p, + configured_platforms[p], + file, + ), + ) + return None + configured_platforms[p] = file + else: + plats = [ + p + for p in DEFAULT_PLATFORMS + if p not in configured_platforms + ] + + contents = ctx.read(file) + + # Parse the requirements file directly in starlark to get the information + # needed for the whl_libary declarations later. + parse_result = parse(contents) + + # Replicate a surprising behavior that WORKSPACE builds allowed: + # Defining a repo with the same name multiple times, but only the last + # definition is respected. + # The requirement lines might have duplicate names because lines for extras + # are returned as just the base package name. e.g., `foo[bar]` results + # in an entry like `("foo", "foo[bar] == 1.0 ...")`. + requirements_dict = { + normalize_name(entry[0]): entry + for entry in sorted( + parse_result.requirements, + # Get the longest match and fallback to original WORKSPACE sorting, + # which should get us the entry with most extras. + # + # FIXME @aignas 2024-05-13: The correct behaviour might be to get an + # entry with all aggregated extras, but it is unclear if we + # should do this now. + key = lambda x: (len(x[1].partition("==")[0]), x), + ) + }.values() + + tokenized_options = [] + for opt in parse_result.options: + for p in opt.split(" "): + tokenized_options.append(p) + + pip_args = tokenized_options + extra_pip_args + for p in plats: + requirements[p] = requirements_dict + options[p] = pip_args + + requirements_by_platform = {} + for target_platform, reqs_ in requirements.items(): + extra_pip_args = options[target_platform] + + for distribution, requirement_line in reqs_: + for_whl = requirements_by_platform.setdefault( + normalize_name(distribution), + {}, + ) + + for_req = for_whl.setdefault( + (requirement_line, ",".join(extra_pip_args)), + struct( + distribution = distribution, + srcs = get_simpleapi_sources(requirement_line), + requirement_line = requirement_line, + target_platforms = [], + extra_pip_args = extra_pip_args, + download = len(platforms) > 0, + ), + ) + for_req.target_platforms.append(target_platform) + + return { + whl_name: [ + struct( + distribution = r.distribution, + srcs = r.srcs, + requirement_line = r.requirement_line, + target_platforms = sorted(r.target_platforms), + extra_pip_args = r.extra_pip_args, + download = r.download, + ) + for r in sorted(reqs.values(), key = lambda r: r.requirement_line) + ] + for whl_name, reqs in requirements_by_platform.items() + } + +def select_requirement(requirements, *, platform): + """A simple function to get a requirement for a particular platform. + + Args: + requirements (list[struct]): The list of requirements as returned by + the `parse_requirements` function above. + platform (str): The host platform. Usually an output of the + `host_platform` function. + + Returns: + None if not found or a struct returned as one of the values in the + parse_requirements function. The requirement that should be downloaded + by the host platform will be returned. + """ + maybe_requirement = [ + req + for req in requirements + if platform in req.target_platforms or req.download + ] + if not maybe_requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + return None + + return maybe_requirement[0] + +def host_platform(repository_os): + """Return a string representation of the repository OS. + + Args: + repository_os (struct): The `module_ctx.os` or `repository_ctx.os` attribute. + See https://bazel.build/rules/lib/builtins/repository_os.html + + Returns: + The string representation of the platform that we can later used in the `pip` + machinery. + """ + return "{}_{}".format( + _translate_os(repository_os.name.lower()), + _translate_cpu(repository_os.arch.lower()), + ) diff --git a/python/private/pypi_index.bzl b/python/private/pypi_index.bzl index 28f1007b48..64d908e32b 100644 --- a/python/private/pypi_index.bzl +++ b/python/private/pypi_index.bzl @@ -17,8 +17,6 @@ A file that houses private functions used in the `bzlmod` extension with the sam """ load("@bazel_features//:features.bzl", "bazel_features") -load("@bazel_skylib//lib:sets.bzl", "sets") -load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load(":auth.bzl", "get_auth") load(":envsubst.bzl", "envsubst") load(":normalize_name.bzl", "normalize_name") @@ -68,7 +66,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True): async_downloads = {} contents = {} index_urls = [attr.index_url] + attr.extra_index_urls - for pkg in get_packages_from_requirements(attr.sources): + for pkg in attr.sources: pkg_normalized = normalize_name(pkg) success = False @@ -204,62 +202,6 @@ def _read_index_result(ctx, result, output, url, cache, cache_key): else: return struct(success = False) -def get_packages_from_requirements(requirements_files): - """Get Simple API sources from a list of requirements files and merge them. - - Args: - requirements_files(list[str]): A list of requirements files contents. - - Returns: - A list. - """ - want_packages = sets.make() - for contents in requirements_files: - parse_result = parse_requirements(contents) - for distribution, _ in parse_result.requirements: - # NOTE: we'll be querying the PyPI servers multiple times if the - # requirements contains non-normalized names, but this is what user - # is specifying to us. - sets.insert(want_packages, distribution) - - return sets.to_list(want_packages) - -def get_simpleapi_sources(line): - """Get PyPI sources from a requirements.txt line. - - We interpret the spec described in - https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers - - Args: - line(str): The requirements.txt entry. - - Returns: - A struct with shas attribute containing a list of shas to download from pypi_index. - """ - head, _, maybe_hashes = line.partition(";") - _, _, version = head.partition("==") - version = version.partition(" ")[0].strip() - - if "@" in head: - shas = [] - else: - maybe_hashes = maybe_hashes or line - shas = [ - sha.strip() - for sha in maybe_hashes.split("--hash=sha256:")[1:] - ] - - if head == line: - head = line.partition("--hash=")[0].strip() - else: - head = head + ";" + maybe_hashes.partition("--hash=")[0].strip() - - return struct( - requirement = line if not shas else head, - version = version, - shas = sorted(shas), - ) - def parse_simple_api_html(*, url, content): """Get the package URLs for given shas by parsing the Simple API HTML. diff --git a/python/private/pypi_index_sources.bzl b/python/private/pypi_index_sources.bzl new file mode 100644 index 0000000000..470a8c9f5a --- /dev/null +++ b/python/private/pypi_index_sources.bzl @@ -0,0 +1,53 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A file that houses private functions used in the `bzlmod` extension with the same name. +""" + +def get_simpleapi_sources(line): + """Get PyPI sources from a requirements.txt line. + + We interpret the spec described in + https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers + + Args: + line(str): The requirements.txt entry. + + Returns: + A struct with shas attribute containing a list of shas to download from pypi_index. + """ + head, _, maybe_hashes = line.partition(";") + _, _, version = head.partition("==") + version = version.partition(" ")[0].strip() + + if "@" in head: + shas = [] + else: + maybe_hashes = maybe_hashes or line + shas = [ + sha.strip() + for sha in maybe_hashes.split("--hash=sha256:")[1:] + ] + + if head == line: + head = line.partition("--hash=")[0].strip() + else: + head = head + ";" + maybe_hashes.partition("--hash=")[0].strip() + + return struct( + requirement = line if not shas else head, + version = version, + shas = sorted(shas), + ) diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl index 4e17f2b4c7..14e178a66b 100644 --- a/python/private/whl_target_platforms.bzl +++ b/python/private/whl_target_platforms.bzl @@ -33,39 +33,6 @@ _LEGACY_ALIASES = { "manylinux2014_x86_64": "manylinux_2_17_x86_64", } -# _translate_cpu and _translate_os from @platforms//host:extension.bzl -def _translate_cpu(arch): - if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]: - return "x86_32" - if arch in ["amd64", "x86_64", "x64"]: - return "x86_64" - if arch in ["ppc", "ppc64", "ppc64le"]: - return "ppc" - if arch in ["arm", "armv7l"]: - return "arm" - if arch in ["aarch64"]: - return "aarch64" - if arch in ["s390x", "s390"]: - return "s390x" - if arch in ["mips64el", "mips64"]: - return "mips64" - if arch in ["riscv64"]: - return "riscv64" - return None - -def _translate_os(os): - if os.startswith("mac os"): - return "osx" - if os.startswith("freebsd"): - return "freebsd" - if os.startswith("openbsd"): - return "openbsd" - if os.startswith("linux"): - return "linux" - if os.startswith("windows"): - return "windows" - return None - # The order of the dictionaries is to keep definitions with their aliases next to each # other _CPU_ALIASES = { @@ -151,14 +118,13 @@ def _whl_priority(value): # Windows does not have multiple wheels for the same target platform return (False, False, 0, 0) -def select_whl(*, whls, want_abis, want_os, want_cpu): +def select_whl(*, whls, want_abis, want_platform): """Select a suitable wheel from a list. Args: whls(list[struct]): A list of candidates. want_abis(list[str]): A list of ABIs that are supported. - want_os(str): The module_ctx.os.name. - want_cpu(str): The module_ctx.os.arch. + want_platform(str): The target platform. Returns: None or a struct with `url`, `sha256` and `filename` attributes for the @@ -209,10 +175,7 @@ def select_whl(*, whls, want_abis, want_os, want_cpu): target_plats[p] = sorted(platform_tags, key = _whl_priority) - want = target_plats.get("{}_{}".format( - _translate_os(want_os), - _translate_cpu(want_cpu), - )) + want = target_plats.get(want_platform) if not want: return want diff --git a/tests/private/parse_requirements/BUILD.bazel b/tests/private/parse_requirements/BUILD.bazel new file mode 100644 index 0000000000..3d7976e406 --- /dev/null +++ b/tests/private/parse_requirements/BUILD.bazel @@ -0,0 +1,3 @@ +load(":parse_requirements_tests.bzl", "parse_requirements_test_suite") + +parse_requirements_test_suite(name = "parse_requirements_tests") diff --git a/tests/private/parse_requirements/parse_requirements_tests.bzl b/tests/private/parse_requirements/parse_requirements_tests.bzl new file mode 100644 index 0000000000..0d6cd4e0e0 --- /dev/null +++ b/tests/private/parse_requirements/parse_requirements_tests.bzl @@ -0,0 +1,374 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility + +def _mock_ctx(): + testdata = { + "requirements_direct": """\ +foo[extra] @ https://some-url +""", + "requirements_linux": """\ +foo==0.0.3 --hash=sha256:deadbaaf +""", + "requirements_lock": """\ +foo[extra]==0.0.1 --hash=sha256:deadbeef +""", + "requirements_lock_dupe": """\ +foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef +foo==0.0.1 --hash=sha256:deadbeef +foo[extra]==0.0.1 --hash=sha256:deadbeef +""", + "requirements_osx": """\ +foo==0.0.3 --hash=sha256:deadbaaf +""", + "requirements_windows": """\ +foo[extra]==0.0.2 --hash=sha256:deadbeef +bar==0.0.1 --hash=sha256:deadb00f +""", + } + + return struct( + os = struct( + name = "linux", + arch = "x86_64", + ), + read = lambda x: testdata[x], + ) + +_tests = [] + +def _test_fail_no_requirements(env): + errors = [] + parse_requirements( + ctx = _mock_ctx(), + fail_fn = errors.append, + ) + env.expect.that_str(errors[0]).equals("""\ +A 'requirements_lock' attribute must be specified, a platform-specific lockfiles via 'requirements_by_platform' or an os-specific lockfiles must be specified via 'requirements_*' attributes""") + +_tests.append(_test_fail_no_requirements) + +def _test_simple(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_lock = "requirements_lock", + ) + got_alternative = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_lock": "*", + }, + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra]==0.0.1", + shas = ["deadbeef"], + version = "0.0.1", + ), + target_platforms = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + ), + ], + }) + env.expect.that_dict(got).contains_exactly(got_alternative) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "linux_ppc", + ).srcs.version, + ).equals("0.0.1") + +_tests.append(_test_simple) + +def _test_dupe_requirements(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_lock = "requirements_lock_dupe", + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra,extra_2]==0.0.1", + shas = ["deadbeef"], + version = "0.0.1", + ), + target_platforms = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + ), + ], + }) + +_tests.append(_test_dupe_requirements) + +def _test_multi_os(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_linux = "requirements_linux", + requirements_osx = "requirements_osx", + requirements_windows = "requirements_windows", + ) + + # This is an alternative way to express the same intent + got_alternative = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_linux": "linux_*", + "requirements_osx": "osx_*", + "requirements_windows": "windows_*", + }, + ) + + env.expect.that_dict(got).contains_exactly({ + "bar": [ + struct( + distribution = "bar", + download = False, + extra_pip_args = [], + requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", + srcs = struct( + requirement = "bar==0.0.1", + shas = ["deadb00f"], + version = "0.0.1", + ), + target_platforms = ["windows_x86_64"], + ), + ], + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + srcs = struct( + requirement = "foo==0.0.3", + shas = ["deadbaaf"], + version = "0.0.3", + ), + target_platforms = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + ], + ), + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.2 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra]==0.0.2", + shas = ["deadbeef"], + version = "0.0.2", + ), + target_platforms = ["windows_x86_64"], + ), + ], + }) + env.expect.that_dict(got).contains_exactly(got_alternative) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "windows_x86_64", + ).srcs.version, + ).equals("0.0.2") + +_tests.append(_test_multi_os) + +def _test_fail_duplicate_platforms(env): + errors = [] + parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_linux": "linux_x86_64", + "requirements_lock": "*", + }, + fail_fn = errors.append, + ) + env.expect.that_collection(errors).has_size(1) + env.expect.that_str(",".join(errors)).equals("Expected the platform 'linux_x86_64' to be map only to a single requirements file, but got multiple: 'requirements_linux', 'requirements_lock'") + +_tests.append(_test_fail_duplicate_platforms) + +def _test_multi_os_download_only_platform(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_lock = "requirements_linux", + extra_pip_args = [ + "--platform", + "manylinux_2_27_x86_64", + "--platform=manylinux_2_12_x86_64", + "--platform manylinux_2_5_x86_64", + ], + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = True, + extra_pip_args = [ + "--platform", + "manylinux_2_27_x86_64", + "--platform=manylinux_2_12_x86_64", + "--platform manylinux_2_5_x86_64", + ], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + srcs = struct( + requirement = "foo==0.0.3", + shas = ["deadbaaf"], + version = "0.0.3", + ), + target_platforms = ["linux_x86_64"], + ), + ], + }) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "windows_x86_64", + ).srcs.version, + ).equals("0.0.3") + +_tests.append(_test_multi_os_download_only_platform) + +def _test_fail_download_only_bad_attr(env): + errors = [] + parse_requirements( + ctx = _mock_ctx(), + requirements_linux = "requirements_linux", + requirements_osx = "requirements_osx", + extra_pip_args = [ + "--platform", + "manylinux_2_27_x86_64", + "--platform=manylinux_2_12_x86_64", + "--platform manylinux_2_5_x86_64", + ], + fail_fn = errors.append, + ) + env.expect.that_str(errors[0]).equals("only a single 'requirements_lock' file can be used when using '--platform' pip argument, consider specifying it via 'requirements_lock' attribute") + +_tests.append(_test_fail_download_only_bad_attr) + +def _test_os_arch_requirements_with_default(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_direct": "linux_super_exotic", + "requirements_linux": "linux_x86_64,linux_aarch64", + }, + requirements_lock = "requirements_lock", + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + srcs = struct( + requirement = "foo==0.0.3", + shas = ["deadbaaf"], + version = "0.0.3", + ), + target_platforms = ["linux_aarch64", "linux_x86_64"], + ), + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra] @ https://some-url", + srcs = struct( + requirement = "foo[extra] @ https://some-url", + shas = [], + version = "", + ), + target_platforms = ["linux_super_exotic"], + ), + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra]==0.0.1", + shas = ["deadbeef"], + version = "0.0.1", + ), + target_platforms = [ + "linux_arm", + "linux_ppc", + "linux_s390x", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + ), + ], + }) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "windows_x86_64", + ).srcs.version, + ).equals("0.0.1") + env.expect.that_str( + select_requirement( + got["foo"], + platform = "linux_x86_64", + ).srcs.version, + ).equals("0.0.3") + +_tests.append(_test_os_arch_requirements_with_default) + +def parse_requirements_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/private/pypi_index/pypi_index_tests.bzl b/tests/private/pypi_index/pypi_index_tests.bzl index e2122b5eeb..fa381065b1 100644 --- a/tests/private/pypi_index/pypi_index_tests.bzl +++ b/tests/private/pypi_index/pypi_index_tests.bzl @@ -16,42 +16,10 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") -load("//python/private:pypi_index.bzl", "get_simpleapi_sources", "parse_simple_api_html") # buildifier: disable=bzl-visibility +load("//python/private:pypi_index.bzl", "parse_simple_api_html") # buildifier: disable=bzl-visibility _tests = [] -def _test_no_simple_api_sources(env): - inputs = [ - "foo==0.0.1", - "foo==0.0.1 @ https://someurl.org", - "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", - "foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef", - ] - for input in inputs: - got = get_simpleapi_sources(input) - env.expect.that_collection(got.shas).contains_exactly([]) - env.expect.that_str(got.version).equals("0.0.1") - -_tests.append(_test_no_simple_api_sources) - -def _test_simple_api_sources(env): - tests = { - "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef": [ - "deadbeef", - "deafbeef", - ], - "foo[extra]==0.0.2; (python_version < 2.7 or something_else == \"@\") --hash=sha256:deafbeef --hash=sha256:deadbeef": [ - "deadbeef", - "deafbeef", - ], - } - for input, want_shas in tests.items(): - got = get_simpleapi_sources(input) - env.expect.that_collection(got.shas).contains_exactly(want_shas) - env.expect.that_str(got.version).equals("0.0.2") - -_tests.append(_test_simple_api_sources) - def _generate_html(*items): return """\ diff --git a/tests/private/pypi_index_sources/BUILD.bazel b/tests/private/pypi_index_sources/BUILD.bazel new file mode 100644 index 0000000000..212615f480 --- /dev/null +++ b/tests/private/pypi_index_sources/BUILD.bazel @@ -0,0 +1,3 @@ +load(":pypi_index_sources_tests.bzl", "pypi_index_sources_test_suite") + +pypi_index_sources_test_suite(name = "pypi_index_sources_tests") diff --git a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl b/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl new file mode 100644 index 0000000000..48d790fc68 --- /dev/null +++ b/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl @@ -0,0 +1,60 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:pypi_index_sources.bzl", "get_simpleapi_sources") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_no_simple_api_sources(env): + inputs = [ + "foo==0.0.1", + "foo==0.0.1 @ https://someurl.org", + "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", + "foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef", + ] + for input in inputs: + got = get_simpleapi_sources(input) + env.expect.that_collection(got.shas).contains_exactly([]) + env.expect.that_str(got.version).equals("0.0.1") + +_tests.append(_test_no_simple_api_sources) + +def _test_simple_api_sources(env): + tests = { + "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef": [ + "deadbeef", + "deafbeef", + ], + "foo[extra]==0.0.2; (python_version < 2.7 or something_else == \"@\") --hash=sha256:deafbeef --hash=sha256:deadbeef": [ + "deadbeef", + "deafbeef", + ], + } + for input, want_shas in tests.items(): + got = get_simpleapi_sources(input) + env.expect.that_collection(got.shas).contains_exactly(want_shas) + env.expect.that_str(got.version).equals("0.0.2") + +_tests.append(_test_simple_api_sources) + +def pypi_index_sources_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/private/whl_target_platforms/select_whl_tests.bzl b/tests/private/whl_target_platforms/select_whl_tests.bzl index 0d6f97d7a5..bed6d6633c 100644 --- a/tests/private/whl_target_platforms/select_whl_tests.bzl +++ b/tests/private/whl_target_platforms/select_whl_tests.bzl @@ -83,37 +83,37 @@ def _match(env, got, want_filename): _tests = [] def _test_selecting(env): - got = select_whl(whls = WHL_LIST, want_abis = ["none"], want_os = "ignored", want_cpu = "ignored") + got = select_whl(whls = WHL_LIST, want_abis = ["none"], want_platform = "ignored") _match(env, got, "pkg-0.0.1-py3-none-any.whl") - got = select_whl(whls = WHL_LIST, want_abis = ["abi3"], want_os = "ignored", want_cpu = "ignored") + got = select_whl(whls = WHL_LIST, want_abis = ["abi3"], want_platform = "ignored") _match(env, got, "pkg-0.0.1-py3-abi3-any.whl") # Check the selection failure - got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_os = "fancy", want_cpu = "exotic") + got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_platform = "fancy_exotic") _match(env, got, None) # Check we match the ABI and not the py version - got = select_whl(whls = WHL_LIST, want_abis = ["cp37m"], want_os = "linux", want_cpu = "amd64") + got = select_whl(whls = WHL_LIST, want_abis = ["cp37m"], want_platform = "linux_x86_64") _match(env, got, "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") # Check we can select a filename with many platform tags - got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_os = "linux", want_cpu = "i686") + got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_platform = "linux_x86_32") _match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl") # Check that we prefer the specific wheel - got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_os = "mac os", want_cpu = "x86_64") + got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_platform = "osx_x86_64") _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl") - got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_os = "mac os", want_cpu = "aarch64") + got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_platform = "osx_aarch64") _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl") # Check that we can use the universal2 if the arm wheel is not available - got = select_whl(whls = [w for w in WHL_LIST if "arm64" not in w.filename], want_abis = ["cp311"], want_os = "mac os", want_cpu = "aarch64") + got = select_whl(whls = [w for w in WHL_LIST if "arm64" not in w.filename], want_abis = ["cp311"], want_platform = "osx_aarch64") _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl") # Check we prefer platform specific wheels - got = select_whl(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_os = "linux", want_cpu = "x86_64") + got = select_whl(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_platform = "linux_x86_64") _match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") _tests.append(_test_selecting)