feat: deploy a single py2.py3 wheel per platform

Add python 2.7 tests

Check exactly 1 final wheel per build exists in ./dist

Only rely on auditwheel to add manylinux* platform tags

Remove dist-staging

Author: mayeut

.circleci/config.yml

@@ -21,6 +21,9 @@ references:
             echo "MANYLINUX_PYTHON [${MANYLINUX_PYTHON}]"
             /opt/python/${MANYLINUX_PYTHON}/bin/pip install scikit-ci
             /opt/python/${MANYLINUX_PYTHON}/bin/ci
+            if [ "${UPLOAD_SDIST}" == "tests-only" ]; then
+              rm -f ./dist/*
+            fi
       - persist_to_workspace:
           root: ./
           paths:
@@ -44,17 +47,17 @@ references:
 jobs:
 
   # x64
-  manylinux-x64_cp27-cp27mu:
-    <<: *x64_build_job
   manylinux-x64_cp37-cp37m_upload-sdist:
     <<: *x64_build_job
 
   # x86
-  manylinux-x86_cp27-cp27mu:
-    <<: *x86_build_job
   manylinux-x86_cp37-cp37m:
     <<: *x86_build_job
 
+  # python 2.7 tests
+  manylinux-x64_cp27-cp27mu_tests-only:
+    <<: *x64_build_job
+
   deploy-master:
     docker:
       - image: circleci/python:3.7.0-stretch
@@ -87,35 +90,35 @@ workflows:
   build-test-deploy:
     jobs:
       # x64
-      - manylinux-x64_cp27-cp27mu:
-          <<: *no_filters
       - manylinux-x64_cp37-cp37m_upload-sdist:
           <<: *no_filters
       # x86
-      - manylinux-x86_cp27-cp27mu:
-          <<: *no_filters
       - manylinux-x86_cp37-cp37m:
           <<: *no_filters
 
+      # python 2.7 tests
+      - manylinux-x64_cp27-cp27mu_tests-only:
+          <<: *no_filters
+
       - deploy-master:
           requires:
             # x64
-            - manylinux-x64_cp27-cp27mu
             - manylinux-x64_cp37-cp37m_upload-sdist
             # x86
-            - manylinux-x86_cp27-cp27mu
             - manylinux-x86_cp37-cp37m
+            # python 2.7 test
+            - manylinux-x64_cp27-cp27mu_tests-only
           filters:
             branches:
               only: master
       - deploy-release:
           requires:
             # x64
-            - manylinux-x64_cp27-cp27mu
             - manylinux-x64_cp37-cp37m_upload-sdist
             # x86
-            - manylinux-x86_cp27-cp27mu
             - manylinux-x86_cp37-cp37m
+            # python 2.7 test
+            - manylinux-x64_cp27-cp27mu_tests-only
           filters:
             tags:
               only: /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/

.travis.yml

@@ -17,12 +17,6 @@ matrix:
         - PYTHON_VERSION=3.7.9
         - MACOSX_DEPLOYMENT_TARGET=10.10
 
-    - os: osx
-      language: generic
-      env:
-        - PYTHON_VERSION=2.7.18
-        - MACOSX_DEPLOYMENT_TARGET=10.10
-
     - arch: arm64-graviton2
       virt: vm
       group: edge

appveyor.yml

@@ -8,18 +8,6 @@ version: "0.0.1.{build}"
 environment:
   matrix:
 
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      PYTHON_DIR: "C:\\Python27"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "32"
-      BLOCK: "0"
-
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      PYTHON_DIR: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "64"
-      BLOCK: "0"
-
     - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       PYTHON_DIR: "C:\\Python37"
       PYTHON_VERSION: "3.7.x"

scikit-ci.yml

@@ -49,7 +49,6 @@ before_install:
                   import os
                   import ci
                   import platform
-                  os.environ["SETUP_BDIST_WHEEL_ARGS"] = "--plat-name %s" % os.environ["AUDITWHEEL_PLAT"]
                   setup_cmake_args = []
                   setup_cmake_args.append("-DSTRIP_EXECUTABLE:FILEPATH=/opt/rh/devtoolset-9/root/usr/" + "/bin/strip")
                   if platform.machine() in {"aarch64", "ppc64le", "s390x"}:
@@ -95,22 +94,19 @@ build:
         - |
           # Since there are no external shared libraries to bundle into the wheels
           # this step will fixup the wheel switching from 'linux' to 'manylinux' tag
-          for whl in dist/*linux*_$(arch).whl; do
-              auditwheel repair --plat ${AUDITWHEEL_PLAT} $whl -w ./dist
+          for whl in dist/*.whl; do
+              auditwheel repair --plat ${AUDITWHEEL_PLAT} ${whl} -w ./dist/
+              rm ${whl}
           done
 
   circle:
     commands:
       - |
-        arch=${AUDITWHEEL_ARCH}
-        if [[ ${arch} == "aarch64" ]]; then
-            exit
-        fi
         # Since there are no external shared libraries to bundle into the wheels
         # this step will fixup the wheel switching from 'linux' to 'manylinux' tag
-        for whl in dist/*linux_${arch}.whl; do
-            auditwheel repair --plat ${AUDITWHEEL_PLAT} $whl -w ./dist/
-            rm $whl
+        for whl in dist/*.whl; do
+            auditwheel repair --plat ${AUDITWHEEL_PLAT} ${whl} -w ./dist/
+            rm ${whl}
         done
 
 test:
@@ -139,7 +135,9 @@ test:
                       additional_platforms.append("macosx_11_0_universal2")
 
               for wheel in wheels:
-                  convert_to_generic_platform_wheel(wheel, remove_original=True, additional_platforms=additional_platforms)
+                  convert_to_generic_platform_wheel(wheel, remove_original=True, py2_py3=True, additional_platforms=additional_platforms)
+              wheels = glob.glob("dist/*.whl")
+              assert len(wheels) == 1, "There shall be exactly 1 final wheel per build"
 
   appveyor:
     commands:

scripts/convert_to_generic_platform_wheel.py

@@ -41,7 +41,7 @@ def _to_generic_pyver(pyver_tags):
     return ['py%s' % tag[2] if tag.startswith('cp') else tag for tag in pyver_tags]
 
 
-def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms):
+def _convert_to_generic_platform_wheel(wheel_ctx, py2_py3, additional_platforms):
     """Switch to generic python tags and remove ABI tags from a wheel
 
     Convert implementation specific python tags to their generic equivalent and
@@ -51,6 +51,8 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms):
     ----------
     wheel_ctx : InWheelCtx
         An open wheel context
+    py2_py3: Bool
+        Wether the pyver tag shall be py2.py3 or just the one inferred from the wheel name
     additional_platforms : Optional[Iterable[str]]
         An optional iterable of additional platform to add to the wheel
     """
@@ -91,6 +93,10 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms):
     original_pyver_tags = fparts['pyver'].split('.')
     logger.debug('Previous pyver tags: %s', ', '.join(original_pyver_tags))
     pyver_tags = _to_generic_pyver(original_pyver_tags)
+    if py2_py3:
+        if len(set(["py2", "py3"]) & set(pyver_tags)) == 0:
+            raise ValueError("pyver_tags does not contain py2 nor py3")
+        pyver_tags = list(sorted(set(pyver_tags + ["py2", "py3"])))
     if pyver_tags != original_pyver_tags:
         logger.debug('New pyver tags ....: %s', ', '.join(pyver_tags))
         fparts['pyver'] = '.'.join(pyver_tags)
@@ -115,8 +121,7 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms):
 
     # Python version, C-API version combinations
     pyc_apis = []
-    for tag in in_info_tags:
-        py_ver = '.'.join(_to_generic_pyver(tag.split('-')[0].split('.')))
+    for py_ver in pyver_tags:
         abi = 'none'
         pyc_apis.append('-'.join([py_ver, abi]))
     # unique Python version, C-API version combinations
@@ -138,7 +143,7 @@ def _convert_to_generic_platform_wheel(wheel_ctx, additional_platforms):
 
 
 def convert_to_generic_platform_wheel(wheel_path, out_dir='./dist/', remove_original=False, verbose=0,
-                                      additional_platforms=None):
+                                      py2_py3=False, additional_platforms=None):
     logging.disable(logging.NOTSET)
     if verbose >= 1:
         logging.basicConfig(level=logging.DEBUG)
@@ -150,7 +155,7 @@ def convert_to_generic_platform_wheel(wheel_path, out_dir='./dist/', remove_orig
 
     with InWheelCtx(wheel_path) as ctx:
         ctx.out_wheel = pjoin(out_dir, wheel_fname)
-        ctx.out_wheel = _convert_to_generic_platform_wheel(ctx, additional_platforms)
+        ctx.out_wheel = _convert_to_generic_platform_wheel(ctx, py2_py3, additional_platforms)
 
     if remove_original:
         logger.info('Removed original wheel %s' % wheel_path)
@@ -179,13 +184,23 @@ def main():
                    dest='remove_original',
                    action='store_true',
                    help='Remove original wheel')
+    p.add_argument("--py2-py3",
+                   dest='py2_py3',
+                   action='store_true',
+                   help='Remove original wheel')
+    p.add_argument("-p",
+                   "--add-platform",
+                   dest='additional_platforms',
+                   action="append",
+                   help='Add a platform tag')
 
     args = p.parse_args()
 
     if not isfile(args.WHEEL_FILE):
         p.error('cannot access %s. No such file' % args.WHEEL_FILE)
 
-    convert_to_generic_platform_wheel(args.WHEEL_FILE, args.WHEEL_DIR, args.remove_original, args.verbose)
+    convert_to_generic_platform_wheel(args.WHEEL_FILE, args.WHEEL_DIR, args.remove_original, args.verbose,
+                                      args.py2_py3, args.additional_platforms)
 
 
 if __name__ == '__main__':