From 93937028c0b321f64e51895927e007b281fe0a36 Mon Sep 17 00:00:00 2001 From: Vadym Doroshenko Date: Tue, 10 Oct 2023 10:59:59 +0200 Subject: [PATCH 1/4] Manylinux build --- build_PyDP_linux.sh | 18 ++++++++++++++++++ pyproject.toml | 10 ++++++---- setup.py | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 56 insertions(+), 7 deletions(-) create mode 100755 build_PyDP_linux.sh diff --git a/build_PyDP_linux.sh b/build_PyDP_linux.sh new file mode 100755 index 00000000..933c6464 --- /dev/null +++ b/build_PyDP_linux.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# This script differs from build_PyDP.sh by installing bazel first (if needed). +BAZELISK_VERSION=v1.8.1 +BAZELISK_BINARY=bazelisk-linux-amd64 +BAZELISK_DOWNLOAD_URL=https://github.com/bazelbuild/bazelisk/releases/download/ + +if command -v bazel &>/dev/null; then + echo "Bazel already installed" +else + python -m wget "${BAZELISK_DOWNLOAD_URL}/${BAZELISK_VERSION}/${BAZELISK_BINARY}" + chmod +x ${BAZELISK_BINARY} + mkdir bazel_dir + echo mv ${BAZELISK_BINARY} bazel_dir/bazel + mv ${BAZELISK_BINARY} bazel_dir/bazel + export PATH="$PATH:`pwd`/bazel_dir" +fi + +./build_PyDP.sh diff --git a/pyproject.toml b/pyproject.toml index 3f375831..5060cd95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ +[build-system] +requires = [ + "setuptools", + "wget" +] + [tool.poetry] name = "pydp" version = "1.1.4" @@ -34,7 +40,3 @@ sphinx = "*" sphinx-rtd-theme = "*" twine = "*" wheel = "*" - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/setup.py b/setup.py index 15de29cd..da5d69c3 100755 --- a/setup.py +++ b/setup.py @@ -2,16 +2,42 @@ # -*- coding: utf-8 -*- """The setup script.""" - -# stdlib import os +import platform +import sys from typing import List -# third party from setuptools import find_packages from setuptools import setup from setuptools.dist import Distribution +from setuptools.command.build_ext import build_ext + +CONDA_PREFIX = os.environ.get("CONDA_PREFIX", None) +WORKING_DIR = os.getcwd() + + +class Build(build_ext): + """Customized setuptools build command - builds protos on build.""" + def run(self): + if platform.system() != "Linux": + # For Windows and Mac setup.py is not used for wheel build but the + # build is performed from GitHub actions files. The special + # treatment for Linux is required because wheel build is performed + # inside manylinux Docker images with the help of cibuildwheel + # (https://github.com/pypa/cibuildwheel). Building with manylinux + # ensures that PyDP can work on old Linux versions. + return + # Build _pydp.so (wrappers for C++). + os.system("./build_PyDP_linux.sh") + + # Copy _pydp.so to cibuildwheel directory. + pydp_lib = "cp src/pydp/_pydp.so" + version_str = f"{sys.version_info.major}{sys.version_info.minor}" + destination_dir = f"build/lib.linux-x86_64-cpython-{version_str}/pydp" + os.system(f"cp {pydp_lib} {destination_dir}") + + build_ext.run(self) class BinaryDistribution(Distribution): """This class is needed in order to create OS specific wheels.""" @@ -42,6 +68,9 @@ def read(fname): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ], + cmdclass={ + 'build_ext': Build + }, description="Python API for Google's Differential Privacy library", distclass=BinaryDistribution, install_requires=requirements, From 1a6181e6df7482e1f7aea5d7c46ef2535c0aef1f Mon Sep 17 00:00:00 2001 From: Vadym Doroshenko Date: Tue, 10 Oct 2023 11:01:45 +0200 Subject: [PATCH 2/4] format --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index da5d69c3..989e0297 100755 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ class Build(build_ext): """Customized setuptools build command - builds protos on build.""" + def run(self): if platform.system() != "Linux": # For Windows and Mac setup.py is not used for wheel build but the @@ -39,6 +40,7 @@ def run(self): build_ext.run(self) + class BinaryDistribution(Distribution): """This class is needed in order to create OS specific wheels.""" @@ -68,9 +70,7 @@ def read(fname): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ], - cmdclass={ - 'build_ext': Build - }, + cmdclass={"build_ext": Build}, description="Python API for Google's Differential Privacy library", distclass=BinaryDistribution, install_requires=requirements, From cc308084b73e615f80910f9a7729333d505353d1 Mon Sep 17 00:00:00 2001 From: Vadym Doroshenko Date: Tue, 10 Oct 2023 11:26:14 +0200 Subject: [PATCH 3/4] Update publish workflow with manylinux build --- .github/workflows/publish.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index db4a9a22..9e21f7b1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -62,8 +62,20 @@ jobs: ./bazel-cache key: ${{ runner.os }}-bazel-${{ hashFiles('WORKSPACE') }} - - name: Build Google DP Unix - if: runner.os != 'Windows' + - name: Build Google DP and wheel on Linux + if: runner.os == 'Linux' + timeout-minutes: 20 + run: | + pip install cibuildwheel + # Set for which Python version to build. + py_version=${matrix.python-version//.} + EXPORT CIBW_BUILD=cp${py_version}-manylinux_x86_64 + # Build wheel + echo "Building ${CIBW_BUILD} wheel" + python -m cibuildwheel --output-dir dist . + + - name: Build Google DP Mac + if: runner.os == 'macOS' timeout-minutes: 20 run: | PYTHONHOME=$(which python) @@ -114,13 +126,13 @@ jobs: run: | poetry install - - name: Build PyDP macOS + - name: Build PyDP macOS wheel if: runner.os == 'macOS' run: | poetry run python setup.py build bdist_wheel --plat-name macosx_10_14_x86_64 - - name: Build PyDP Linux / Windows - if: runner.os != 'macOS' + - name: Build PyDP Windows wheel + if: runner.os == 'Windows' run: | poetry run python setup.py build bdist_wheel From 3272b6d609aeff031587a73f39065cdc2b1d2f14 Mon Sep 17 00:00:00 2001 From: Vadym Doroshenko Date: Tue, 10 Oct 2023 19:51:09 +0200 Subject: [PATCH 4/4] Minor fixes --- .github/workflows/publish.yml | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9e21f7b1..b8e0ba53 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -67,8 +67,9 @@ jobs: timeout-minutes: 20 run: | pip install cibuildwheel + export CIBW_PLATFORM=linux # Set for which Python version to build. - py_version=${matrix.python-version//.} + py_version=${matrix.python-version//.} # 3.11 -> 311 EXPORT CIBW_BUILD=cp${py_version}-manylinux_x86_64 # Build wheel echo "Building ${CIBW_BUILD} wheel" diff --git a/setup.py b/setup.py index 989e0297..4d05ebac 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def run(self): os.system("./build_PyDP_linux.sh") # Copy _pydp.so to cibuildwheel directory. - pydp_lib = "cp src/pydp/_pydp.so" + pydp_lib = "src/pydp/_pydp.so" version_str = f"{sys.version_info.major}{sys.version_info.minor}" destination_dir = f"build/lib.linux-x86_64-cpython-{version_str}/pydp" os.system(f"cp {pydp_lib} {destination_dir}")