From acea3d4bacde2cc26f3e6c9dc516aa62f58e73a9 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Fri, 17 Mar 2023 14:20:25 +0000 Subject: [PATCH] ci: build Python wheels This change adds extra steps to the CI workflows to build Python wheels so that the Austin binary can be installed with pip from PyPI. --- .github/workflows/build_arch.yml | 45 +++++- .github/workflows/release.yml | 90 ++++++++++++ .github/workflows/release_arch.yml | 47 +++++++ .github/workflows/tests.yml | 133 +++++++++++++++++- README.md | 30 +++- scripts/build-wheel.py | 215 +++++++++++++++++++++++++++++ scripts/build_arch.sh | 5 + scripts/requirements-bw.txt | 2 + 8 files changed, 557 insertions(+), 10 deletions(-) create mode 100644 scripts/build-wheel.py create mode 100644 scripts/requirements-bw.txt diff --git a/.github/workflows/build_arch.yml b/.github/workflows/build_arch.yml index 42a0cf05..860e6202 100644 --- a/.github/workflows/build_arch.yml +++ b/.github/workflows/build_arch.yml @@ -18,9 +18,9 @@ jobs: steps: - uses: actions/checkout@v2 name: Checkout sources - + - uses: uraimo/run-on-arch-action@v2.0.5 - name: Build on ${{ matrix.arch }} + name: Build Austin on ${{ matrix.arch }} id: build-on-arch with: arch: ${{ matrix.arch }} @@ -31,6 +31,47 @@ jobs: mkdir -p ./artifacts run: ARCH=${{ matrix.arch }} bash scripts/build_arch.sh + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Build wheels on ${{ matrix.arch }} + run: | + python3.10 -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r scripts/requirements-bw.txt + + export VERSION=$(cat src/austin.h | sed -r -n "s/^#define VERSION[ ]+\"(.+)\"/\1/p"); + + case ${{ matrix.arch }} in + armv7) + PLATFORM=manylinux_2_17_armv7l.manylinux2014_armv7l + MUSL_PLATFORM=musllinux_1_1_armv7l + ;; + aarch64) + PLATFORM=manylinux_2_17_aarch64.manylinux2014_aarch64 + MUSL_PLATFORM=musllinux_1_1_aarch64 + ;; + ppc64le) + PLATFORM=manylinux_2_17_ppc64le.manylinux2014_ppc64le + MUSL_PLATFORM=musllinux_1_1_ppc64le + ;; + esac + + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=$PLATFORM \ + --files austin:./artifacts/austin austinp:./artifacts/austinp + + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=$MUSL_PLATFORM \ + --files austin:./artifacts/austin.musl + + deactivate + - name: Show artifacts run: | ls -al ./artifacts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc5a082c..e06100f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,8 +13,18 @@ jobs: - uses: actions/checkout@v2 name: Checkout Austin + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Generate artifacts run: | + python3.10 -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r scripts/requirements-bw.txt + sudo apt-get update sudo apt-get -y install autoconf build-essential libunwind-dev binutils-dev libiberty-dev musl-tools zlib1g-dev @@ -30,12 +40,26 @@ jobs: tar -Jcf austinp-$VERSION-gnu-linux-amd64.tar.xz austinp popd + # Build gnu wheel + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=manylinux_2_12_x86_64.manylinux2010_x86_64 \ + --files austin:src/austin austinp:src/austinp + # Build with musl musl-gcc -O3 -Os -s -Wall -pthread src/*.c -o src/austin -D__MUSL__ pushd src tar -Jcf austin-$VERSION-musl-linux-amd64.tar.xz austin popd + # Build musl wheel + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=musllinux_1_1_x86_64 \ + --files austin:src/austin + + deactivate + - name: Upload artifacts to release uses: svenstaro/upload-release-action@v2 with: @@ -44,6 +68,12 @@ jobs: tag: ${{ github.ref }} overwrite: true file_glob: true + + - name: Upload Python wheels to PyPI + run: | + source .venv/bin/activate + twine upload dist/*.whl --username __token__ --password ${{ secrets.PYPI_TOKEN }} + deactivate release-win: runs-on: windows-latest @@ -109,6 +139,29 @@ jobs: overwrite: true file_glob: true + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Build Python wheels + shell: bash + run: | + py -3.10 -m pip install --upgrade pip + py -3.10 -m pip install -r scripts/requirements-bw.txt + + export VERSION=$(cat src/austin.h | sed -r -n "s/^#define VERSION[ ]+\"(.+)\"/\1/p") + + py -3.10 scripts/build-wheel.py \ + --version=$VERSION \ + --platform=win_amd64 \ + --files austin.exe:src/austin.exe + + - name: Upload Python wheels to PyPI + shell: bash + run: | + py -3.10 -m twine upload dist/*.whl --username __token__ --password ${{ secrets.PYPI_TOKEN }} + release-osx: runs-on: macos-latest strategy: @@ -118,8 +171,18 @@ jobs: - uses: actions/checkout@v2 name: Checkout Austin + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Generate artifacts run: | + python3.10 -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r scripts/requirements-bw.txt + export VERSION=$(cat src/austin.h | sed -n -E "s/^#define VERSION[ ]+\"(.+)\"/\1/p") echo "::set-output name=version::$VERSION" @@ -129,6 +192,26 @@ jobs: zip -r austin-${VERSION}-mac64.zip austin popd + # Build intel wheel + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=macosx_11_0_x86_64 \ + --files austin:src/austin + + clang -Wall -O3 -Os -o src/austin src/*.c -target arm64-apple-macos11 + + pushd src + zip -r austin-${VERSION}-mac-arm64.zip austin + popd + + # Build arm wheel + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=macosx_11_0_arm64 \ + --files austin:src/austin + + deactivate + - name: Upload artifacts to release uses: svenstaro/upload-release-action@v2 with: @@ -137,3 +220,10 @@ jobs: tag: ${{ github.ref }} overwrite: true file_glob: true + + - name: Upload Python wheels to PyPI + shell: bash + run: | + source .venv/bin/activate + twine upload dist/*.whl --username __token__ --password ${{ secrets.PYPI_TOKEN }} + deactivate diff --git a/.github/workflows/release_arch.yml b/.github/workflows/release_arch.yml index 78ab404f..3c513b34 100644 --- a/.github/workflows/release_arch.yml +++ b/.github/workflows/release_arch.yml @@ -39,3 +39,50 @@ jobs: tag: ${{ github.ref }} overwrite: true file_glob: true + + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Build wheels on ${{ matrix.arch }} + run: | + python3.10 -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r scripts/requirements-bw.txt + + export VERSION=$(cat src/austin.h | sed -r -n "s/^#define VERSION[ ]+\"(.+)\"/\1/p"); + + case ${{ matrix.arch }} in + armv7) + PLATFORM=manylinux_2_17_armv7l.manylinux2014_armv7l + MUSL_PLATFORM=musllinux_1_1_armv7l + ;; + aarch64) + PLATFORM=manylinux_2_17_aarch64.manylinux2014_aarch64 + MUSL_PLATFORM=musllinux_1_1_aarch64 + ;; + ppc64le) + PLATFORM=manylinux_2_17_ppc64le.manylinux2014_ppc64le + MUSL_PLATFORM=musllinux_1_1_ppc64le + ;; + esac + + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=$PLATFORM \ + --files austin:./artifacts/austin austinp:./artifacts/austinp + + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=$MUSL_PLATFORM \ + --files austin:./artifacts/austin.musl + + deactivate + + - name: Upload wheels + run: | + source .venv/bin/activate + twine upload dist/*.whl --username __token__ --password ${{ secrets.PYPI_TOKEN }}s + deactivate diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b7b06e75..19569d35 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,10 +35,30 @@ jobs: src/austin src/austinp + build-linux-musl: + runs-on: ubuntu-20.04 + name: Build Austin on Linux (musl) + steps: + - uses: actions/checkout@v2 + + - name: Install build dependencies + run: | + sudo apt-get -y install musl-tools + + - name: Compile Austin + run: | + musl-gcc -O3 -Os -s -Wall -pthread src/*.c -o src/austin.musl -D__MUSL__ + + - uses: actions/upload-artifact@v3 + with: + name: austin-binaries + path: | + src/austin.musl + tests-linux: runs-on: ubuntu-20.04 - needs: build-linux + needs: [build-linux, build-linux-musl] strategy: fail-fast: false @@ -86,6 +106,48 @@ jobs: .venv/bin/pytest --pastebin=failed --no-flaky-report -sr a deactivate + wheels-linux: + runs-on: ubuntu-20.04 + + needs: build-linux + + name: Build Linux wheels + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v3 + with: + name: austin-binaries + path: src + + - run: chmod +x src/austin && chmod +x src/austinp + + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Build wheels + run: | + python3.10 -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r scripts/requirements-bw.txt + + export VERSION=$(cat src/austin.h | sed -r -n "s/^#define VERSION[ ]+\"(.+)\"/\1/p"); + + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=manylinux_2_12_x86_64.manylinux2010_x86_64 \ + --files austin:src/austin austinp:src/austinp + + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=musllinux_1_1_x86_64 \ + --files austin:src/austin.musl + + deactivate + build-osx-gcc: runs-on: macos-latest @@ -166,6 +228,43 @@ jobs: sudo -E pytest --ignore=test/cunit --pastebin=failed --no-flaky-report -sr a deactivate + wheels-osx: + runs-on: macos-latest + + needs: [build-osx-gcc, build-osx-clang] + + name: Build macOS wheels + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v3 + with: + name: austin-binary + path: src + + - run: chmod +x src/austin + + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Build wheels + run: | + python3.10 -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r scripts/requirements-bw.txt + + export VERSION=$(cat src/austin.h | sed -n -E "s/^#define VERSION[ ]+\"(.+)\"/\1/p") + + python scripts/build-wheel.py \ + --version=$VERSION \ + --platform=macosx_11_0_x86_64 \ + --files austin:src/austin + + deactivate + build-win: runs-on: windows-latest @@ -226,6 +325,38 @@ jobs: python -m pytest --ignore=test\cunit --pastebin=failed --no-flaky-report -sr a deactivate + wheels-win: + runs-on: windows-latest + + needs: build-win + + name: Build Windows wheels + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v3 + with: + name: austin-binary + path: src + + - uses: actions/setup-python@v2 + name: Install Python 3.10 + with: + python-version: '3.10' + + - name: Build wheels + shell: bash + run: | + py -3.10 -m pip install --upgrade pip + py -3.10 -m pip install -r scripts/requirements-bw.txt + + export VERSION=$(cat src/austin.h | sed -r -n "s/^#define VERSION[ ]+\"(.+)\"/\1/p") + + py -3.10 scripts/build-wheel.py \ + --version=$VERSION \ + --platform=win_amd64 \ + --files austin.exe:src/austin.exe + validation: runs-on: ubuntu-20.04 diff --git a/README.md b/README.md index bef4d226..e5dc0e61 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,10 @@
- + + PyPI version + Chocolatey Version @@ -44,10 +47,10 @@ Conda Version - + homebrew @@ -154,9 +157,20 @@ project.
🙏

# Installation -Austin is available from the major software repositories of the most popular -platforms. Check out the [latest release] page for pre-compiled binaries and -installation packages. +Austin is available to install from [PyPI](pypi) and from the major software +repositories of the most popular platforms. Check out the [latest release] page +for pre-compiled binaries and installation packages. + +On all supported platforms and architectures, Austin can be installed from PyPI +with `pip` or `pipx` via the commands + +~~~ +pip install austin-dist +~~~ +or +~~~ +pipx install austin-dist +~~~ On Linux, it can be installed using `autotools` or as a snap from the [Snap Store](https://snapcraft.io/store). The latter will automatically perform the @@ -171,7 +185,7 @@ On Windows, Austin can be easily installed from the command line using either On macOS, Austin can be easily installed from the command line using [Homebrew]. Anaconda users can install Austin from [Conda Forge]. -For any other platform, compiling Austin from sources is as easy as cloning the +For any other platforms, compiling Austin from sources is as easy as cloning the repository and running the C compiler. The [Releases][releases] page has many pre-compiled binaries that are ready to be uncompressed and used. @@ -574,6 +588,7 @@ following platforms and architectures | ----------- | ------------------------- | ------------------------ | -------------------------- | | **x86_64** | ✓ | ✓ | ✓ | | **i686** | ✓ | | ✓ | +| **armv7** | ✓ | | | | **arm64** | ✓ | | ✓ | | **ppc64le** | ✓ | | | @@ -871,6 +886,7 @@ by chipping in a few pennies on [PayPal.Me](https://www.paypal.me/gtornetta/1). [MOJO]: https://github.com/P403n1x87/austin/wiki/The-MOJO-file-format [pprof]: https://github.com/google/pprof [pyenv]: https://github.com/pyenv/pyenv +[pypi]: https://pypi.org/project/austin-dist/ [releases]: https://github.com/P403n1x87/austin/releases [Scoop]: https://scoop.sh/ [Speedscope]: https://speedscope.app diff --git a/scripts/build-wheel.py b/scripts/build-wheel.py new file mode 100644 index 00000000..7df3860a --- /dev/null +++ b/scripts/build-wheel.py @@ -0,0 +1,215 @@ +from argparse import ArgumentParser +from io import BytesIO, StringIO +import json +from pathlib import Path +import tarfile +from urllib.error import HTTPError +from urllib.request import urlopen +from wheel.wheelfile import WheelFile +from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED + +METADATA = { + "Summary": "Austin - Frame Stack Sampler for CPython", + "Author": "Gabriele N. Tornetta", + "License": "GPLv3+", + "Classifier": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + ], + "Project-URL": [ + "Homepage, https://github.com/P403n1x87/austin", + "Source Code, https://github.com/P403n1x87/austin", + "Bug Tracker, https://github.com/P403n1x87/austin/issues", + "Changelog, https://github.com/P403n1x87/austin/blob/master/ChangeLog", + "Documentation, https://github.com/P403n1x87/austin/blob/master/README.md", + "Funding, https://github.com/sponsors/P403n1x87", + "Release Notes, https://github.com/P403n1x87/austin/releases", + ], + "Description-Content-Type": "text/markdown", +} + + +AUSTIN_WHEELS = { + "manylinux_2_17_aarch64.manylinux2014_aarch64": ( + "gnu-linux-aarch64.tar.xz", + ("austin", "austinp"), + ), + "manylinux_2_12_x86_64.manylinux2010_x86_64": ( + "gnu-linux-amd64.tar.xz", + ("austin", "austinp"), + ), + "manylinux_2_17_armv7l.manylinux2014_armv7l": ( + "gnu-linux-armv7.tar.xz", + ("austin", "austinp"), + ), + "manylinux_2_17_ppc64le.manylinux2014_ppc64le": ( + "gnu-linux-ppc64le.tar.xz", + ("austin", "austinp"), + ), + "macosx_11_0_x86_64": ("mac64.zip", ("austin",)), + "macosx_11_0_arm64": ("mac-arm64.zip", ("austin",)), + "musllinux_1_1_aarch64": ("musl-linux-aarch64.tar.xz", ("austin",)), + "musllinux_1_1_x86_64": ("musl-linux-amd64.tar.xz", ("austin",)), + "musllinux_1_1_armv7l": ("musl-linux-armv7.tar.xz", ("austin",)), + "musllinux_1_1_ppc64le": ("musl-linux-ppc64le.tar.xz", ("austin",)), + "win_amd64": ("win64.zip", ("austin",)), +} + + +def make_message(headers, payload=None): + message = StringIO() + + for name, value in headers.items(): + if isinstance(value, list): + for value_part in value: + print(f"{name}: {value_part}", file=message) + else: + print(f"{name}: {value}", file=message) + + if payload: + print(file=message) + print(payload, file=message) + + return message.getvalue().encode("utf-8") + + +def write_austin_wheel(out_dir, *, version, platform, austin_bin_data): + package_name = "austin-dist" + python = ".".join(("py2", "py3")) + dist_name = package_name.replace("-", "_") + wheel_name = f"{dist_name}-{version}-{python}-none-{platform}.whl" + dist_info = f"{dist_name}-{version}.dist-info" + wheel_path = out_dir / wheel_name + + print(f"Building {wheel_name} ... ", end="", flush=True) + + contents = {} + + for binary_name, binary_data in austin_bin_data: + zip_info = ZipInfo(f"{dist_name}-{version}.data/scripts/{binary_name}") + zip_info.external_attr |= 33261 << 16 + contents[zip_info] = binary_data + + contents[f"{dist_info}/METADATA"] = make_message( + { + "Metadata-Version": "2.1", + "Name": package_name, + "Version": version, + **METADATA, + }, + Path("README.md").read_text("utf-8"), + ) + contents[f"{dist_info}/WHEEL"] = make_message( + { + "Wheel-Version": "1.0", + "Generator": "austin-dist build-wheel.py", + "Root-Is-Purelib": "false", + "Tag": f"{python}-none-{platform}", + } + ) + + with WheelFile(str(wheel_path), "w") as wheel: + for member_info, member_source in contents.items(): + if not isinstance(member_info, ZipInfo): + member_info = ZipInfo(member_info) + member_info.external_attr = 0o644 << 16 + member_info.file_size = len(member_source) + member_info.compress_type = ZIP_DEFLATED + wheel.writestr(member_info, bytes(member_source)) + + assert wheel_path.exists(), "wheel file created" + + print("done") + + +def get_latest_release() -> str: + with urlopen( + "https://api.github.com/repos/p403n1x87/austin/releases/latest" + ) as stream: + return json.loads(stream.read().decode("utf-8"))["tag_name"].strip("v") + + +def download_release( + version: str, suffix: str, variant_name: str = "austin" +) -> tuple[str, bytes]: + prefix = "https://github.com/p403n1x87/austin/releases/download/" + try: + with urlopen(f"{prefix}v{version}/{variant_name}-{version}-{suffix}") as stream: + buffer = BytesIO(stream.read()) + if suffix.endswith(".tar.xz"): + with tarfile.open(fileobj=buffer, mode="r:xz") as tar: + return variant_name, tar.extractfile(variant_name).read() + elif suffix.endswith(".zip"): + with ZipFile(buffer) as zip: + try: + return variant_name, zip.read(variant_name) + except KeyError: + file_name = f"{variant_name}.exe" + return file_name, zip.read(file_name) + raise ValueError(f"Unknown archive extension: {suffix}") + except HTTPError: + raise RuntimeError(f"Could not download Austin version {version}") + + +if __name__ == "__main__": + argp = ArgumentParser() + + argp.add_argument( + "--version", + help="Austin version to build wheels for", + default=None, + ) + + argp.add_argument( + "--platform", + help="Platform to build wheels for", + choices=AUSTIN_WHEELS.keys(), + default=None, + ) + + argp.add_argument( + "--files", + help="The variant to binary file mapping (e.g. austin:src/austin)", + nargs="+", + type=str, + default=None, + ) + + args = argp.parse_args() + + version = args.version or get_latest_release() + + dist_dir = Path.cwd() / "dist" + dist_dir.mkdir(exist_ok=True) + + for platform, (suffix, variants) in AUSTIN_WHEELS.items(): + if args.platform is not None and args.platform != platform: + continue + + bin_data = ( + ( + download_release(version, suffix, variant_name=variant) + for variant in variants + ) + if args.files is None + else ( + (file_name, Path(bin_path).read_bytes()) + for file_name, bin_path in (file.split(":") for file in args.files) + ) + ) + + write_austin_wheel( + dist_dir, + version=version, + platform=platform, + austin_bin_data=bin_data, + ) diff --git a/scripts/build_arch.sh b/scripts/build_arch.sh index 92662475..070048a2 100644 --- a/scripts/build_arch.sh +++ b/scripts/build_arch.sh @@ -25,9 +25,14 @@ pushd src tar -Jcf austin-$VERSION-gnu-linux-$ARCH.tar.xz austin tar -Jcf austinp-$VERSION-gnu-linux-$ARCH.tar.xz austinp + cp austin /artifacts/austin + cp austinp /artifacts/austinp + musl-gcc -O3 -Os -s -Wall -pthread *.c -o austin -D__MUSL__ tar -Jcf austin-$VERSION-musl-linux-$ARCH.tar.xz austin + cp austin /artifacts/austin.musl + mv austin-$VERSION-gnu-linux-$ARCH.tar.xz /artifacts mv austinp-$VERSION-gnu-linux-$ARCH.tar.xz /artifacts mv austin-$VERSION-musl-linux-$ARCH.tar.xz /artifacts diff --git a/scripts/requirements-bw.txt b/scripts/requirements-bw.txt new file mode 100644 index 00000000..6c4932c1 --- /dev/null +++ b/scripts/requirements-bw.txt @@ -0,0 +1,2 @@ +wheel +twine