From e28df59f8163bd300883f712417ee68396054814 Mon Sep 17 00:00:00 2001 From: pasqu4le Date: Thu, 7 Sep 2023 22:09:14 +0200 Subject: [PATCH] tmp-release-test --- .buildkite/pipeline-raw.yml | 228 +++++++--------------------------- scripts/autorelease.sh | 13 +- scripts/build-all-bottles.sh | 14 +-- scripts/check-bottle-built.sh | 2 +- scripts/del-test-release.sh | 8 ++ 5 files changed, 64 insertions(+), 201 deletions(-) create mode 100755 scripts/del-test-release.sh diff --git a/.buildkite/pipeline-raw.yml b/.buildkite/pipeline-raw.yml index 44cbe821d..6a7354143 100644 --- a/.buildkite/pipeline-raw.yml +++ b/.buildkite/pipeline-raw.yml @@ -7,50 +7,10 @@ env: USE_NEWER_NIX: 1 steps: - # We need to sign commits that update brew formulae separately - - label: Sign formulae update commits - if: build.branch =~ /^auto\/update-brew-formulae-.*/ - commands: - - nix develop .#buildkite - --command './scripts/sign-commits.sh' - # To avoid race-conditions for the gpg key between jobs which sometimes leads to weird errors - - wait - - - label: reuse lint - command: nix shell .#reuse -c reuse lint - - label: check trailing whitespace - command: .buildkite/check-trailing-whitespace.sh - - label: crossref-verify - command: nix shell -f https://github.com/serokell/crossref-verifier/archive/68a1f9d25b6e7835fea8299b18a3e6c61dbb2a5c.tar.gz -c crossref-verify - soft_fail: true - - label: lint python code - command: nix shell .#python39Packages.black -c black --check --diff --color . - - label: lint bash scripts - command: nix shell .#shellcheck -c shellcheck --shell=bash --exclude=SC1091 -x $(find . -name '*.sh') - - label: pipeline-filtering - command: nix develop .#buildkite --command 'nix shell .#bats -c ./tests/buildkite/filter-pipeline.bats' - only_changes: - - .buildkite/filter-pipeline.py - - tests/buildkite/.* - - label: check auto-inserting bottle hashes - commands: - - cd tests/bottle-hashes/ - - ./test-hash-insert.sh - - - label: build via nix - commands: - # NOTE: if this changes, remember to update the instructions in the nix/README.md too - - nix build . -o binaries - branches: "!master" - - label: check flake - commands: - - nix flake check -L - branches: "!master" - # nix builds are usually cached, so we don't care about only_changes setting for it - - label: build-via-docker # this step is used as a dependency, so we're defining 'key' explicitely key: build-via-docker + if: build.tag == "test-695" commands: - eval "$SET_VERSION" - cd docker @@ -59,15 +19,11 @@ steps: - ./docker/octez-* agents: queue: "docker" - only_changes: &static_binaries_changes_regexes - - docker/build/.* - - docker/docker-static-build.sh - - meta.json - - protocols.json - label: build-arm-via-docker # this step is used as a dependency, so we're defining 'key' explicitely key: build-arm-via-docker + if: build.tag == "test-695" commands: - eval "$SET_VERSION" - cd docker @@ -80,156 +36,68 @@ steps: - ./docker/octez-* agents: queue: "arm64-darwin" - only_changes: *static_binaries_changes_regexes - - label: test docker-built binaries + - label: create auto release/pre-release + key: auto-release + if: build.tag == "test-695" commands: - - buildkite-agent artifact download "docker/*" . --step "build-via-docker" - - chmod +x ./docker/* - - nix-build . -A binaries-test --no-out-link --arg path-to-binaries ./docker - branches: "!master" + - mkdir binaries + - mkdir arm-binaries + - buildkite-agent artifact download "docker/*" binaries --step "build-via-docker" + - buildkite-agent artifact download "docker/*" arm-binaries --step "build-arm-via-docker" + - ls binaries + - nix develop .#autorelease -c ./scripts/autorelease.sh "$BUILDKITE_MESSAGE" depends_on: - "build-via-docker" - only_changes: - - tests/tezos-binaries.nix - - tests/test_script.py - # files from nix/ are massively used in tests infrastructure - - nix/.* + - "build-arm-via-docker" - - label: test deb source packages via docker - commands: - - eval "$SET_VERSION" - - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type source - artifact_paths: - - ./out/* - branches: "!master" - timeout_in_minutes: 60 - agents: - queue: "docker" - only_changes: &native_packaging_changes_regexes - - docker/package/.* - - docker/build/ubuntu/build.py - - docker/build/util/build.py - - meta.json - - protocols.json - - label: test deb binary packages via docker - commands: - - eval "$SET_VERSION" - # Building all binary packages will take significant amount of time, so we build only one - # in order to ensure package generation sanity - - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type binary -p tezos-baker-PtMumbai - - rm -rf out - # It takes much time to build binary package, so we do it only on master - branches: "master" - timeout_in_minutes: 240 + - label: Build Big Sur x86_64 bottles + key: build-bottles-big-sur-x86_64 + if: build.tag == "test-695" + depends_on: + - "auto-release" agents: - queue: "docker" - only_changes: *native_packaging_changes_regexes - - label: test rpm source packages via docker + queue: "x86_64-rosetta-darwin" commands: - - eval "$SET_VERSION" - - nix develop .#docker-tezos-packages -c ./docker/build/fedora/build.py --type source + - nix develop .#autorelease-macos -c ./scripts/build-all-bottles.sh "big_sur" artifact_paths: - - ./out/* - branches: "!master" - timeout_in_minutes: 60 - agents: - queue: "docker" - only_changes: *native_packaging_changes_regexes - - label: test rpm binary packages via docker - commands: - - eval "$SET_VERSION" - # Building all binary packages will take significant amount of time, so we build only one - # in order to ensure package generation sanity - - nix develop .#docker-tezos-packages -c ./docker/build/fedora/build.py --type binary -p tezos-baker-PtMumbai - - rm -rf out - # It takes much time to build binary package, so we do it only on master - branches: "master" - timeout_in_minutes: 180 - agents: - queue: "docker" - only_changes: *native_packaging_changes_regexes + - '*.bottle.*' + retry: + automatic: + limit: 1 - - label: build deb packages with static binaries - key: build-static-deb + - label: Build Big Sur arm64 bottles + key: build-bottles-big-sur-arm64 + if: build.tag == "test-695" depends_on: - - "build-via-docker" + - "auto-release" agents: - queue: "docker" + queue: "arm64-darwin" commands: - - eval "$SET_VERSION" - - buildkite-agent artifact download "docker/octez-*" . --step build-via-docker - - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --os ubuntu --type binary --binaries-dir docker --distributions focal + - nix develop .#autorelease-macos -c ./scripts/build-all-bottles.sh "arm64_big_sur" artifact_paths: - - ./out/* - only_changes: *native_packaging_changes_regexes + - '*.bottle.*' + retry: + automatic: + limit: 1 - - label: test gen_systemd_service_file.py script - commands: - - eval "$SET_VERSION" - - nix develop .#autorelease -c ./gen_systemd_service_file.py tezos-node - branches: "!master" - agents: - queue: "docker" - only_changes: - - gen_systemd_service_file.py - - docker/package/.* - - - label: test bundled systemd services - # Used in .buildkite/hooks/pre-exit - key: test-systemd-services + - label: Add Big Sur bottle hashes to formulae + key: list-bottles depends_on: - - "build-static-deb" - only_changes: *native_packaging_changes_regexes - agents: - queue: "default" + - "build-bottles-big-sur-arm64" + - "build-bottles-big-sur-x86_64" + if: build.tag == "test-695" + soft_fail: true # No artifacts to download if all the bottles are already built commands: - - buildkite-agent artifact download "out/*~focal_amd64.deb" . --step build-static-deb - - cd tests/systemd - - nix shell ../..#legacyPackages.x86_64-linux.vagrant ../..#legacyPackages.x86_64-linux.curl -c vagrant --packages-directory=../../out up --provider=libvirt + - mkdir -p "Big Sur" + - gh release download build.tag -D "Big Sur/" -p "*.bottle.tar.gz" + - ls "Big Sur" - - label: create auto release/pre-release - key: auto-release + - label: delete test-release pre-release commands: - - mkdir binaries - - mkdir arm-binaries - - buildkite-agent artifact download "docker/*" binaries --step "build-via-docker" - - buildkite-agent artifact download "docker/*" arm-binaries --step "build-arm-via-docker" - - ls binaries - - nix develop .#autorelease -c ./scripts/autorelease.sh "$BUILDKITE_MESSAGE" - branches: master - depends_on: - - "build-via-docker" - - "build-arm-via-docker" - only_changes: - - scripts/autorelease.sh - - scripts/shell.nix - # files from 'nix/' directory are used in the autorelease script - - nix/.* - - tezos-release.nix - - release.nix - - # Update mirror repositories on master if brew formulae have been updated or bottle hashes added - - label: update stable mirror repository - if: | - build.branch == "master" && - ( build.message =~ /^Merge pull request .* from serokell\/auto\/v[0-9]+\.[0-9]+-release/ || - build.message =~ /^Merge pull request .* from serokell\/auto\/update-brew-formulae-v[.0-9]+-[0-9]+/ ) - depends_on: - - "auto-release" - env: - MIRROR_REPO: "git@github.com:serokell/tezos-packaging-stable.git" - commands: &update_mirror - - git pull origin "$BUILDKITE_BRANCH:$BUILDKITE_BRANCH" - - git push --mirror "$$MIRROR_REPO" - - - label: update RC mirror repository - if: | - build.branch == "master" && - (build.message =~ /^Merge pull request .* from serokell\/auto\/v[0-9]+\.[0-9]+-(rc|beta).*-release/ || - build.message =~ /^Merge pull request .* from serokell\/auto\/update-brew-formulae-v[.0-9]+-(rc|beta).*/) + - nix develop .#autorelease -c ./scripts/del-test-release.sh "$BUILDKITE_MESSAGE" + if: build.tag == "test-695" depends_on: - - "auto-release" - env: - MIRROR_REPO: "git@github.com:serokell/tezos-packaging-rc.git" - commands: *update_mirror + - "build-bottles-big-sur-x86_64" + - "build-bottles-big-sur-arm64" + - "list-bottles" diff --git a/scripts/autorelease.sh b/scripts/autorelease.sh index 921fbd150..e5628c31c 100755 --- a/scripts/autorelease.sh +++ b/scripts/autorelease.sh @@ -38,18 +38,7 @@ rc_regex="^v[0-9]+\.[0-9]+-(rc|beta)[0-9]+" # We create a pre-release in any case except if we are merging a stable version release PR mode_flag="--prerelease" - -if [[ ${1-""} =~ $msg_regex ]]; then - tag="${BASH_REMATCH[1]}-$(jq -r '.release' meta.json)" - if [[ ! $tag =~ $rc_regex ]]; then - mode_flag="" - fi -else - tag="auto-release" - - # Delete autorelease if it exists - gh release delete auto-release --yes || true -fi +tag="test-release" # Update the tag git fetch # So that the script can be run from an arbitrary checkout diff --git a/scripts/build-all-bottles.sh b/scripts/build-all-bottles.sh index 746dcdc69..cb7a4bfc5 100755 --- a/scripts/build-all-bottles.sh +++ b/scripts/build-all-bottles.sh @@ -19,7 +19,7 @@ set -euo pipefail retval="0" # we don't bottle meta-formulas that contain only services -formulae=("tezos-accuser-PtMumbai" "tezos-accuser-PtNairob" "tezos-admin-client" "tezos-baker-PtMumbai" "tezos-baker-PtNairob" "tezos-client" "tezos-codec" "tezos-node" "tezos-signer" "tezos-smart-rollup-client-PtMumbai" "tezos-smart-rollup-client-PtNairob" "tezos-smart-rollup-node-PtMumbai" "tezos-smart-rollup-node-PtNairob") +formulae=("tezos-client") # tezos-sapling-params is used as a dependency for some of the formulas # so we handle it separately. @@ -31,13 +31,11 @@ for f in "${formulae[@]}"; do if ./scripts/check-bottle-built.sh "$f" "$1"; then # build a bottle if ./scripts/build-one-bottle.sh "$f"; then - # upload the bottle to its respective release - FORMULA_TAG="$(sed -n "s/^\s\+version \"\(.*\)\"/\1/p" "./Formula/$f.rb")" - if ! gh release upload "$FORMULA_TAG" "$f"*.bottle.*; then - # we want a non-0 exit code if any of the bottles couldn't be uploaded - retval="1"; - >&2 echo "Bottle for $f couldn't be uploaded to $FORMULA_TAG release." - fi + # upload the bottle to the test release + gh release upload "test-release" "$f"*.bottle.* + # we want a non-0 exit code if the bottle was uploaded, to simulate a failure + retval="1"; + >&2 echo "Bottle for $f was now uploaded to test-release." else # we want a non-0 exit code if any of the bottles couldn't be built retval="1"; diff --git a/scripts/check-bottle-built.sh b/scripts/check-bottle-built.sh index 151a1bf14..fb44f9e32 100755 --- a/scripts/check-bottle-built.sh +++ b/scripts/check-bottle-built.sh @@ -17,7 +17,7 @@ if [ ! -f "./Formula/$1.rb" ] ; then exit 2 fi -tag=$(sed -n "s/^\s\+version \"\(.*\)\"/\1/p" "./Formula/$1.rb") +tag="test-release" set +e diff --git a/scripts/del-test-release.sh b/scripts/del-test-release.sh new file mode 100755 index 000000000..11fe55e37 --- /dev/null +++ b/scripts/del-test-release.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# SPDX-FileCopyrightText: 2023 Oxhead Alpha +# SPDX-License-Identifier: LicenseRef-MIT-OA + +set -euo pipefail + +gh release delete test-release --yes || true