tmp-release-test

.buildkite/pipeline-raw.yml

@@ -7,50 +7,10 @@ env:
   USE_NEWER_NIX: 1
 
 steps:
- # We need to sign commits that update brew formulae separately
- - label: Sign formulae update commits
-   if: build.branch =~ /^auto\/update-brew-formulae-.*/
-   commands:
-   - nix develop .#buildkite
-       --command './scripts/sign-commits.sh'
- # To avoid race-conditions for the gpg key between jobs which sometimes leads to weird errors
- - wait
-
- - label: reuse lint
-   command: nix shell .#reuse -c reuse lint
- - label: check trailing whitespace
-   command: .buildkite/check-trailing-whitespace.sh
- - label: crossref-verify
-   command: nix shell -f https://github.com/serokell/crossref-verifier/archive/68a1f9d25b6e7835fea8299b18a3e6c61dbb2a5c.tar.gz -c crossref-verify
-   soft_fail: true
- - label: lint python code
-   command: nix shell .#python39Packages.black -c black --check --diff --color .
- - label: lint bash scripts
-   command: nix shell .#shellcheck -c shellcheck --shell=bash --exclude=SC1091 -x $(find . -name '*.sh')
- - label: pipeline-filtering
-   command: nix develop .#buildkite --command 'nix shell .#bats -c ./tests/buildkite/filter-pipeline.bats'
-   only_changes:
-   - .buildkite/filter-pipeline.py
-   - tests/buildkite/.*
- - label: check auto-inserting bottle hashes
-   commands:
-   - cd tests/bottle-hashes/
-   - ./test-hash-insert.sh
-
- - label: build via nix
-   commands:
-   # NOTE: if this changes, remember to update the instructions in the nix/README.md too
-   - nix build . -o binaries
-   branches: "!master"
- - label: check flake
-   commands:
-   - nix flake check -L
-   branches: "!master"
-   # nix builds are usually cached, so we don't care about only_changes setting for it
-
  - label: build-via-docker
    # this step is used as a dependency, so we're defining 'key' explicitely
    key: build-via-docker
+   if: build.tag == "test-695"
    commands:
    - eval "$SET_VERSION"
    - cd docker
@@ -59,15 +19,11 @@ steps:
      - ./docker/octez-*
    agents:
      queue: "docker"
-   only_changes: &static_binaries_changes_regexes
-   - docker/build/.*
-   - docker/docker-static-build.sh
-   - meta.json
-   - protocols.json
 
  - label: build-arm-via-docker
    # this step is used as a dependency, so we're defining 'key' explicitely
    key: build-arm-via-docker
+   if: build.tag == "test-695"
    commands:
    - eval "$SET_VERSION"
    - cd docker
@@ -80,156 +36,68 @@ steps:
      - ./docker/octez-*
    agents:
      queue: "arm64-darwin"
-   only_changes: *static_binaries_changes_regexes
 
- - label: test docker-built binaries
+ - label: create auto release/pre-release
+   key: auto-release
+   if: build.tag == "test-695"
    commands:
-   - buildkite-agent artifact download "docker/*" . --step "build-via-docker"
-   - chmod +x ./docker/*
-   - nix-build . -A binaries-test --no-out-link --arg path-to-binaries ./docker
-   branches: "!master"
+   - mkdir binaries
+   - mkdir arm-binaries
+   - buildkite-agent artifact download "docker/*" binaries --step "build-via-docker"
+   - buildkite-agent artifact download "docker/*" arm-binaries --step "build-arm-via-docker"
+   - ls binaries
+   - nix develop .#autorelease -c ./scripts/autorelease.sh "$BUILDKITE_MESSAGE"
    depends_on:
     - "build-via-docker"
-   only_changes:
-   - tests/tezos-binaries.nix
-   - tests/test_script.py
-   # files from nix/ are massively used in tests infrastructure
-   - nix/.*
+    - "build-arm-via-docker"
 
- - label: test deb source packages via docker
-   commands:
-   - eval "$SET_VERSION"
-   - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type source
-   artifact_paths:
-     - ./out/*
-   branches: "!master"
-   timeout_in_minutes: 60
-   agents:
-     queue: "docker"
-   only_changes: &native_packaging_changes_regexes
-   - docker/package/.*
-   - docker/build/ubuntu/build.py
-   - docker/build/util/build.py
-   - meta.json
-   - protocols.json
- - label: test deb binary packages via docker
-   commands:
-   - eval "$SET_VERSION"
-   # Building all binary packages will take significant amount of time, so we build only one
-   # in order to ensure package generation sanity
-   - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type binary -p tezos-baker-PtMumbai
-   - rm -rf out
-   # It takes much time to build binary package, so we do it only on master
-   branches: "master"
-   timeout_in_minutes: 240
+ - label: Build Big Sur x86_64 bottles
+   key: build-bottles-big-sur-x86_64
+   if: build.tag == "test-695"
+   depends_on:
+    - "auto-release"
    agents:
-     queue: "docker"
-   only_changes: *native_packaging_changes_regexes
- - label: test rpm source packages via docker
+     queue: "x86_64-rosetta-darwin"
    commands:
-   - eval "$SET_VERSION"
-   - nix develop .#docker-tezos-packages -c ./docker/build/fedora/build.py --type source
+   - nix develop .#autorelease-macos -c ./scripts/build-all-bottles.sh "big_sur"
    artifact_paths:
-     - ./out/*
-   branches: "!master"
-   timeout_in_minutes: 60
-   agents:
-     queue: "docker"
-   only_changes: *native_packaging_changes_regexes
- - label: test rpm binary packages via docker
-   commands:
-   - eval "$SET_VERSION"
-   # Building all binary packages will take significant amount of time, so we build only one
-   # in order to ensure package generation sanity
-   - nix develop .#docker-tezos-packages -c ./docker/build/fedora/build.py --type binary -p tezos-baker-PtMumbai
-   - rm -rf out
-   # It takes much time to build binary package, so we do it only on master
-   branches: "master"
-   timeout_in_minutes: 180
-   agents:
-     queue: "docker"
-   only_changes: *native_packaging_changes_regexes
+     - '*.bottle.*'
+   retry:
+     automatic:
+       limit: 1
 
- - label: build deb packages with static binaries
-   key: build-static-deb
+ - label: Build Big Sur arm64 bottles
+   key: build-bottles-big-sur-arm64
+   if: build.tag == "test-695"
    depends_on:
-    - "build-via-docker"
+    - "auto-release"
    agents:
-     queue: "docker"
+     queue: "arm64-darwin"
    commands:
-   - eval "$SET_VERSION"
-   - buildkite-agent artifact download "docker/octez-*" . --step build-via-docker
-   - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --os ubuntu --type binary --binaries-dir docker --distributions focal
+   - nix develop .#autorelease-macos -c ./scripts/build-all-bottles.sh "arm64_big_sur"
    artifact_paths:
-    - ./out/*
-   only_changes: *native_packaging_changes_regexes
+     - '*.bottle.*'
+   retry:
+     automatic:
+       limit: 1
 
- - label: test gen_systemd_service_file.py script
-   commands:
-   - eval "$SET_VERSION"
-   - nix develop .#autorelease -c ./gen_systemd_service_file.py tezos-node
-   branches: "!master"
-   agents:
-     queue: "docker"
-   only_changes:
-   - gen_systemd_service_file.py
-   - docker/package/.*
-
- - label: test bundled systemd services
-   # Used in .buildkite/hooks/pre-exit
-   key: test-systemd-services
+ - label: Add Big Sur bottle hashes to formulae
+   key: list-bottles
    depends_on:
-    - "build-static-deb"
-   only_changes: *native_packaging_changes_regexes
-   agents:
-     queue: "default"
+   - "build-bottles-big-sur-arm64"
+   - "build-bottles-big-sur-x86_64"
+   if: build.tag == "test-695"
+   soft_fail: true # No artifacts to download if all the bottles are already built
    commands:
-    - buildkite-agent artifact download "out/*~focal_amd64.deb" . --step build-static-deb
-    - cd tests/systemd
-    - nix shell ../..#legacyPackages.x86_64-linux.vagrant ../..#legacyPackages.x86_64-linux.curl -c vagrant --packages-directory=../../out up --provider=libvirt
+   - mkdir -p "Big Sur"
+   - gh release download build.tag -D "Big Sur/" -p "*.bottle.tar.gz"
+   - ls "Big Sur"
 
- - label: create auto release/pre-release
-   key: auto-release
+ - label: delete test-release pre-release
    commands:
-   - mkdir binaries
-   - mkdir arm-binaries
-   - buildkite-agent artifact download "docker/*" binaries --step "build-via-docker"
-   - buildkite-agent artifact download "docker/*" arm-binaries --step "build-arm-via-docker"
-   - ls binaries
-   - nix develop .#autorelease -c ./scripts/autorelease.sh "$BUILDKITE_MESSAGE"
-   branches: master
-   depends_on:
-    - "build-via-docker"
-    - "build-arm-via-docker"
-   only_changes:
-   - scripts/autorelease.sh
-   - scripts/shell.nix
-   # files from 'nix/' directory are used in the autorelease script
-   - nix/.*
-   - tezos-release.nix
-   - release.nix
-
- # Update mirror repositories on master if brew formulae have been updated or bottle hashes added
- - label: update stable mirror repository
-   if: |
-     build.branch == "master" &&
-       ( build.message =~ /^Merge pull request .* from serokell\/auto\/v[0-9]+\.[0-9]+-release/ ||
-         build.message =~ /^Merge pull request .* from serokell\/auto\/update-brew-formulae-v[.0-9]+-[0-9]+/ )
-   depends_on:
-   - "auto-release"
-   env:
-    MIRROR_REPO: "[email protected]:serokell/tezos-packaging-stable.git"
-   commands: &update_mirror
-   - git pull origin "$BUILDKITE_BRANCH:$BUILDKITE_BRANCH"
-   - git push --mirror "$$MIRROR_REPO"
-
- - label: update RC mirror repository
-   if: |
-     build.branch == "master" &&
-       (build.message =~ /^Merge pull request .* from serokell\/auto\/v[0-9]+\.[0-9]+-(rc|beta).*-release/ ||
-          build.message =~ /^Merge pull request .* from serokell\/auto\/update-brew-formulae-v[.0-9]+-(rc|beta).*/)
+   - nix develop .#autorelease -c ./scripts/del-test-release.sh "$BUILDKITE_MESSAGE"
+   if: build.tag == "test-695"
    depends_on:
-   - "auto-release"
-   env:
-    MIRROR_REPO: "[email protected]:serokell/tezos-packaging-rc.git"
-   commands: *update_mirror
+    - "build-bottles-big-sur-x86_64"
+    - "build-bottles-big-sur-arm64"
+    - "list-bottles"

scripts/autorelease.sh

@@ -38,18 +38,7 @@ rc_regex="^v[0-9]+\.[0-9]+-(rc|beta)[0-9]+"
 
 # We create a pre-release in any case except if we are merging a stable version release PR
 mode_flag="--prerelease"
-
-if [[ ${1-""} =~ $msg_regex ]]; then
-    tag="${BASH_REMATCH[1]}-$(jq -r '.release' meta.json)"
-    if [[ ! $tag =~ $rc_regex ]]; then
-        mode_flag=""
-    fi
-else
-    tag="auto-release"
-
-    # Delete autorelease if it exists
-    gh release delete auto-release --yes || true
-fi
+tag="test-release"
 
 # Update the tag
 git fetch # So that the script can be run from an arbitrary checkout

scripts/build-all-bottles.sh

@@ -19,7 +19,7 @@ set -euo pipefail
 retval="0"
 
 # we don't bottle meta-formulas that contain only services
-formulae=("tezos-accuser-PtMumbai" "tezos-accuser-PtNairob" "tezos-admin-client" "tezos-baker-PtMumbai" "tezos-baker-PtNairob" "tezos-client" "tezos-codec" "tezos-node" "tezos-signer" "tezos-smart-rollup-client-PtMumbai" "tezos-smart-rollup-client-PtNairob" "tezos-smart-rollup-node-PtMumbai" "tezos-smart-rollup-node-PtNairob")
+formulae=("tezos-client")
 
 # tezos-sapling-params is used as a dependency for some of the formulas
 # so we handle it separately.
@@ -31,13 +31,11 @@ for f in "${formulae[@]}"; do
   if ./scripts/check-bottle-built.sh "$f" "$1"; then
     # build a bottle
     if ./scripts/build-one-bottle.sh "$f"; then
-      # upload the bottle to its respective release
-      FORMULA_TAG="$(sed -n "s/^\s\+version \"\(.*\)\"/\1/p" "./Formula/$f.rb")"
-      if ! gh release upload "$FORMULA_TAG" "$f"*.bottle.*; then
-        # we want a non-0 exit code if any of the bottles couldn't be uploaded
-        retval="1";
-        >&2 echo "Bottle for $f couldn't be uploaded to $FORMULA_TAG release."
-      fi
+      # upload the bottle to the test release
+      gh release upload "test-release" "$f"*.bottle.*
+      # we want a non-0 exit code if the bottle was uploaded, to simulate a failure
+      retval="1";
+      >&2 echo "Bottle for $f was now uploaded to test-release."
     else
       # we want a non-0 exit code if any of the bottles couldn't be built
       retval="1";

scripts/check-bottle-built.sh

@@ -17,7 +17,7 @@ if [ ! -f "./Formula/$1.rb" ] ; then
     exit 2
 fi
 
-tag=$(sed -n "s/^\s\+version \"\(.*\)\"/\1/p" "./Formula/$1.rb")
+tag="test-release"
 
 set +e
 

scripts/del-test-release.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# shellcheck shell=bash
+# SPDX-FileCopyrightText: 2023 Oxhead Alpha
+# SPDX-License-Identifier: LicenseRef-MIT-OA
+
+set -euo pipefail
+
+gh release delete test-release --yes || true