Skip to content

Commit

Permalink
Merge branch 'main' into gh-oblt/add-asciidoc-freeze-warning
Browse files Browse the repository at this point in the history
  • Loading branch information
bmorelli25 authored Jan 30, 2025
2 parents 59902d2 + c5d105f commit d5e3778
Show file tree
Hide file tree
Showing 257 changed files with 10,858 additions and 17,552 deletions.
21 changes: 2 additions & 19 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,8 @@

set -eo pipefail

retry() {
local retries=$1
shift

local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ $count -lt "$retries" ]; then
>&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
sleep $wait
else
>&2 echo "Retry $count/$retries exited $exit, no more retries left."
return $exit
fi
done
return 0
}
# shellcheck disable=SC1091
source .buildkite/scripts/utils.sh

get_os_details() {
case $(uname | tr '[:upper:]' '[:lower:]') in
Expand Down
35 changes: 20 additions & 15 deletions .buildkite/package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,52 +5,57 @@ notify:
if: "build.state != 'passed'"

env:
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"
IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2404"
IMAGE_UBUNTU_ARM_64: "core-ubuntu-2404-aarch64"
GENERATE_WOLFI_IMAGES: true

steps:
- group: "Package"
key: "package"
steps:
- label: "Package Ubuntu-20 x86_64"
- label: "Package x86_64"
key: "package-x86-64"
command: ".buildkite/scripts/package.sh {{matrix.type}}"
command: ".buildkite/scripts/package.sh {{matrix}}"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
matrix:
setup:
type:
- "snapshot"
- "staging"
- "snapshot"
- "staging"
artifact_paths:
- "build/distributions/**/*"
- "build/dependencies*.csv"
retry:
automatic:
- limit: 1

- label: "Package Ubuntu-20 aarch64"
- label: "Package aarch64"
key: "package-arm"
command: ".buildkite/scripts/package.sh {{matrix.type}}"
command: ".buildkite/scripts/package.sh {{matrix}}"
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "t4g.2xlarge"
matrix:
setup:
type:
- "snapshot"
- "staging"
- "snapshot"
- "staging"
artifact_paths:
- "build/distributions/**/*"
retry:
automatic:
- limit: 1

- label: "DRA"
key: "dra"
command: ".buildkite/scripts/dra.sh"
command: ".buildkite/scripts/dra.sh {{matrix}}"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
matrix:
- "snapshot"
- "staging"
depends_on:
- step: "package"
allow_failure: false
82 changes: 33 additions & 49 deletions .buildkite/scripts/dra.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,12 @@

set -eo pipefail

## Read current version.
VERSION=$(make get-version)
# Either staging or snapshot
TYPE="$1"

# NOTE: load the shared functions
# shellcheck disable=SC1091
source .buildkite/scripts/utils.sh

echo "--- Restoring Artifacts"
buildkite-agent artifact download "build/**/*" .
Expand All @@ -31,48 +35,28 @@ fi
# by default it uses the buildkite branch
DRA_BRANCH="$BUILDKITE_BRANCH"
# by default it publishes the DRA artifacts, for such it uses the collect command.
dra_command=collect
DRA_COMMAND=collect
VERSION=$(make get-version-only)
BRANCHES_URL=https://storage.googleapis.com/artifacts-api/snapshots/branches.json
curl -s "${BRANCHES_URL}" > active-branches.json
# as long as `8.x` is not in the active branches, we will explicitly add the condition.
if [ "$BUILDKITE_BRANCH" == "8.x" ] || grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
echo "--- :arrow_right: Release Manager only supports the current active branches and 8.x, running"
else
if ! grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
# If no active branches are found, let's see if it is a feature branch.
echo "--- :arrow_right: Release Manager only supports the current active branches, skipping"
echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
echo "BUILDKITE_COMMIT=$BUILDKITE_COMMIT"
echo "VERSION=$VERSION"
echo "Supported branches:"
cat active-branches.json
if [[ $BUILDKITE_BRANCH =~ "feature/" ]]; then
buildkite-agent annotate "${BUILDKITE_BRANCH} will list DRA artifacts. Feature branches are not supported. Look for the supported branches in ${BRANCHES_URL}" --style 'info' --context 'ctx-info'
dra_command=list

# use a different branch since DRA does not support feature branches but main/release branches
# for such we will use the VERSION and https://storage.googleapis.com/artifacts-api/snapshots/<major.minor>.json
# to know if the branch was branched out from main or the release branches.
MAJOR_MINOR=${VERSION%.*}
if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/main.json" | grep -q "$VERSION" ; then
DRA_BRANCH=main
else
if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/$MAJOR_MINOR.json" | grep -q "$VERSION" ; then
DRA_BRANCH="$MAJOR_MINOR"
else
buildkite-agent annotate "It was not possible to know the original base branch for ${BUILDKITE_BRANCH}. This won't fail - this is a feature branch." --style 'info' --context 'ctx-info-feature-branch'
exit 0
fi
fi
else
buildkite-agent annotate "${BUILDKITE_BRANCH} is not supported yet. Look for the supported branches in ${BRANCHES_URL}" --style 'warning' --context 'ctx-warn'
exit 1
fi
dra_process_other_branches
fi

echo "--- :arrow_right: Release Manager only supports the current active branches"
echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
echo "BUILDKITE_COMMIT=$BUILDKITE_COMMIT"
echo "VERSION=$VERSION"
echo "Supported branches:"
cat active-branches.json

dra() {
local workflow=$1
local command=$2
local qualifier=${3:-""}
echo "--- Run release manager $workflow (DRA command: $command)"
set -x
docker run --rm \
--name release-manager \
-e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
Expand All @@ -86,23 +70,23 @@ dra() {
--commit $BUILDKITE_COMMIT \
--workflow $workflow \
--artifact-set main \
--qualifier "$qualifier" \
--version $VERSION | tee rm-output.txt
set +x

# Create Buildkite annotation similarly done in Beats:
# https://github.com/elastic/beats/blob/90f9e8f6e48e76a83331f64f6c8c633ae6b31661/.buildkite/scripts/dra.sh#L74-L81
if [[ "$command" == "collect" ]]; then
# extract the summary URL from a release manager output line like:
# Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/apm-server/18.22.0-ABCDEFGH/summary-18.22.0.html
SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}')
rm rm-output.txt
create_annotation_dra_summary "$command" "$workflow" rm-output.txt
}

# and make it easily clickable as a Builkite annotation
printf "**${workflow} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append
if [[ "${TYPE}" == "staging" ]]; then
qualifier=$(fetch_elastic_qualifier "$DRA_BRANCH")
# TODO: main and 8.x are not needed to run the DRA for staging
# but main is needed until we do alpha1 releases of 9.0.0
if [[ "${DRA_BRANCH}" != "8.x" ]]; then
dra "${TYPE}" "$DRA_COMMAND" "${qualifier}"
fi
}
fi

dra "snapshot" "$dra_command"
if [[ "${DRA_BRANCH}" != "main" && "${DRA_BRANCH}" != "8.x" ]]; then
echo "DRA_BRANCH is neither 'main' nor '8.x'"
dra "staging" "$dra_command"
if [[ "${TYPE}" == "snapshot" ]]; then
# NOTE: qualifier is not needed for snapshots, let's unset it.
dra "${TYPE}" "$DRA_COMMAND" ""
fi
11 changes: 11 additions & 0 deletions .buildkite/scripts/package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,17 @@ if [[ ${TYPE} == "snapshot" ]]; then
MAKE_GOAL="${MAKE_GOAL}-snapshot"
fi

if [[ ${TYPE} == "staging" ]]; then
echo "--- Prepare the Elastic Qualifier"
# NOTE: load the shared functions
# shellcheck disable=SC1091
source .buildkite/scripts/utils.sh
dra_process_other_branches
ELASTIC_QUALIFIER=$(fetch_elastic_qualifier "$DRA_BRANCH")
export ELASTIC_QUALIFIER
fi

echo "--- Run $MAKE_GOAL for $DRA_BRANCH"
make $MAKE_GOAL

ls -l build/distributions/
80 changes: 80 additions & 0 deletions .buildkite/scripts/utils.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# An opinionated approach to manage the Elatic Qualifier for the DRA in a Google Bucket
# Instead of using the ELASTIC_QUALIFIER env variable.
fetch_elastic_qualifier() {
local branch=$1
qualifier=""
URL="https://storage.googleapis.com/dra-qualifier/$branch"
if curl -sf -o /dev/null "$URL" ; then
qualifier=$(curl -s "$URL")
fi
echo "$qualifier"
}

retry() {
local retries=$1
shift

local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ $count -lt "$retries" ]; then
>&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
sleep $wait
else
>&2 echo "Retry $count/$retries exited $exit, no more retries left."
return $exit
fi
done
return 0
}

#
# An opinionated approach to detect if unsupported Unified Release branches
# can be used, this is handy for testing feature branches in dry-run mode
# It produces the below environment variables:
# - VERSION
# - DRA_COMMAND
# - DRA_BRANCH
dra_process_other_branches() {
## Read current version without the qualifier
VERSION=$(make get-version-only)
DRA_BRANCH="$BUILDKITE_BRANCH"
if [[ $BUILDKITE_BRANCH =~ "feature/" ]]; then
buildkite-agent annotate "${BUILDKITE_BRANCH} will list DRA artifacts. Feature branches are not supported. Look for the supported branches in ${BRANCHES_URL}" --style 'info' --context 'ctx-info'
DRA_COMMAND=list

# use a different branch since DRA does not support feature branches but main/release branches
# for such we will use the VERSION and https://storage.googleapis.com/artifacts-api/snapshots/<major.minor>.json
# to know if the branch was branched out from main or the release branches.
MAJOR_MINOR=${VERSION%.*}
if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/main.json" | grep -q "$VERSION" ; then
DRA_BRANCH=main
else
if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/$MAJOR_MINOR.json" | grep -q "$VERSION" ; then
DRA_BRANCH="$MAJOR_MINOR"
else
buildkite-agent annotate "It was not possible to know the original base branch for ${BUILDKITE_BRANCH}. This won't fail - this is a feature branch." --style 'info' --context 'ctx-info-feature-branch'
fi
fi
fi
export DRA_BRANCH DRA_COMMAND VERSION
}

# Create Buildkite annotation similarly done in Beats:
# https://github.com/elastic/beats/blob/90f9e8f6e48e76a83331f64f6c8c633ae6b31661/.buildkite/scripts/dra.sh#L74-L81
create_annotation_dra_summary() {
local command=$1
local workflow=$2
local output=$3
if [[ "$command" == "collect" ]]; then
# extract the summary URL from a release manager output line like:
# Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/apm-server/18.22.0-ABCDEFGH/summary-18.22.0.html
SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' "$output" | grep -oP 'https://\S+' | awk '{print $1}')
rm "$output"

# and make it easily clickable as a Builkite annotation
printf "**${workflow} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append
fi
}
41 changes: 33 additions & 8 deletions .ci/scripts/bench.sh
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,28 @@ function buildkite::download_last_artifact {
BUILDKITE_DOWNLOAD_URL="${download_url}" buildkite::req_download_artifact
}

#######################################
# Retry n number of times the given command
function retry() {
local retries=$1
shift

local count=0
until "$@"; do
exit=$?
wait=$((2 ** count))
count=$((count + 1))
if [ $count -lt "$retries" ]; then
>&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
sleep $wait
else
>&2 echo "Retry $count/$retries exited $exit, no more retries left."
return $exit
fi
done
return 0
}

## Buildkite specific configuration
if [ "${CI}" == "true" ] ; then
# If HOME is not set then use the Buildkite workspace
Expand All @@ -147,17 +169,20 @@ if [ "${CI}" == "true" ] ; then
[ -z "${REPO}" ] && echo "Environment variable 'REPO' must be defined" && exit 1;
export BUILDKITE_TOKEN="${BUILDKITE_TOKEN_SECRET}"

echo "--- Setting up the :golang: environment..."
GO_VERSION=$(cat .go-version)
OS_NAME=linux
OS_ARCH=amd64
retry 5 curl -sL -o "${BASE_PROJECT}"/gvm "https://github.com/andrewkroh/gvm/releases/download/v0.5.2/gvm-${OS_NAME}-${OS_ARCH}"
chmod +x "${BASE_PROJECT}"/gvm
retry 5 "${BASE_PROJECT}"/gvm install "$GO_VERSION"
eval "$("${BASE_PROJECT}"/gvm use "$GO_VERSION")"
echo "Golang version:"
go version
GOPATH=$(command go env GOPATH)
# Redirect go env to current dir
export GOROOT="${BASE_PROJECT}/.go"
export GOPATH="${BASE_PROJECT}/go"
export PATH="${GOPATH}/bin:${PATH}"

# Install g (go version manager)
curl -sSL https://git.io/g-install | bash -s -- -y

# Install specific version
g install $(cat .go-version)

# Make sure gomod can be deleted automatically as part of the CI
function teardown {
local arg=${?}
Expand Down
2 changes: 1 addition & 1 deletion .ci/scripts/download-png-from-kibana.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ JOB_PARAMS=$(echo "
" | tr -d "[:space:]")


png_url_path=$(curl -XPOST -v -L -u "$kibana_user:$kibana_pwd" -H 'kbn-xsrf: true' --data-urlencode "jobParams=${JOB_PARAMS}" $kibana_host/api/reporting/generate/pngV2 | jq -r '.path')
png_url_path=$(curl -L -u "$kibana_user:$kibana_pwd" -H 'kbn-xsrf: true' --data-urlencode "jobParams=${JOB_PARAMS}" $kibana_host/api/reporting/generate/pngV2 | jq -r '.path')

if [[ "$png_url_path" == "null" ]]
then
Expand Down
1 change: 0 additions & 1 deletion .ci/updatecli/bump-golang.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ targets:
files:
- go.mod
- cmd/intake-receiver/go.mod
- internal/glog/go.mod
- systemtest/go.mod
- tools/go.mod
matchpattern: 'go \d+.\d+'
Loading

0 comments on commit d5e3778

Please sign in to comment.