From 9f59bbb0a3fea1ca49fa5435c7fff87fb312d261 Mon Sep 17 00:00:00 2001 From: max funk Date: Tue, 16 Apr 2024 19:49:19 -0700 Subject: [PATCH] image builder convenience scripts --- scripts/README.md | 12 +++++- scripts/build-all-images.sh | 86 +++++++++++++++++++++++++++++++++++++ scripts/pull-all-images.sh | 23 ++++++++++ scripts/zip-services.sh | 31 +++++++++++++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 scripts/build-all-images.sh create mode 100644 scripts/pull-all-images.sh create mode 100644 scripts/zip-services.sh diff --git a/scripts/README.md b/scripts/README.md index 69baac46..4b26bd52 100755 --- a/scripts/README.md +++ b/scripts/README.md @@ -300,4 +300,14 @@ used in integration test workflow after cloud integration tests pass ### `build-image-job.sh` -used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry \ No newline at end of file +used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry + +### `zip-services.sh` + +adds services to zip file. used by `scripts/build-all-images.sh` before triggering `.github/workflows/build-all-images.yaml` + +### `build-all-images.sh` +zips and pushes current service code to s3, then triggers `.github/workflows/build-all-images.yaml` to avoid building almost a dozen rust images locally + +### `pull-all-images.sh` +pulls images built and pushed by `.github/workflows/build-all-images.yaml` \ No newline at end of file diff --git a/scripts/build-all-images.sh b/scripts/build-all-images.sh new file mode 100644 index 00000000..49705cd1 --- /dev/null +++ b/scripts/build-all-images.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -e + +if [[ -z $GITHUB_PAT ]]; then + echo "set GITHUB_PAT variable in shell to continue" + exit 1 +fi + +ENV=dev +PROJECT_CONF=project.yaml +ENV_ID=$(source ./scripts/print-env-id.sh) +ID_ENV="$ENV_ID-$ENV" +REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +ARTIFACTS_BUCKET_PREFIX=$(yq '.infrastructure.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) +ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" +IMAGE_BUILDER_WORKFLOW=$(yq '.[".github"].workflows.env_var.set.IMAGE_BUILDER_WORKFLOW.default' $PROJECT_CONF) +WORKFLOW_ID=$IMAGE_BUILDER_WORKFLOW +GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) +GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) +SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + +source scripts/zip-services.sh + +echo '*** uploading archive to s3' +aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET --region $REGION + +echo "*** triggering .github/workflows/$WORKFLOW_ID" + +curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_PAT" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/dispatches \ + -d "{\"ref\":\"develop\"}" + +if [[ $(uname) == "Darwin" ]]; then + # store utc time in iso8601 format minus 10 seconds + CREATED=$(date -u -v-10S "+%Y-%m-%dT%H:%M:%SZ") + # store utc time in unix timestamp format plus 10 minutes + TEN_MIN_MAX=$(date -u -v+10M "+%s") +else + CREATED=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 seconds ago" +'%Y-%m-%dT%H:%M:%SZ') + TEN_MIN_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 minutes" +'%s') +fi + +# wait 5 seconds +sleep 5 + +RUN_ID=$(curl -sL \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_PAT" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/runs?created=>$CREATED" | yq '.workflow_runs[0].id') + +echo "*** waiting for $WORKFLOW_ID github workflow to complete" + +function get_run() { + RUN=$(curl -sL \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GITHUB_PAT" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/runs/$RUN_ID) + + STATUS=$(echo "$RUN" | yq '.status') + CONCLUSION=$(echo "$RUN" | yq '.conclusion') +} + +STATUS='queued' +CONCLUSION=null + +while [[ $STATUS != 'completed' && $(date +%s) -lt $TEN_MIN_MAX ]]; do + sleep 5 + get_run + printf '%s' '.' +done + +echo "" + +if [[ $CONCLUSION != 'success' ]]; then + echo "build failed" + exit 1 +fi + +source scripts/pull-all-images.sh \ No newline at end of file diff --git a/scripts/pull-all-images.sh b/scripts/pull-all-images.sh new file mode 100644 index 00000000..9d10735d --- /dev/null +++ b/scripts/pull-all-images.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +PROJECT_CONF=project.yaml +GITHUB_REGISTRY=$(yq '.[".github"].workflows.env_var.set.GITHUB_REGISTRY.default' $PROJECT_CONF) +GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) +GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) +LOCAL_TAG_VERSION=$(yq '.docker.env_var.set.LOCAL_TAG_VERSION.default' $PROJECT_CONF) + +NAMESPACE=$GITHUB_ORG/$GITHUB_REPO_NAME +REGISTRY_URI=$GITHUB_REGISTRY/$NAMESPACE + +SERVICES=($(bash scripts/list-dir-paths.sh --type app | grep -v client | xargs basename -a)) + +for SERVICE in "${SERVICES[@]}"; do + IMAGE_NAME=$SERVICE:$LOCAL_TAG_VERSION + docker pull $REGISTRY_URI/$IMAGE_NAME + docker tag $REGISTRY_URI/$IMAGE_NAME $IMAGE_NAME +done + +echo "" +echo '*** "make compose-up" to start services in docker' \ No newline at end of file diff --git a/scripts/zip-services.sh b/scripts/zip-services.sh new file mode 100644 index 00000000..c1d63e20 --- /dev/null +++ b/scripts/zip-services.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +if [[ $(basename $(pwd)) != "mxfactorial" ]]; then + echo "error: current directory not project root. run this script from project root" + exit 1 +fi + +PROJECT_CONF=project.yaml +SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + +rm -f $SERVICES_ZIP + +echo '*** archiving current services code' + +zip -r $SERVICES_ZIP \ + Cargo.toml \ + Cargo.lock \ + .cargo \ + docker \ + make \ + makefile \ + project.yaml \ + services \ + crates \ + tests \ + migrations/go-migrate \ + --exclude='*/.env' \ + --exclude='*/README.md' \ + --exclude='tests/testdata/*' \ + --exclude='tests/thunder-tests/*' \ + --exclude='services/graphql/postman/*' 1>/dev/null \ No newline at end of file