image builder convenience scripts

scripts/README.md

@@ -300,4 +300,14 @@ used in integration test workflow after cloud integration tests pass
 
 ### `build-image-job.sh`
 
-used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry
+used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry
+
+### `zip-services.sh`
+
+adds services to zip file. used by `scripts/build-all-images.sh` before triggering `.github/workflows/build-all-images.yaml`
+
+### `build-all-images.sh`
+zips and pushes current service code to s3, then triggers `.github/workflows/build-all-images.yaml` to avoid building almost a dozen rust images locally
+
+### `pull-all-images.sh`
+pulls images built and pushed by `.github/workflows/build-all-images.yaml`

scripts/build-all-images.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z $GITHUB_PAT ]]; then
+    echo "set GITHUB_PAT variable in shell to continue"
+    exit 1
+fi
+
+ENV=dev
+PROJECT_CONF=project.yaml
+ENV_ID=$(source ./scripts/print-env-id.sh)
+ID_ENV="$ENV_ID-$ENV"
+REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF)
+ARTIFACTS_BUCKET_PREFIX=$(yq '.infrastructure.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF)
+ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV"
+IMAGE_BUILDER_WORKFLOW=$(yq '.[".github"].workflows.env_var.set.IMAGE_BUILDER_WORKFLOW.default' $PROJECT_CONF)
+WORKFLOW_ID=$IMAGE_BUILDER_WORKFLOW
+GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF)
+GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF)
+SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF)
+
+source scripts/zip-services.sh
+
+echo '*** uploading archive to s3'
+aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET --region $REGION
+
+echo "*** triggering .github/workflows/$WORKFLOW_ID"
+
+curl -L \
+    -X POST \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $GITHUB_PAT" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/dispatches \
+    -d "{\"ref\":\"develop\"}"
+
+if [[ $(uname) == "Darwin" ]]; then
+    # store utc time in iso8601 format minus 10 seconds
+    CREATED=$(date -u -v-10S "+%Y-%m-%dT%H:%M:%SZ")
+    # store utc time in unix timestamp format plus 10 minutes
+    TEN_MIN_MAX=$(date -u -v+10M "+%s")
+else
+    CREATED=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 seconds ago" +'%Y-%m-%dT%H:%M:%SZ')
+    TEN_MIN_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 minutes" +'%s')
+fi
+
+# wait 5 seconds
+sleep 5
+
+RUN_ID=$(curl -sL \
+  -H "Accept: application/vnd.github+json" \
+  -H "Authorization: Bearer $GITHUB_PAT" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  "https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/runs?created=>$CREATED" | yq '.workflow_runs[0].id')
+
+echo "*** waiting for $WORKFLOW_ID github workflow to complete"
+
+function get_run() {
+    RUN=$(curl -sL \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $GITHUB_PAT" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/runs/$RUN_ID)
+
+    STATUS=$(echo "$RUN" | yq '.status')
+    CONCLUSION=$(echo "$RUN" | yq '.conclusion')
+}
+
+STATUS='queued'
+CONCLUSION=null
+
+while [[ $STATUS != 'completed' && $(date +%s) -lt $TEN_MIN_MAX ]]; do
+  sleep 5
+  get_run
+  printf '%s' '.'
+done
+
+echo ""
+
+if [[ $CONCLUSION != 'success' ]]; then
+  echo "build failed"
+  exit 1
+fi
+
+source scripts/pull-all-images.sh

scripts/pull-all-images.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_CONF=project.yaml
+GITHUB_REGISTRY=$(yq '.[".github"].workflows.env_var.set.GITHUB_REGISTRY.default' $PROJECT_CONF)
+GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF)
+GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF)
+LOCAL_TAG_VERSION=$(yq '.docker.env_var.set.LOCAL_TAG_VERSION.default' $PROJECT_CONF)
+
+NAMESPACE=$GITHUB_ORG/$GITHUB_REPO_NAME
+REGISTRY_URI=$GITHUB_REGISTRY/$NAMESPACE
+
+SERVICES=($(bash scripts/list-dir-paths.sh --type app | grep -v client | xargs basename -a))
+
+for SERVICE in "${SERVICES[@]}"; do
+    IMAGE_NAME=$SERVICE:$LOCAL_TAG_VERSION
+    docker pull $REGISTRY_URI/$IMAGE_NAME
+    docker tag $REGISTRY_URI/$IMAGE_NAME $IMAGE_NAME
+done
+
+echo ""
+echo '*** "make compose-up" to start services in docker'

scripts/zip-services.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+if [[ $(basename $(pwd)) != "mxfactorial" ]]; then
+	echo "error: current directory not project root. run this script from project root"
+	exit 1
+fi
+
+PROJECT_CONF=project.yaml
+SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF)
+
+rm -f $SERVICES_ZIP
+
+echo '*** archiving current services code'
+
+zip -r $SERVICES_ZIP \
+	Cargo.toml \
+	Cargo.lock \
+	.cargo \
+	docker \
+	make \
+	makefile \
+	project.yaml \
+	services \
+	crates \
+	tests \
+	migrations/go-migrate \
+	--exclude='*/.env' \
+	--exclude='*/README.md' \
+	--exclude='tests/testdata/*' \
+	--exclude='tests/thunder-tests/*' \
+	--exclude='services/graphql/postman/*' 1>/dev/null