Merge pull request #357 from systemaccounting/354-rust-image-builder

354 rust image builder

README.md

@@ -51,7 +51,7 @@ encryption and replication are secondary
 government is not above failure, nor is it entitled to steal from the private sector to conceal its failure. improving government depends on failure [predicting](https://en.wikipedia.org/wiki/Time_travel_debugging) the individuals and laws that must be replaced. flying a flag and demanding loyalty before this step is just misdirection  
 
 **q.** how does systemaccounting manage expectation?  
-**a.** central banks providing "forward guidance" *appease* more than they **set** expectation when they allow interest rate manipulation and money printing. systemaccounting prices capital by switching the "risk-free" rate from referencing the hackable price of debt to the immutably recorded price of equity. when the risk-free rate refers to the empirical rate, i.e. to the historical and not the expected, the economy remains protected from the catastrophic failure indulged by intended government mispricing  
+**a.** central banks providing "forward guidance" appease more than they set expectation when they allow interest rate manipulation and money printing. systemaccounting prices capital by switching the "risk-free" rate from referencing the hackable price of debt to the immutably recorded price of equity. when the risk-free rate refers to the empirical rate, i.e. to the historical and not the expected, the economy remains protected from the catastrophic failure indulged by intended government mispricing  
 
 removing financial appeasement guides the freedom of speech by recalibrating expectation to the empirical  
 

scripts/README.md

@@ -300,4 +300,14 @@ used in integration test workflow after cloud integration tests pass
 
 ### `build-image-job.sh`
 
-used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry
+used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry
+
+### `zip-services.sh`
+
+adds services to zip file. used by `scripts/build-all-images.sh` before triggering `.github/workflows/build-all-images.yaml`
+
+### `build-all-images.sh`
+zips and pushes current service code to s3, then triggers `.github/workflows/build-all-images.yaml` to avoid building almost a dozen rust images locally
+
+### `pull-all-images.sh`
+pulls images built and pushed by `.github/workflows/build-all-images.yaml`

scripts/build-all-images.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z $GITHUB_PAT ]]; then
+    echo "set GITHUB_PAT variable in shell to continue"
+    exit 1
+fi
+
+ENV=dev
+PROJECT_CONF=project.yaml
+ENV_ID=$(source ./scripts/print-env-id.sh)
+ID_ENV="$ENV_ID-$ENV"
+REGION=$(yq '.infrastructure.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF)
+ARTIFACTS_BUCKET_PREFIX=$(yq '.infrastructure.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF)
+ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV"
+IMAGE_BUILDER_WORKFLOW=$(yq '.[".github"].workflows.env_var.set.IMAGE_BUILDER_WORKFLOW.default' $PROJECT_CONF)
+WORKFLOW_ID=$IMAGE_BUILDER_WORKFLOW
+GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF)
+GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF)
+SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF)
+
+source scripts/zip-services.sh
+
+echo '*** uploading archive to s3'
+aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET --region $REGION
+
+echo "*** triggering .github/workflows/$WORKFLOW_ID"
+
+curl -L \
+    -X POST \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $GITHUB_PAT" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/dispatches \
+    -d "{\"ref\":\"develop\"}"
+
+if [[ $(uname) == "Darwin" ]]; then
+    # store utc time in iso8601 format minus 10 seconds
+    CREATED=$(date -u -v-10S "+%Y-%m-%dT%H:%M:%SZ")
+    # store utc time in unix timestamp format plus 10 minutes
+    TEN_MIN_MAX=$(date -u -v+10M "+%s")
+else
+    CREATED=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 seconds ago" +'%Y-%m-%dT%H:%M:%SZ')
+    TEN_MIN_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 minutes" +'%s')
+fi
+
+# wait 5 seconds
+sleep 5
+
+RUN_ID=$(curl -sL \
+  -H "Accept: application/vnd.github+json" \
+  -H "Authorization: Bearer $GITHUB_PAT" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  "https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/runs?created=>$CREATED" | yq '.workflow_runs[0].id')
+
+echo "*** waiting for $WORKFLOW_ID github workflow to complete"
+
+function get_run() {
+    RUN=$(curl -sL \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $GITHUB_PAT" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/runs/$RUN_ID)
+
+    STATUS=$(echo "$RUN" | yq '.status')
+    CONCLUSION=$(echo "$RUN" | yq '.conclusion')
+}
+
+STATUS='queued'
+CONCLUSION=null
+
+while [[ $STATUS != 'completed' && $(date +%s) -lt $TEN_MIN_MAX ]]; do
+  sleep 5
+  get_run
+  printf '%s' '.'
+done
+
+echo ""
+
+if [[ $CONCLUSION != 'success' ]]; then
+  echo "build failed"
+  exit 1
+fi
+
+source scripts/pull-all-images.sh

scripts/pull-all-images.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_CONF=project.yaml
+GITHUB_REGISTRY=$(yq '.[".github"].workflows.env_var.set.GITHUB_REGISTRY.default' $PROJECT_CONF)
+GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF)
+GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF)
+LOCAL_TAG_VERSION=$(yq '.docker.env_var.set.LOCAL_TAG_VERSION.default' $PROJECT_CONF)
+
+NAMESPACE=$GITHUB_ORG/$GITHUB_REPO_NAME
+REGISTRY_URI=$GITHUB_REGISTRY/$NAMESPACE
+
+SERVICES=($(bash scripts/list-dir-paths.sh --type app | grep -v client | xargs basename -a))
+
+for SERVICE in "${SERVICES[@]}"; do
+    IMAGE_NAME=$SERVICE:$LOCAL_TAG_VERSION
+    docker pull $REGISTRY_URI/$IMAGE_NAME
+    docker tag $REGISTRY_URI/$IMAGE_NAME $IMAGE_NAME
+done
+
+echo ""
+echo '*** "make compose-up" to start services in docker'

scripts/zip-services.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+if [[ $(basename $(pwd)) != "mxfactorial" ]]; then
+	echo "error: current directory not project root. run this script from project root"
+	exit 1
+fi
+
+PROJECT_CONF=project.yaml
+SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF)
+
+rm -f $SERVICES_ZIP
+
+echo '*** archiving current services code'
+
+zip -r $SERVICES_ZIP \
+	Cargo.toml \
+	Cargo.lock \
+	.cargo \
+	docker \
+	make \
+	makefile \
+	project.yaml \
+	services \
+	crates \
+	tests \
+	migrations/go-migrate \
+	--exclude='*/.env' \
+	--exclude='*/README.md' \
+	--exclude='tests/testdata/*' \
+	--exclude='tests/thunder-tests/*' \
+	--exclude='services/graphql/postman/*' 1>/dev/null