Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update build workflow to use re-useable workflow #17

Closed
wants to merge 5 commits into from
Closed

update build workflow to use re-useable workflow #17

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
18748f6
update build workflow to use reuseable workflow
DeeAjayi Nov 4, 2024
96b8a61
update workflow
DeeAjayi Nov 4, 2024
2077db6
test workflow
DeeAjayi Nov 4, 2024
171568e
fix workflow error
DeeAjayi Nov 4, 2024
aa98c6a
update build tag to v0.1.1-rc.1
DeeAjayi Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 49 additions & 127 deletions .github/workflows/build-and-push-image.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ on:
push:
tags:
- v*
branches:
- update-workflow

permissions:
id-token: write
Expand All @@ -18,142 +20,62 @@ env:
TAG_PREFIX: "v"

jobs:
build_image:
name: Build Image
set_version_tags:
name: Set version tags
runs-on: ubuntu-latest
outputs:
version_tag: ${{ steps.set_version_tags.outputs.version_tag }}
jupyter_version_tag: ${{ steps.set_version_tags.outputs.jupyter_version_tag }}
alt_version_tag: ${{ steps.set_version_tags.outputs.alt_version_tag }}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: false
swap-storage: false

- name: Checkout
uses: actions/checkout@v3

- name: Set version tags
id: set_version_tags
run: |
echo "Ref name is ${{ github.ref_name }}"
echo "TAG_PREFIX is ${{ env.TAG_PREFIX }}"

REF_NAME=${{ github.ref_name }}
TAG_PREFIX=${{ env.TAG_PREFIX }}
VERSION_TAG=$(echo $REF_NAME | cut -d $TAG_PREFIX -f 2-)
echo "Setting VERSION_TAG equal to $VERSION_TAG"
echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_ENV
VERSION_TAG=$(echo $REF_NAME | awk -F$TAG_PREFIX '{print $2}')
echo "Setting VERSION_TAG to $VERSION_TAG"
echo "version_tag=$VERSION_TAG" >> $GITHUB_OUTPUT

JUPYTER_VERSION_TAG=$(echo $REF_NAME | cut -d $TAG_PREFIX -f 2-)-jupyter
echo "Setting JUPYTER VERSION_TAG equal to JUPYTER_VERSION_TAG"
echo "jupyter_version_tag=JUPYTER_VERSION_TAG" >> $GITHUB_OUTPUT

ALT_VERSION_TAG=$(echo $VERSION_TAG | awk -F- '{print $1}')-${GITHUB_SHA::7}
echo "Setting ALT_VERSION_TAG equal to $ALT_VERSION_TAG"
echo "ALT_VERSION_TAG=$ALT_VERSION_TAG" >> $GITHUB_ENV

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
aws-region: us-east-1

- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
with:
registry-type: public

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Set repo uri
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
REGISTRY_ALIAS: truefoundrycloud
REPOSITORY: llm-finetune
run: |
REPOSITORY_URI=${{ env.REGISTRY }}/${{ env.REGISTRY_ALIAS }}/${{ env.REPOSITORY }}
echo "Setting REPOSITORY_URI equal to $REPOSITORY_URI"
echo "REPOSITORY_URI=$REPOSITORY_URI" >> $GITHUB_ENV

- name: Build and push image
uses: docker/build-push-action@v5
env:
IMAGE_TAG_1: ${{ env.REPOSITORY_URI }}:${{ env.GITHUB_SHA }}
IMAGE_TAG_2: ${{ env.REPOSITORY_URI }}:${{ env.VERSION_TAG }}
IMAGE_TAG_3: ${{ env.REPOSITORY_URI }}:${{ env.ALT_VERSION_TAG }}
with:
context: .
file: Dockerfile
platforms: linux/amd64
push: true
tags: ${{ env.IMAGE_TAG_1 }},${{ env.IMAGE_TAG_2 }},${{ env.IMAGE_TAG_3 }}
cache-from: type=registry,ref=${{ env.REPOSITORY_URI }}:buildcache
cache-to: mode=max,image-manifest=true,type=registry,ref=${{ env.REPOSITORY_URI }}:buildcache

build_notebook_image:
name: Build Notebook Image
runs-on: ubuntu-latest
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: false
swap-storage: false

- name: Checkout
uses: actions/checkout@v3

- name: Set version tags
run: |
echo "Ref name is ${{ github.ref_name }}"
echo "TAG_PREFIX is ${{ env.TAG_PREFIX }}"

REF_NAME=${{ github.ref_name }}
TAG_PREFIX=${{ env.TAG_PREFIX }}
VERSION_TAG=$(echo $REF_NAME | cut -d $TAG_PREFIX -f 2-)-jupyter
echo "Setting VERSION_TAG equal to $VERSION_TAG"
echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_ENV

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.PUBLIC_ECR_IAM_ROLE_ARN }}
aws-region: us-east-1

- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
with:
registry-type: public

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Set repo uri
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
REGISTRY_ALIAS: truefoundrycloud
REPOSITORY: llm-finetune
run: |
REPOSITORY_URI=${{ env.REGISTRY }}/${{ env.REGISTRY_ALIAS }}/${{ env.REPOSITORY }}
echo "Setting REPOSITORY_URI equal to $REPOSITORY_URI"
echo "REPOSITORY_URI=$REPOSITORY_URI" >> $GITHUB_ENV

- name: Build and push image
uses: docker/build-push-action@v5
env:
IMAGE_TAG_1: ${{ env.REPOSITORY_URI }}:${{ env.VERSION_TAG }}
with:
context: .
file: Dockerfile-notebook
platforms: linux/amd64
push: true
tags: ${{ env.IMAGE_TAG_1 }}
cache-from: type=registry,ref=${{ env.REPOSITORY_URI }}:jupyter-buildcache
cache-to: mode=max,image-manifest=true,type=registry,ref=${{ env.REPOSITORY_URI }}:jupyter-buildcache
echo "alt_version_tag=$ALT_VERSION_TAG" >> $GITHUB_OUTPUT

build_finetune_image:
name: Build and Push llm-finetune image
needs: set_version_tags
uses: truefoundry/workflows/.github/workflows/[email protected]
with:
image_tag: ${{ github.sha }}
extra_image_tag: |
${{ needs.set_version_tags.outputs.version_tag }}
${{ needs.set_version_tags.outputs.alt_version_tag }}
platforms: linux/amd64
free_disk_space: true
image_artifact_name: ${{ github.event.repository.name }}
artifactory_registry_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }}
artifactory_repository_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }}
secrets:
artifactory_username: ${{ secrets.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_USERNAME }}
artifactory_password: ${{ secrets.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_PASSWORD }}

build_jupyter_image:
name: Build and Push jupyter image
uses: truefoundry/workflows/.github/workflows/[email protected]
with:
image_tag: ${{ needs.set_version_tags.outputs.jupyter_version_tag }}
platforms: linux/amd64
free_disk_space: true
image_artifact_name: ${{ github.event.repository.name }}
artifactory_registry_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }}
artifactory_repository_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }}
secrets:
artifactory_username: ${{ secrets.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_USERNAME }}
artifactory_password: ${{ secrets.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_PASSWORD }}