Merge pull request #15 from shaneknapp/fine-tune-image-building

update test.yaml to only run if the image config files updated
berkeley-dsep-infra · Jun 25, 2024 · 21617d5 · 21617d5
2 parents 044b7ef + 178dc67
commit 21617d5
Show file tree

Hide file tree

Showing 3 changed files with 126 additions and 68 deletions.
diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml
@@ -10,43 +10,74 @@ jobs:
     runs-on: ubuntu-latest
     env:
       DOCKER_CONFIG: $HOME/.docker
+      HUB: logodev
 
     steps:
+      - name: Test changed-files
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # OR "2" -> To retrieve the preceding commit.
 
-    # For biggish images, github actions runs out of disk space.
-    # So we cleanup some unwanted things in the disk image, and reclaim that space for our docker use
-    # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
-    # and https://github.com/easimon/maximize-build-space/blob/b4d02c14493a9653fe7af06cc89ca5298071c66e/action.yml#L104
-    # This gives us a total of about 52G of free space, which should be enough for now
-    - name: cleanup disk space
-      run: |
-        sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
-        df -h
-
-    - name: Checkout files in repo
-      uses: actions/checkout@main
-
-    - name: Log in to GAR
-      uses: docker/login-action@v1
-      with:
-        registry: us-central1-docker.pkg.dev
-        username: _json_key
-        password: ${{ secrets.GAR_SECRET_KEY }}
-
-    - name: Build the image and push to artifact registry
-      uses: jupyterhub/repo2docker-action@master
-      with:
-        # Make sure username & password/token pair matches your registry credentials
-        DOCKER_REGISTRY: us-central1-docker.pkg.dev
-        IMAGE_NAME: ucb-datahub-2018/user-images/logodev-user-image
-        # Disable pushing a 'latest' tag, as this often just causes confusion
-        # LATEST_TAG_OFF: true
-        # Put repo contents in /srv/repo, rather than the default (/home/jovyan). The home directory
-        # is mounted over by persistent storage when we are using the built image in a JupyterHub, and
-        # so all contents put in /home/jovyan are lost. This particularly prevents any 'start' script from
-        # working, as it is needed in runtime.
-        REPO_DIR: /srv/repo
-
-    # Lets us monitor disks getting full as images get bigger over time
-    - name: Show how much disk space is left
-      run: df -h
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+          files: |
+            **.bash
+            **.py
+            **.r
+            **.R
+            **.sh
+            **.txt
+            **.toml
+            **.yaml
+            **.yml
+            Dockerfile
+            infra-requirements.txt
+            Pipfile**
+            postBuild
+            requirements.txt
+            setup.py
+            start
+          files_ignore: |
+            .github/**
+            images/**
+      # NOTE: `since_last_remote_commit: true` is implied by default and falls back to the previous local commit.
+
+      # For biggish images, github actions runs out of disk space.
+      # So we cleanup some unwanted things in the disk image, and reclaim that space for our docker use
+      # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
+      # and https://github.com/easimon/maximize-build-space/blob/b4d02c14493a9653fe7af06cc89ca5298071c66e/action.yml#L104
+      # This gives us a total of about 52G of free space, which should be enough for now
+      - name: cleanup disk space
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
+          df -h
+
+      - name: Log in to GAR
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: docker/login-action@v1
+        with:
+          registry: us-central1-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.GAR_SECRET_KEY }}
+
+      - name: Build the image and push to artifact registry
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: jupyterhub/repo2docker-action@master
+        with:
+          # Make sure username & password/token pair matches your registry credentials
+          DOCKER_REGISTRY: us-central1-docker.pkg.dev
+          IMAGE_NAME: ucb-datahub-2018/user-images/$HUB-user-image
+          # Disable pushing a 'latest' tag, as this often just causes confusion
+          # LATEST_TAG_OFF: true
+          # Put repo contents in /srv/repo, rather than the default (/home/jovyan). The home directory
+          # is mounted over by persistent storage when we are using the built image in a JupyterHub, and
+          # so all contents put in /home/jovyan are lost. This particularly prevents any 'start' script from
+          # working, as it is needed in runtime.
+          REPO_DIR: /srv/repo
+
+      # Lets us monitor disks getting full as images get bigger over time
+      - name: Show how much disk space is left
+        run: df -h
diff --git a/.github/workflows/hubploy.yaml b/.github/workflows/hubploy.yaml
@@ -9,6 +9,8 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    env:
+      HUB: logodev
     steps:
 
     # For biggish images, github actions runs out of disk space.
@@ -26,7 +28,7 @@ jobs:
       with:
         repository: 'berkeley-dsep-infra/datahub'
         sparse-checkout: |
-          deployments/logodev
+          deployments/$HUB
 
     - name: install python + requirements
       uses: actions/setup-python@v5
@@ -37,7 +39,7 @@ jobs:
 
     - name: see if the correct dir has been checked out, and the requirements have been installed
       run: |
-        ls -la deployments/logodev
+        ls -la deployments/$HUB
         pip freeze
 
     # Lets us monitor disks getting full as images get bigger over time

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -10,33 +10,58 @@ jobs:
       DOCKER_CONFIG: $HOME/.docker
 
     steps:
+      # For biggish images, github actions runs out of disk space.
+      # So we cleanup some unwanted things in the disk image, and reclaim that space for our docker use
+      # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
+      # and https://github.com/easimon/maximize-build-space/blob/b4d02c14493a9653fe7af06cc89ca5298071c66e/action.yml#L104
+      # This gives us a total of about 52G of free space, which should be enough for now
+      - name: Cleanup disk space
+        run: |
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
+          df -h
 
-    # For biggish images, github actions runs out of disk space.
-    # So we cleanup some unwanted things in the disk image, and reclaim that space for our docker use
-    # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
-    # and https://github.com/easimon/maximize-build-space/blob/b4d02c14493a9653fe7af06cc89ca5298071c66e/action.yml#L104
-    # This gives us a total of about 52G of free space, which should be enough for now
-    - name: Cleanup disk space
-      run: |
-        sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
-        df -h
-
-    - name: Checkout files in repo
-      uses: actions/checkout@main
-
-    - name: Build and test the image
-      uses: jupyterhub/repo2docker-action@master
-      with:
-        # Make sure username & password/token pair matches your registry credentials
-        DOCKER_REGISTRY: us-central1-docker.pkg.dev
-        IMAGE_NAME: ucb-datahub-2018/user-images/logodev-user-image
-        NO_PUSH: "true"
-        # Put repo contents in /srv/repo, rather than the default (/home/jovyan). The home directory
-        # is mounted over by persistent storage when we are using the built image in a JupyterHub, and
-        # so all contents put in /home/jovyan are lost. This particularly prevents any 'start' script from
-        # working, as it is needed in runtime.
-        REPO_DIR: /srv/repo
-
-    # Lets us monitor disks getting full as images get bigger over time
-    - name: Show how much disk space is left
-      run: df -h
+      - name: Checkout files in repo
+        uses: actions/checkout@v4
+
+      - name: See if the image config changed
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+            files: |
+              **.bash
+              **.py
+              **.r
+              **.R
+              **.sh
+              **.txt
+              **.toml
+              **.yaml
+              **.yml
+              Dockerfile
+              infra-requirements.txt
+              Pipfile**
+              postBuild
+              requirements.txt
+              setup.py
+              start
+            files_ignore: |
+              .github/**
+              images/**
+
+      - name: What files changed?
+        if: steps.changed-files.outputs.any_changed == 'true'
+        env:
+            CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+            echo "One or more image file(s) has changed:"
+            echo "$CHANGED_FILES"
+
+      - name: Build and test the image if any image file changed
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: jupyterhub/repo2docker-action@master
+        with:
+          REPO_DIR: /srv/repo
+
+      # Lets us monitor disks getting full as images get bigger over time
+      - name: Show how much disk space is left
+        run: df -h