Separate webpage build from data processing steps in action #262 (#272)

* Separate webpage build from data processing steps in action #262

* artifacts checks

---------

Co-authored-by: Lukas Wallrich <[email protected]>

Author: richarddushime

.github/workflows/data-processing.yml

@@ -0,0 +1,66 @@
+name: Data Processing
+
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Daily at midnight
+  workflow_dispatch:
+
+jobs:
+  process-data:
+    name: Process Data
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+    env:
+      PYTHON_VERSION: "3.11"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
+      - name: Install Python dependencies
+        run: python3 -m pip install -r ./requirements.txt
+
+      - name: Run Tenzing script
+        run: python3 scripts/forrt_contribs/tenzing.py
+
+      - name: Run Curated Resources script
+        run: python3 content/resources/resource.py
+
+      - name: Move and validate Tenzing output
+        run: |
+          mv scripts/forrt_contribs/tenzing.md content/contributors/tenzing.md
+          if [ ! -f content/contributors/tenzing.md ]; then
+            echo "tenzing.md not found"
+            exit 1
+          fi
+
+      - name: Validate curated resources
+        run: |
+          for file in content/curated_resources/*; do
+            if [ ! -f "$file" ]; then
+              echo "Non-markdown file found: $file"
+              exit 1
+            fi
+          done
+
+      - name: Download GA Data
+        env:
+          GA_API_CREDENTIALS: ${{ secrets.GA_API_CREDENTIALS }}
+          GA_PROPERTY_ID: ${{ secrets.GA_PROPERTY_ID }}
+        run: python scripts/download_ga_data.py
+
+      - name: Upload data artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: data-artifact
+          path: |
+            content/contributors/tenzing.md
+            content/curated_resources/
+            data/ # GA data
+          retention-days: 1

.github/workflows/deploy.yaml

@@ -1,9 +1,5 @@
 name: deploy
 
-# Build and deploy the website
-# Contains Tenzing's script to update the contributors list
-# Contains a script to update the curated resources list
-
 on:
   push:
     branches:
@@ -12,7 +8,7 @@ on:
     branches:
       - master
   schedule:
-  - cron: '0 0 * * *'  # Run daily at midnight
+    - cron: '0 1 * * *'  # 1 AM UTC (1 hour after data processing)
   workflow_dispatch:
 
 jobs:
@@ -21,80 +17,52 @@ jobs:
     runs-on: ubuntu-22.04
     permissions:
       contents: read
-      pull-requests: read
+      actions: read  # Needed for artifact access
     env:
       HUGO_VERSION: "0.123.3"
       HUGO_EXTENDED: true
-      PYTHON_VERSION: "3.11"
     steps:
-      - name: Checkout
+      - name: Checkout repository
         uses: actions/checkout@v4
         with:
           ref: ${{ github.ref }}
           fetch-depth: 0
 
-      - name: Hugo - Setup
-        uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f
-        with:
-          hugo-version: ${{ env.HUGO_VERSION }}
-          extended: ${{ env.HUGO_EXTENDED }}
-
-      - name: Python - Setup
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
-
-      - name: Cache - Pip Dependencies
-        id: pip-cache
-        uses: actions/cache@v4
+      - name: Try to download data artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        continue-on-error: true
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          workflow: data-processing.yml
+          name: data-artifact
+          path: .
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Python - Install dependencies
+      - name: Run data processing if needed
+        if: steps.download-artifact.outcome == 'failure'
+        env:
+          PYTHON_VERSION: "3.11"
         run: |
+          # Install Python dependencies
           python3 -m pip install -r ./requirements.txt
-
-      - name: Python - Tenzing
-        run: |
+          
+          # Generate data files
           python3 scripts/forrt_contribs/tenzing.py
-
-      - name: Python - Curated Resources
-        run: |
           python3 content/resources/resource.py
-
-      - name: Script - Move Tenzing's output and validate
-        run: |
           mv scripts/forrt_contribs/tenzing.md content/contributors/tenzing.md
-
-          if [ ! -f content/contributors/tenzing.md ]; then
-            echo "tenzing.md not found"
-            exit 1
+          
+          # Download GA data if possible
+          if [ "${{ github.event_name }}" != 'pull_request' ]; then
+            python scripts/download_ga_data.py
           fi
 
-      - name: Script - Validate curated resources
-        run: |
-          for file in content/curated_resources/*; do
-            if [ ! -f "$file" ]; then
-              echo "There is a non-markdown file ( $file ) in the curated resources folder. Exiting."
-              exit 1
-            fi
-          done
-
-          NUM_FILES=$(ls -1 content/curated_resources/*.md | wc -l)
-          echo "Number of curated resources: $NUM_FILES"
-
-      - name: Download GA Data
-        if: ${{ github.event_name != 'pull_request' || github.event.inputs.GA_API_CREDENTIALS != '' }}
-        env:
-          GA_API_CREDENTIALS: ${{ secrets.GA_API_CREDENTIALS }}
-          GA_PROPERTY_ID: ${{ secrets.GA_PROPERTY_ID }}
-        run: python scripts/download_ga_data.py
+      - name: Setup Hugo
+        uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f
+        with:
+          hugo-version: ${{ env.HUGO_VERSION }}
+          extended: ${{ env.HUGO_EXTENDED }}
 
-      - name: Hugo - Build
+      - name: Build site
         run: |
           if [ "$BRANCH" != 'refs/heads/master' ]; then
             hugo --gc --minify --cleanDestinationDir --destination public --baseURL https://staging.forrt.org
@@ -104,15 +72,13 @@ jobs:
         env:
           BRANCH: ${{ github.ref }}
 
-      - name: Upload Artifact - Website
+      - name: Upload site artifact
         uses: actions/upload-artifact@v4
-        env:
-          ARTIFACT_NAME: forrt-website-${{ github.run_number }}
         with:
-          name: ${{ env.ARTIFACT_NAME }}
+          name: forrt-website-${{ github.run_number }}
           path: public/
           retention-days: 1
-
+    
   deploy-test:
     name: Deploy - Test
     runs-on: ubuntu-22.04