feat: Add service.yaml and improve push_test script

.github/workflows/push_test.yml

@@ -1,4 +1,4 @@
-name: Push
+name: Push Testing for Cloud Run
 
 on:
   push:
@@ -7,46 +7,128 @@ on:
 
 env:
   ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+  PROJECT_ID: policyengine-api
+  SERVICE_NAME: policyengine-api-cloud-run-test
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  docker:
-    name: Docker
+  # Temporarily disable
+  # lint:
+  #   runs-on: ubuntu-latest
+  #   if: |
+  #     (github.repository == 'PolicyEngine/policyengine-uk')
+  #     && (github.event.head_commit.message == 'Update PolicyEngine API')
+  #   steps:
+  #     - name: Checkout repo
+  #       uses: actions/checkout@v3
+  #     - name: Check formatting
+  #       uses: "lgeiger/black-action@master"
+  #       with:
+  #         args: ". -l 79 --check"
+  # versioning:
+  #   name: Update versioning
+  #   if: |
+  #     (github.repository == 'PolicyEngine/policyengine-api')
+  #     && !(github.event.head_commit.message == 'Update PolicyEngine API')
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout repo
+  #       uses: actions/checkout@v3
+  #       with:
+  #         repository: ${{ github.event.pull_request.head.repo.full_name }}
+  #         ref: ${{ github.event.pull_request.head.ref }}
+  #         token: ${{ secrets.POLICYENGINE_GITHUB }}
+  #     - name: Setup Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: "3.10"
+  #     - name: Build changelog
+  #       run: pip install yaml-changelog && make changelog
+  #     - name: Preview changelog update
+  #       run: ".github/get-changelog-diff.sh"
+  #     - name: Update changelog
+  #       uses: EndBug/add-and-commit@v9
+  #       with:
+  #         add: "."
+  #         committer_name: Github Actions[bot]
+  #         author_name: Github Actions[bot]
+  #         message: Update PolicyEngine API
+  deploy:
+    name: Deploy API to Cloud Run
     runs-on: ubuntu-latest
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    permissions:
+      contents: read
+      packages: write
+
     steps:
-      - name: Checkout repo
+      - name: Checkout repository
         uses: actions/checkout@v3
+
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
         with:
-          registry: ghcr.io
+          registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
-          password: ${{ secrets.POLICYENGINE_DOCKER }}
-      - name: Build container
-        run: docker build -t ghcr.io/policyengine/policyengine docker
-      # Temporarily avoid pushing Docker container
-      # - name: Push container
-      #   run: docker push ghcr.io/policyengine/policyengine
-  deploy:
-    name: Deploy API to Cloud Run
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Setup Python
-        uses: actions/setup-python@v5
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
-          python-version: "3.10"
-      - name: GCP authentication
-        uses: "google-github-actions/auth@v2"
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Setup Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
         with:
-          credentials_json: "${{ secrets.GCP_SA_KEY }}"
-      - name: Set up GCloud
-        uses: "google-github-actions/setup-gcloud@v1"
-      - name: Deploy
-        run: make deploy
-        env:
-          POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
-          GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GCP_SA_KEY }}
-          POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+      - name: Prepare service.yaml
+        run: |
+          sed -i 's|${SERVICE_NAME}|${{ env.SERVICE_NAME }}|g' service.yaml
+          sed -i 's|${REGISTRY}|${{ env.REGISTRY }}|g' service.yaml
+          sed -i 's|${IMAGE_NAME}|${{ env.IMAGE_NAME }}|g' service.yaml
+          sed -i 's|${IMAGE_TAG}|${{ github.sha }}|g' service.yaml
+
+      - name: Deploy to Cloud Run
+        run: |
+          gcloud run services replace service.yaml \
+            --platform managed \
+            --region us-central1
+
+      - name: Set up secrets in Secret Manager
+        run: |
+          echo "${{ secrets.POLICYENGINE_DB_PASSWORD }}" | gcloud secrets create policyengine-db-password --data-file=- || gcloud secrets versions add policyengine-db-password --data-file=-
+          echo "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" | gcloud secrets create policyengine-github-microdata-auth-token --data-file=- || gcloud secrets versions add policyengine-github-microdata-auth-token --data-file=-
+          echo "${{ secrets.ANTHROPIC_API_KEY }}" | gcloud secrets create anthropic-api-key --data-file=- || gcloud secrets versions add anthropic-api-key --data-file=-
+          echo "${{ secrets.OPENAI_API_KEY }}" | gcloud secrets create openai-api-key --data-file=- || gcloud secrets versions add openai-api-key --data-file=-
+
+  # Temporary disable
+  # docker:
+  #   name: Docker
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout repo
+  #       uses: actions/checkout@v3
+  #     - name: Log in to the Container registry
+  #       uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+  #       with:
+  #         registry: ghcr.io
+  #         username: ${{ github.actor }}
+  #         password: ${{ secrets.POLICYENGINE_DOCKER }}
+  #     - name: Build container
+  #       run: docker build -t ghcr.io/policyengine/policyengine docker
+  #     - name: Push container
+  #       run: docker push ghcr.io/policyengine/policyengine

gcp/policyengine_api/service.yaml

@@ -1,15 +1,39 @@
-# apiVersion: serving.knative.dev/v1
-# kind: Service
-# metadata:
-#   name: test-service-run 
-# spec:
-#   template:
-#     metadata:
-#       annotations:
-#         run.googleapis.com/client-name: cloud-console
-#     spec:
-#       containers:
-#       - image: gcr.io/PROJECT_ID/test-service-run:latest
-#         env:
-#         - name: ENVIRONMENT
-#           value: "test"
+apiVersion: serving.knative.dev/v1
+kind: Service
+metadata:
+  name: ${SERVICE_NAME}
+spec:
+  template:
+    metadata:
+      annotations:
+        autoscaling.knative.dev/maxScale: "10"
+    spec:
+      containers:
+        - image: ${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}
+          ports:
+            - containerPort: 8080
+          env:
+            - name: POLICYENGINE_DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: policyengine-secrets
+                  key: db-password
+            - name: POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: policyengine-secrets
+                  key: github-microdata-auth-token
+            - name: ANTHROPIC_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: policyengine-secrets
+                  key: anthropic-api-key
+            - name: OPENAI_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: policyengine-secrets
+                  key: openai-api-key
+          resources:
+            limits:
+              cpu: 1000m
+              memory: 512Mi