Liquibase Manual Deploy Action (#687)

* liquibase manual deploy workflow

* fixed lint

* added cloud run files

* fixed credentials typing

* added exception

* added cloud run config

* fixed the image and updated commands

* moved liquibase to Dockerfile

* added workflow to trigger the schema update

* added region param to workflow gcloud calls

* added temp dir and cleaned code

.github/workflows/deploy_schema_updater.yaml

@@ -0,0 +1,31 @@
+name: Deploy Schema Updater
+
+on:
+  workflow_dispatch:
+
+
+jobs:
+  build image:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - id: "google-cloud-auth"
+        name: "Authenticate to Google Cloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          workload_identity_provider: "projects/774248915715/locations/global/workloadIdentityPools/gh-deploy-pool/providers/gh-provider"
+          service_account: "[email protected]"
+
+      - id: "google-cloud-sdk-setup"
+        name: "Set up Cloud SDK"
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Build image with Cloud Build
+        run: |
+          gcloud builds submit --tag gcr.io/sample-metadata/schema-updater:latest ./db/deploy
+
+      - name: Deploy image to Cloud Run
+        run: |
+          gcloud run deploy schema-updater --image gcr.io/sample-metadata/schema-updater --platform managed --region australia-southeast1 --no-allow-unauthenticated

.github/workflows/trigger_schema_updater.yaml

@@ -0,0 +1,41 @@
+name: Trigger Schema Updater
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Target environment (prod or dev)'
+        required: true
+        default: 'dev'
+        type: choice
+        options:
+          - prod
+          - dev
+
+jobs:
+  invoke-cloud-run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - id: "google-cloud-auth"
+        name: "Authenticate to Google Cloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          workload_identity_provider: "projects/774248915715/locations/global/workloadIdentityPools/gh-deploy-pool/providers/gh-provider"
+          service_account: "[email protected]"
+
+      - id: "google-cloud-sdk-setup"
+        name: "Set up Cloud SDK"
+        uses: google-github-actions/setup-gcloud@v1
+
+      - id: get_url
+        name: Get Cloud Run service URL
+        run: |
+          echo "CLOUD_RUN_URL=$(gcloud run services describe schema-updater --region australia-southeast1 --format 'value(status.url)')/execute-liquibase?environment=${{ github.event.inputs.environment }}" >> $GITHUB_ENV
+
+      - name: Manual Trigger - Invoke Cloud Run
+        run: |
+          curl -X POST -H "Authorization: Bearer $(gcloud auth print-identity-token)" -H "Content-Type: application/xml" --data-binary "@db/project.xml" $CLOUD_RUN_URL
+        env:
+          CLOUD_RUN_URL: ${{ env.CLOUD_RUN_URL }}

.gitignore

@@ -1,6 +1,7 @@
 db/postgres*.jar
 .vscode/
 env/
+venv/
 __pycache__/
 *.pyc
 .DS_Store

db/deploy/Dockerfile

@@ -0,0 +1,35 @@
+# Use the official lightweight Python image.
+FROM python:3.11-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+ENV LIQUIBASE_VERSION=4.26.0
+ENV MARIADB_JDBC_VERSION=3.0.3
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends gcc git ssh default-jdk wget unzip
+
+# Download and install Liquibase
+RUN wget https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.zip \
+    && unzip liquibase-${LIQUIBASE_VERSION}.zip -d /opt/liquibase \
+    && chmod +x /opt/liquibase/liquibase \
+    # Clean up to reduce layer size
+    && rm liquibase-${LIQUIBASE_VERSION}.zip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Download the MariaDB JDBC driver
+RUN wget https://downloads.mariadb.com/Connectors/java/connector-java-${MARIADB_JDBC_VERSION}/mariadb-java-client-${MARIADB_JDBC_VERSION}.jar
+RUN mv mariadb-java-client-${MARIADB_JDBC_VERSION}.jar /opt/
+
+# Copy local code to the container image.
+ENV APP_HOME /app
+WORKDIR $APP_HOME
+COPY . ./
+
+# Install Python dependencies
+RUN python3 -m pip install --no-cache-dir --break-system-packages -r requirements.txt
+
+# Run the FastAPI app on container startup
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]

db/deploy/main.py

@@ -0,0 +1,82 @@
+import contextlib
+import json
+import os
+import subprocess
+import tempfile
+from typing import Dict, Literal
+
+from fastapi import FastAPI, HTTPException, Query, Request
+from google.cloud import logging, secretmanager
+
+app = FastAPI()
+
+# Setup for Google Cloud clients and logging
+SECRET_CLIENT = secretmanager.SecretManagerServiceClient()
+LOGGING_CLIENT = logging.Client()
+SECRET_PROJECT = 'sample-metadata'
+SECRET_NAME = 'liquibase-schema-updater'
+log_name = 'lb_schema_update_log'
+logger = LOGGING_CLIENT.logger(log_name)
+
+# Important to maintain this filename otherwise Liquibase fails to recognise previous migrations
+changelog_file = 'project.xml'
+
+
+def read_db_credentials(env: Literal['prod', 'dev']) -> Dict[Literal['dbname', 'username', 'password', 'host'], str]:
+    """Get database credentials from Secret Manager."""
+    try:
+        secret_path = SECRET_CLIENT.secret_version_path(SECRET_PROJECT, SECRET_NAME, 'latest')
+        response = SECRET_CLIENT.access_secret_version(request={'name': secret_path})
+        return json.loads(response.payload.data.decode('UTF-8'))[env]
+    except Exception as e:  # Broad exception for example; refine as needed
+        text = f'Failed to retrieve or parse secrets: {e}'
+        logger.log_text(text, severity='ERROR')
+        raise HTTPException(status_code=500, detail=text) from e
+
+
+@app.post('/execute-liquibase')
+async def execute_liquibase(request: Request, environment: Literal['prod', 'dev'] = Query(default='dev', regex='^(prod|dev)$')):
+    """Endpoint to remotely trigger Liquibase commands on a GCP VM using XML content."""
+    xml_content = await request.body()
+
+    # Clean up the local temporary file
+    credentials = read_db_credentials(env=environment)
+    db_username = credentials['username']
+    db_password = credentials['password']
+    db_hostname = credentials['host']
+    db_name = credentials['dbname']
+
+    # Temporary file creation with XML content
+    with tempfile.TemporaryDirectory() as tempdir:
+        # Specify the file path within the temporary directory
+        with contextlib.chdir(tempdir):  # pylint: disable=E1101
+            with open(changelog_file, 'wb') as temp_file:
+                temp_file.write(xml_content)
+                temp_file_path = temp_file.name  # Store file path to use later
+                remote_file_path = os.path.basename(temp_file_path)
+
+            # The actual command to run on the VM
+            liquibase_command = [
+                '/opt/liquibase/liquibase',
+                f'--changeLogFile={remote_file_path}',
+                f'--url=jdbc:mariadb://{db_hostname}/{db_name}',
+                f'--driver=org.mariadb.jdbc.Driver',
+                f'--classpath=/opt/mariadb-java-client-3.0.3.jar',
+                'update',
+            ]
+
+            try:
+                # Execute the gcloud command
+                result = subprocess.run(liquibase_command, check=True, capture_output=True, text=True, env={'LIQUIBASE_COMMAND_PASSWORD': db_password, 'LIQUIBASE_COMMAND_USERNAME': db_username, **os.environ},)
+                logger.log_text(f'Liquibase update successful: {result.stdout}', severity='INFO')
+                os.remove(temp_file_path)
+                return {'message': 'Liquibase update executed successfully', 'output': result.stdout}
+            except subprocess.CalledProcessError as e:
+                text = f'Failed to execute Liquibase update: {e.stderr}'
+                logger.log_text(text, severity='ERROR')
+                raise HTTPException(status_code=500, detail=text) from e
+
+
+if __name__ == '__main__':
+    import uvicorn
+    uvicorn.run(app, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))

db/deploy/requirements.txt

@@ -0,0 +1,4 @@
+fastapi
+uvicorn
+google-cloud-secret-manager
+google-cloud-logging