From dcc604ecffe8f8a4e28acb2c6899a3ffe644e691 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 15 Mar 2024 08:05:12 +1100 Subject: [PATCH] Liquibase Manual Deploy Action (#687) * liquibase manual deploy workflow * fixed lint * added cloud run files * fixed credentials typing * added exception * added cloud run config * fixed the image and updated commands * moved liquibase to Dockerfile * added workflow to trigger the schema update * added region param to workflow gcloud calls * added temp dir and cleaned code --- .github/workflows/deploy_schema_updater.yaml | 31 +++++++ .github/workflows/trigger_schema_updater.yaml | 41 ++++++++++ .gitignore | 1 + db/deploy/Dockerfile | 35 ++++++++ db/deploy/main.py | 82 +++++++++++++++++++ db/deploy/requirements.txt | 4 + 6 files changed, 194 insertions(+) create mode 100644 .github/workflows/deploy_schema_updater.yaml create mode 100644 .github/workflows/trigger_schema_updater.yaml create mode 100644 db/deploy/Dockerfile create mode 100644 db/deploy/main.py create mode 100644 db/deploy/requirements.txt diff --git a/.github/workflows/deploy_schema_updater.yaml b/.github/workflows/deploy_schema_updater.yaml new file mode 100644 index 000000000..f3f3341e5 --- /dev/null +++ b/.github/workflows/deploy_schema_updater.yaml @@ -0,0 +1,31 @@ +name: Deploy Schema Updater + +on: + workflow_dispatch: + + +jobs: + build image: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - id: "google-cloud-auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v1" + with: + workload_identity_provider: "projects/774248915715/locations/global/workloadIdentityPools/gh-deploy-pool/providers/gh-provider" + service_account: "sample-metadata-deploy@sample-metadata.iam.gserviceaccount.com" + + - id: "google-cloud-sdk-setup" + name: "Set up Cloud SDK" + uses: google-github-actions/setup-gcloud@v1 + + - name: Build image with Cloud Build + run: | + gcloud builds submit --tag gcr.io/sample-metadata/schema-updater:latest ./db/deploy + + - name: Deploy image to Cloud Run + run: | + gcloud run deploy schema-updater --image gcr.io/sample-metadata/schema-updater --platform managed --region australia-southeast1 --no-allow-unauthenticated diff --git a/.github/workflows/trigger_schema_updater.yaml b/.github/workflows/trigger_schema_updater.yaml new file mode 100644 index 000000000..4f223977c --- /dev/null +++ b/.github/workflows/trigger_schema_updater.yaml @@ -0,0 +1,41 @@ +name: Trigger Schema Updater + +on: + workflow_dispatch: + inputs: + environment: + description: 'Target environment (prod or dev)' + required: true + default: 'dev' + type: choice + options: + - prod + - dev + +jobs: + invoke-cloud-run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - id: "google-cloud-auth" + name: "Authenticate to Google Cloud" + uses: "google-github-actions/auth@v1" + with: + workload_identity_provider: "projects/774248915715/locations/global/workloadIdentityPools/gh-deploy-pool/providers/gh-provider" + service_account: "sample-metadata-deploy@sample-metadata.iam.gserviceaccount.com" + + - id: "google-cloud-sdk-setup" + name: "Set up Cloud SDK" + uses: google-github-actions/setup-gcloud@v1 + + - id: get_url + name: Get Cloud Run service URL + run: | + echo "CLOUD_RUN_URL=$(gcloud run services describe schema-updater --region australia-southeast1 --format 'value(status.url)')/execute-liquibase?environment=${{ github.event.inputs.environment }}" >> $GITHUB_ENV + + - name: Manual Trigger - Invoke Cloud Run + run: | + curl -X POST -H "Authorization: Bearer $(gcloud auth print-identity-token)" -H "Content-Type: application/xml" --data-binary "@db/project.xml" $CLOUD_RUN_URL + env: + CLOUD_RUN_URL: ${{ env.CLOUD_RUN_URL }} diff --git a/.gitignore b/.gitignore index d5fa95d5a..eb675baf5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ db/postgres*.jar .vscode/ env/ +venv/ __pycache__/ *.pyc .DS_Store diff --git a/db/deploy/Dockerfile b/db/deploy/Dockerfile new file mode 100644 index 000000000..24facb0f3 --- /dev/null +++ b/db/deploy/Dockerfile @@ -0,0 +1,35 @@ +# Use the official lightweight Python image. +FROM python:3.11-slim + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +ENV LIQUIBASE_VERSION=4.26.0 +ENV MARIADB_JDBC_VERSION=3.0.3 + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends gcc git ssh default-jdk wget unzip + +# Download and install Liquibase +RUN wget https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.zip \ + && unzip liquibase-${LIQUIBASE_VERSION}.zip -d /opt/liquibase \ + && chmod +x /opt/liquibase/liquibase \ + # Clean up to reduce layer size + && rm liquibase-${LIQUIBASE_VERSION}.zip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Download the MariaDB JDBC driver +RUN wget https://downloads.mariadb.com/Connectors/java/connector-java-${MARIADB_JDBC_VERSION}/mariadb-java-client-${MARIADB_JDBC_VERSION}.jar +RUN mv mariadb-java-client-${MARIADB_JDBC_VERSION}.jar /opt/ + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME +COPY . ./ + +# Install Python dependencies +RUN python3 -m pip install --no-cache-dir --break-system-packages -r requirements.txt + +# Run the FastAPI app on container startup +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/db/deploy/main.py b/db/deploy/main.py new file mode 100644 index 000000000..7e0f30715 --- /dev/null +++ b/db/deploy/main.py @@ -0,0 +1,82 @@ +import contextlib +import json +import os +import subprocess +import tempfile +from typing import Dict, Literal + +from fastapi import FastAPI, HTTPException, Query, Request +from google.cloud import logging, secretmanager + +app = FastAPI() + +# Setup for Google Cloud clients and logging +SECRET_CLIENT = secretmanager.SecretManagerServiceClient() +LOGGING_CLIENT = logging.Client() +SECRET_PROJECT = 'sample-metadata' +SECRET_NAME = 'liquibase-schema-updater' +log_name = 'lb_schema_update_log' +logger = LOGGING_CLIENT.logger(log_name) + +# Important to maintain this filename otherwise Liquibase fails to recognise previous migrations +changelog_file = 'project.xml' + + +def read_db_credentials(env: Literal['prod', 'dev']) -> Dict[Literal['dbname', 'username', 'password', 'host'], str]: + """Get database credentials from Secret Manager.""" + try: + secret_path = SECRET_CLIENT.secret_version_path(SECRET_PROJECT, SECRET_NAME, 'latest') + response = SECRET_CLIENT.access_secret_version(request={'name': secret_path}) + return json.loads(response.payload.data.decode('UTF-8'))[env] + except Exception as e: # Broad exception for example; refine as needed + text = f'Failed to retrieve or parse secrets: {e}' + logger.log_text(text, severity='ERROR') + raise HTTPException(status_code=500, detail=text) from e + + +@app.post('/execute-liquibase') +async def execute_liquibase(request: Request, environment: Literal['prod', 'dev'] = Query(default='dev', regex='^(prod|dev)$')): + """Endpoint to remotely trigger Liquibase commands on a GCP VM using XML content.""" + xml_content = await request.body() + + # Clean up the local temporary file + credentials = read_db_credentials(env=environment) + db_username = credentials['username'] + db_password = credentials['password'] + db_hostname = credentials['host'] + db_name = credentials['dbname'] + + # Temporary file creation with XML content + with tempfile.TemporaryDirectory() as tempdir: + # Specify the file path within the temporary directory + with contextlib.chdir(tempdir): # pylint: disable=E1101 + with open(changelog_file, 'wb') as temp_file: + temp_file.write(xml_content) + temp_file_path = temp_file.name # Store file path to use later + remote_file_path = os.path.basename(temp_file_path) + + # The actual command to run on the VM + liquibase_command = [ + '/opt/liquibase/liquibase', + f'--changeLogFile={remote_file_path}', + f'--url=jdbc:mariadb://{db_hostname}/{db_name}', + f'--driver=org.mariadb.jdbc.Driver', + f'--classpath=/opt/mariadb-java-client-3.0.3.jar', + 'update', + ] + + try: + # Execute the gcloud command + result = subprocess.run(liquibase_command, check=True, capture_output=True, text=True, env={'LIQUIBASE_COMMAND_PASSWORD': db_password, 'LIQUIBASE_COMMAND_USERNAME': db_username, **os.environ},) + logger.log_text(f'Liquibase update successful: {result.stdout}', severity='INFO') + os.remove(temp_file_path) + return {'message': 'Liquibase update executed successfully', 'output': result.stdout} + except subprocess.CalledProcessError as e: + text = f'Failed to execute Liquibase update: {e.stderr}' + logger.log_text(text, severity='ERROR') + raise HTTPException(status_code=500, detail=text) from e + + +if __name__ == '__main__': + import uvicorn + uvicorn.run(app, host='0.0.0.0', port=int(os.environ.get('PORT', 8080))) diff --git a/db/deploy/requirements.txt b/db/deploy/requirements.txt new file mode 100644 index 000000000..3850d12b8 --- /dev/null +++ b/db/deploy/requirements.txt @@ -0,0 +1,4 @@ +fastapi +uvicorn +google-cloud-secret-manager +google-cloud-logging