From b69b18c94ba231a4cf79e0e45a9f7cde4654a403 Mon Sep 17 00:00:00 2001 From: Eugene P <144219719+EugeneLightsOn@users.noreply.github.com> Date: Wed, 3 Jul 2024 18:34:51 +0200 Subject: [PATCH] Automated migrations workflow(optionally SSH, Database URL or API call) (#211) * Migrations Git Workflow + protect migrate API endpoint * Migrations Git Workflow + protect migrate endpoint * Migrations Git Workflow + protect migrate endpoint * Migrations Git Workflow + protect migrate endpoint * Migrations Git Workflow + protect migrate endpoint * Migrations Git Workflow + protect migrate endpoint black --- .env-template | 4 ++ .github/scripts/run_docker_migration.sh | 5 ++ .github/scripts/run_src_migration.sh | 6 ++ .github/workflows/run_alembic_migrations.yml | 64 ++++++++++++++++++++ README.md | 1 + docs/github_migrations_action.md | 53 ++++++++++++++++ src/backend/config/auth.py | 22 +++++++ src/backend/main.py | 12 ++-- 8 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 .github/scripts/run_docker_migration.sh create mode 100644 .github/scripts/run_src_migration.sh create mode 100644 .github/workflows/run_alembic_migrations.yml create mode 100644 docs/github_migrations_action.md diff --git a/.env-template b/.env-template index 3ef548cd27..3902856752 100644 --- a/.env-template +++ b/.env-template @@ -2,6 +2,10 @@ NEXT_PUBLIC_API_HOSTNAME=http://localhost:8000 DATABASE_URL=postgresql+psycopg2://postgres:postgres@db:5432 +# Migrate access token, used to authenticate requests to the migrate endpoint. +# You can generate it using some random string generator. +MIGRATE_TOKEN= + # TOOLS PYTHON_INTERPRETER_URL=http://terrarium:8080 TAVILY_API_KEY= diff --git a/.github/scripts/run_docker_migration.sh b/.github/scripts/run_docker_migration.sh new file mode 100644 index 0000000000..beedc57fed --- /dev/null +++ b/.github/scripts/run_docker_migration.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd $SSH_PROJECT_DIR +git pull +make migrate +echo "Migration complete" diff --git a/.github/scripts/run_src_migration.sh b/.github/scripts/run_src_migration.sh new file mode 100644 index 0000000000..9f17541cf1 --- /dev/null +++ b/.github/scripts/run_src_migration.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd $SSH_PROJECT_DIR +git pull +source .venv/bin/activate +alembic -c src/backend/alembic.ini upgrade head +echo "Migration complete" diff --git a/.github/workflows/run_alembic_migrations.yml b/.github/workflows/run_alembic_migrations.yml new file mode 100644 index 0000000000..353004bbe0 --- /dev/null +++ b/.github/workflows/run_alembic_migrations.yml @@ -0,0 +1,64 @@ +name: Automated Migrations +# This workflow provides a way to run Alembic migrations using SSH or migration API or database URL. +on: + push: + branches: [ main ] +jobs: + run-alembic-migrations-ssh: + runs-on: ubuntu-latest + environment: production + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_HOST: ${{secrets.SSH_HOST}} + SSH_PORT: ${{secrets.SSH_PORT}} + SSH_USER_NAME: ${{secrets.SSH_USER_NAME}} + DATABASE_URL: ${{secrets.MIGRATION_DATABASE_URL}} + MIGRATION_API_ENDPOINT: ${{secrets.MIGRATION_API_ENDPOINT}} + MIGRATION_API_TOKEN: ${{secrets.MIGRATION_API_TOKEN}} + # If migration API params are present, use backend API to run migrations. + USE_MIGRATION_API: ${{ (secrets.MIGRATION_API_ENDPOINT != '') && (secrets.MIGRATION_API_TOKEN != '') }} + # If database URL is present, run migrations on DB connection. + # Please note that the database connection should be open to the IP from where the migrations are being run. + # If you are using a private network, you can set up SSH tunnel to run migrations. + USE_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL != '' }} + # If SSH params are present, use SSH to run migrations. + USE_SSH: ${{ (secrets.SSH_PRIVATE_KEY != '') && (secrets.SSH_HOST != '') && (secrets.SSH_USER_NAME != '') && (secrets.SSH_PORT != '')}} + + steps: + - name: Run Alembic Migrations SSH + if: ${{ (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'false') }} + run: | + echo "Running migrations using SSH" + echo "$SSH_PRIVATE_KEY" > private_key && chmod 600 private_key + ssh -o StrictHostKeyChecking=no -i private_key ${SSH_USER_NAME}@${SSH_HOST} -p ${SSH_PORT} ' + export SSH_PROJECT_DIR=~/cohere-toolkit && # Set the project directory here + sh $SSH_PROJECT_DIR/.github/scripts/run_docker_migration.sh && # Delete this command if your toolkit is not in a docker container + sh $SSH_PROJECT_DIR/.github/scripts/run_src_migration.sh # Delete this command if your toolkit is in a docker container + ' + - name: Run Alembic Migrations API + if: ${{ (env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'false') }} + run: | + echo "Running migrations using the migration API" + curl -H "Authorization: Bearer $MIGRATION_API_TOKEN" -X POST $MIGRATION_API_ENDPOINT + - name: Checkout repository + if: ${{(env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'true') || (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'true') }} + uses: actions/checkout@v4 + - name: Set up Python 3.11 + if: ${{(env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'true') || (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'true') }} + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Run Alembic Migrations Database + if: ${{(env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'true') || (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'true') }} + run: | + echo "Running migrations using the database URL" + pip install poetry==1.7.1 + export POETRY_VIRTUALENVS_IN_PROJECT=true + poetry install + echo "DATABASE_URL=$DATABASE_URL" > .env + source .venv/bin/activate + alembic -c src/backend/alembic.ini upgrade head + - name: Fail Migration + if: ${{ (env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'false') }} + run: | + echo "Migration failed. Please provide either SSH, Migration API or Database URL environment variables. See the README for more information." diff --git a/README.md b/README.md index c026d30d9e..db95953b4e 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Toolkit is a collection of prebuilt components enabling users to quickly build a - [How to add tools](/docs/custom_tool_guides/tool_guide.md) - [How to add authentication](/docs/auth_guide.md) - [How to deploy toolkit services](/docs/service_deployments.md) + - [How to set up Github Actions for automated DB migrations](/docs/github_migrations_action.md) - [How to customize the theme](/docs/theming.md) - [How to contribute](#contributing) - [Try Cohere's Command Showcase](https://coral.cohere.com/) diff --git a/docs/github_migrations_action.md b/docs/github_migrations_action.md new file mode 100644 index 0000000000..84a86d26cc --- /dev/null +++ b/docs/github_migrations_action.md @@ -0,0 +1,53 @@ +# Github Actions for Automated DB Migrations + +This guide will help you set up a Github Action to automatically run database migrations when you push changes to your repository. + +## Prerequisites +To set up the Github Action, you will need to have the following: + - access with admin permissions to the repository you want to set up the action for + - a database that you want to run migrations on + +## Steps +This guide will walk you through the steps to set up the Github Action for automated database migrations. +We will provide several types of migrations execution, including: + - running migrations using SSH access to a remote server + - running migrations on a remote database using connection strings + - running migrations on a remote database using Cohere Toolkit's API migrations endpoint +For each type of migration execution we need to setup Github environment and secrets. +To seting up the Github environment and secrets, follow the steps below: +- navigate to your repository on Github +- click on the `Settings` tab +- click on the `Environments` menu on the left +- click on the `New environment` button or select an existing environment(we will use the `Production` environment in this guide) +- fill in the environment name and click on the `Configure environment` button +- optionally set rules for the environment, for example, you can set the `Production` environment to be available only for the `main` branch +- click on the `Add secret` button + +### Running Migrations using SSH access to a remote server +To run migrations using SSH access to a remote server, you will need to set up the following secrets +- `SSH_PRIVATE_KEY` - the private key to use for SSH authentication +- `SSH_HOST` - the hostname or IP address of the remote server +- `SSH_PORT` - the port to use for SSH authentication +- `SSH_USER_NAME` - the username to use for SSH authentication +The sample Github Action workflow file is located at `.github/workflows/run_alembic_migrations.yml` +Set the `SSH_PROJECT_DIR` variable to the path to the project directory on the remote server. See comments in the `run_alembic_migrations.yml` file for more details. +The action scripts are located at `.github/scripts` directory. The `run_docker_migration.sh` script is used +to run the migrations using Docker. The `run_src_migration.sh` script is used to run the migrations using the source code. +If the Toolkit is deployed on a remote server using Docker, please remove `sh $SSH_PROJECT_DIR/.github/scripts/run_src_migration.sh` string from the `run_alembic_migrations.yml` +If the Toolkit is deployed on a remote server using the source code, please remove `sh $SSH_PROJECT_DIR/.github/scripts/run_docker_migration.sh` string from the `run_alembic_migrations.yml` +Please note that this is a basic example and you may need to adjust the scripts to fit your specific use case. + +### Running Migrations on a remote database using connection strings +To run migrations on a remote database using connection strings, you will need to set up the following secrets +In this example, we will use a PostgreSQL database available from the internet. If you are using a private database, you may need to set up a VPN or SSH tunnel to access the database. +- `MIGRATION_DATABASE_URL` - the connection string to the database. It should be in the format `postgresql+psycopg2://username:password@host:port/database` eg. `postgresql+psycopg2://postgres:postgres@db:5432` + +### Running Migrations on a remote database using Cohere Toolkit's API migrations endpoint +To run migrations on a remote database using Cohere Toolkit's API migrations endpoint, you will need to set up the following secrets +- `MIGRATION_API_ENDPOINT` - the URL of the Cohere Toolkit API migrations endpoint, eg. `https://>Your Cohere Toolkit host name<:>API port, 8000 by default bool: """ diff --git a/src/backend/main.py b/src/backend/main.py index 3146185e03..b969626cbc 100644 --- a/src/backend/main.py +++ b/src/backend/main.py @@ -5,11 +5,15 @@ from alembic.command import upgrade from alembic.config import Config from dotenv import load_dotenv -from fastapi import FastAPI, HTTPException +from fastapi import Depends, FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware from starlette.middleware.sessions import SessionMiddleware -from backend.config.auth import get_auth_strategy_endpoints, is_authentication_enabled +from backend.config.auth import ( + get_auth_strategy_endpoints, + is_authentication_enabled, + verify_migrate_token, +) from backend.config.routers import ROUTER_DEPENDENCIES from backend.routers.agent import router as agent_router from backend.routers.auth import router as auth_router @@ -95,7 +99,7 @@ async def health(): return {"status": "OK"} -@app.post("/migrate") +@app.post("/migrate", dependencies=[Depends(verify_migrate_token)]) async def apply_migrations(): """ Applies Alembic migrations - useful for serverless applications @@ -108,4 +112,4 @@ async def apply_migrations(): status_code=500, detail=f"Error while applying Alembic migrations: {str(e)}" ) - return {"status": "Done"} + return {"status": "Migration successful"}