Automated migrations workflow(optionally SSH, Database URL or API cal…

…l) (#211)

* Migrations Git Workflow + protect migrate API endpoint

* Migrations Git Workflow + protect migrate endpoint

* Migrations Git Workflow + protect migrate endpoint

* Migrations Git Workflow + protect migrate endpoint

* Migrations Git Workflow + protect migrate endpoint

* Migrations Git Workflow + protect migrate endpoint black

.env-template

@@ -2,6 +2,10 @@
 NEXT_PUBLIC_API_HOSTNAME=http://localhost:8000
 DATABASE_URL=postgresql+psycopg2://postgres:postgres@db:5432
 
+# Migrate access token, used to authenticate requests to the migrate endpoint.
+# You can generate it using some random string generator.
+MIGRATE_TOKEN=<MIGRATE_TOKEN>
+
 # TOOLS
 PYTHON_INTERPRETER_URL=http://terrarium:8080
 TAVILY_API_KEY=<API_KEY_HERE>

.github/scripts/run_docker_migration.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+cd $SSH_PROJECT_DIR
+git pull
+make migrate
+echo "Migration complete"

.github/scripts/run_src_migration.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd $SSH_PROJECT_DIR
+git pull
+source .venv/bin/activate
+alembic -c src/backend/alembic.ini upgrade head
+echo "Migration complete"

.github/workflows/run_alembic_migrations.yml

@@ -0,0 +1,64 @@
+name: Automated Migrations
+# This workflow provides a way to run Alembic migrations using SSH or migration API or database URL.
+on:
+  push:
+    branches: [ main ]
+jobs:
+  run-alembic-migrations-ssh:
+    runs-on: ubuntu-latest
+    environment: production
+    env:
+        SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+        SSH_HOST: ${{secrets.SSH_HOST}}
+        SSH_PORT: ${{secrets.SSH_PORT}}
+        SSH_USER_NAME: ${{secrets.SSH_USER_NAME}}
+        DATABASE_URL: ${{secrets.MIGRATION_DATABASE_URL}}
+        MIGRATION_API_ENDPOINT: ${{secrets.MIGRATION_API_ENDPOINT}}
+        MIGRATION_API_TOKEN: ${{secrets.MIGRATION_API_TOKEN}}
+        # If migration API params are present, use backend API to run migrations.
+        USE_MIGRATION_API: ${{ (secrets.MIGRATION_API_ENDPOINT != '') && (secrets.MIGRATION_API_TOKEN != '') }}
+        # If database URL is present, run migrations on DB connection.
+        # Please note that the database connection should be open to the IP from where the migrations are being run.
+        # If you are using a private network, you can set up SSH tunnel to run migrations.
+        USE_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL != '' }}
+        # If SSH params are present, use SSH to run migrations.
+        USE_SSH: ${{ (secrets.SSH_PRIVATE_KEY != '') && (secrets.SSH_HOST != '') && (secrets.SSH_USER_NAME != '') && (secrets.SSH_PORT != '')}}
+
+    steps:
+        - name: Run Alembic Migrations SSH
+          if: ${{ (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'false') }}
+          run: |
+            echo "Running migrations using SSH"
+            echo "$SSH_PRIVATE_KEY" > private_key && chmod 600 private_key
+            ssh -o StrictHostKeyChecking=no -i private_key ${SSH_USER_NAME}@${SSH_HOST} -p ${SSH_PORT} '
+                export SSH_PROJECT_DIR=~/cohere-toolkit && # Set the project directory here
+                sh $SSH_PROJECT_DIR/.github/scripts/run_docker_migration.sh && # Delete this command if your toolkit is not in a docker container
+                sh $SSH_PROJECT_DIR/.github/scripts/run_src_migration.sh # Delete this command if your toolkit is in a docker container
+            '
+        - name: Run Alembic Migrations API
+          if: ${{ (env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'false') }}
+          run: |
+            echo "Running migrations using the migration API"
+            curl -H "Authorization: Bearer $MIGRATION_API_TOKEN" -X POST $MIGRATION_API_ENDPOINT
+        - name: Checkout repository
+          if: ${{(env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'true')  || (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'true') }}
+          uses: actions/checkout@v4
+        - name: Set up Python 3.11
+          if: ${{(env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'true')  || (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'true') }}
+          uses: actions/setup-python@v5
+          with:
+            python-version: "3.11"
+        - name: Run Alembic Migrations Database
+          if: ${{(env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'true')  || (env.USE_SSH == 'true' && env.USE_MIGRATION_API == 'true' && env.USE_DATABASE_URL == 'true') }}
+          run: |
+            echo "Running migrations using the database URL"
+            pip install poetry==1.7.1
+            export POETRY_VIRTUALENVS_IN_PROJECT=true
+            poetry install
+            echo "DATABASE_URL=$DATABASE_URL" > .env
+            source .venv/bin/activate
+            alembic -c src/backend/alembic.ini upgrade head
+        - name: Fail Migration
+          if: ${{ (env.USE_SSH == 'false' && env.USE_MIGRATION_API == 'false' && env.USE_DATABASE_URL == 'false') }}
+          run: |
+            echo "Migration failed. Please provide either SSH, Migration API or Database URL environment variables. See the README for more information."

README.md

@@ -13,6 +13,7 @@ Toolkit is a collection of prebuilt components enabling users to quickly build a
   - [How to add tools](/docs/custom_tool_guides/tool_guide.md)
   - [How to add authentication](/docs/auth_guide.md)
   - [How to deploy toolkit services](/docs/service_deployments.md)
+  - [How to set up Github Actions for automated DB migrations](/docs/github_migrations_action.md)
   - [How to customize the theme](/docs/theming.md)
   - [How to contribute](#contributing)
 - [Try Cohere's Command Showcase](https://coral.cohere.com/)

docs/github_migrations_action.md

@@ -0,0 +1,53 @@
+# Github Actions for Automated DB Migrations
+
+This guide will help you set up a Github Action to automatically run database migrations when you push changes to your repository.
+
+## Prerequisites
+To set up the Github Action, you will need to have the following:
+    - access with admin permissions to the repository you want to set up the action for
+    - a database that you want to run migrations on
+
+## Steps
+This guide will walk you through the steps to set up the Github Action for automated database migrations.
+We will provide several types of migrations execution, including:
+    - running migrations using SSH access to a remote server
+    - running migrations on a remote database using connection strings
+    - running migrations on a remote database using Cohere Toolkit's API migrations endpoint
+For each type of migration execution we need to setup Github environment and secrets.
+To seting up the Github environment and secrets, follow the steps below:
+- navigate to your repository on Github
+- click on the `Settings` tab
+- click on the `Environments` menu on the left
+- click on the `New environment` button or select an existing environment(we will use the `Production` environment in this guide)
+- fill in the environment name and click on the `Configure environment` button
+- optionally set rules for the environment, for example, you can set the `Production` environment to be available only for the `main` branch
+- click on the `Add secret` button
+
+### Running Migrations using SSH access to a remote server
+To run migrations using SSH access to a remote server, you will need to set up the following secrets
+- `SSH_PRIVATE_KEY` - the private key to use for SSH authentication
+- `SSH_HOST` - the hostname or IP address of the remote server
+- `SSH_PORT` - the port to use for SSH authentication
+- `SSH_USER_NAME` - the username to use for SSH authentication
+The sample Github Action workflow file is located at `.github/workflows/run_alembic_migrations.yml`
+Set the `SSH_PROJECT_DIR` variable to the path to the project directory on the remote server. See comments in the `run_alembic_migrations.yml` file for more details.
+The action scripts are located at `.github/scripts` directory. The `run_docker_migration.sh` script is used
+to run the migrations using Docker. The `run_src_migration.sh` script is used to run the migrations using the source code.
+If the Toolkit is deployed on a remote server using Docker, please remove `sh $SSH_PROJECT_DIR/.github/scripts/run_src_migration.sh` string from the `run_alembic_migrations.yml`
+If the Toolkit is deployed on a remote server using the source code, please remove `sh $SSH_PROJECT_DIR/.github/scripts/run_docker_migration.sh` string from the `run_alembic_migrations.yml`
+Please note that this is a basic example and you may need to adjust the scripts to fit your specific use case.
+
+### Running Migrations on a remote database using connection strings
+To run migrations on a remote database using connection strings, you will need to set up the following secrets
+In this example, we will use a PostgreSQL database available from the internet. If you are using a private database, you may need to set up a VPN or SSH tunnel to access the database.
+- `MIGRATION_DATABASE_URL` - the connection string to the database. It should be in the format `postgresql+psycopg2://username:password@host:port/database` eg. `postgresql+psycopg2://postgres:postgres@db:5432`
+
+### Running Migrations on a remote database using Cohere Toolkit's API migrations endpoint
+To run migrations on a remote database using Cohere Toolkit's API migrations endpoint, you will need to set up the following secrets
+- `MIGRATION_API_ENDPOINT` - the URL of the Cohere Toolkit API migrations endpoint, eg. `https://>Your Cohere Toolkit host name<:>API port, 8000 by default</migrate`
+- `MIGRATION_API_TOKEN` - the API token to use for authentication with the Cohere Toolkit API migrations endpoint. You can get the API token from the Toolkit's `.env` file
+
+Please note that this is a basic example and you may need to adjust the scripts to fit your specific use case.
+If all the secrets above are set up, the remote database migrations type will used as default.
+If some variables are not set, the migration type will be selected based on the set variables.
+If script cant select the migration type(some variables are not set), the script will exit with an error message.

src/backend/config/auth.py

@@ -1,5 +1,13 @@
+import os
+
+from dotenv import load_dotenv
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
 from backend.services.auth import BasicAuthentication, GoogleOAuth, OpenIDConnect
 
+load_dotenv()
+
 # Add Auth strategy classes here to enable them
 # Ex: [BasicAuthentication]
 ENABLED_AUTH_STRATEGIES = []
@@ -9,6 +17,20 @@
 # Ex: {"Basic": BasicAuthentication()}
 ENABLED_AUTH_STRATEGY_MAPPING = {cls.NAME: cls() for cls in ENABLED_AUTH_STRATEGIES}
 
+# Token to authorize migration requests
+MIGRATE_TOKEN = os.environ.get("MIGRATE_TOKEN", None)
+
+security = HTTPBearer()
+
+
+def verify_migrate_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    if not MIGRATE_TOKEN or credentials.credentials != MIGRATE_TOKEN:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or missing token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
 
 def is_authentication_enabled() -> bool:
     """

src/backend/main.py

@@ -5,11 +5,15 @@
 from alembic.command import upgrade
 from alembic.config import Config
 from dotenv import load_dotenv
-from fastapi import FastAPI, HTTPException
+from fastapi import Depends, FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from starlette.middleware.sessions import SessionMiddleware
 
-from backend.config.auth import get_auth_strategy_endpoints, is_authentication_enabled
+from backend.config.auth import (
+    get_auth_strategy_endpoints,
+    is_authentication_enabled,
+    verify_migrate_token,
+)
 from backend.config.routers import ROUTER_DEPENDENCIES
 from backend.routers.agent import router as agent_router
 from backend.routers.auth import router as auth_router
@@ -95,7 +99,7 @@ async def health():
     return {"status": "OK"}
 
 
-@app.post("/migrate")
+@app.post("/migrate", dependencies=[Depends(verify_migrate_token)])
 async def apply_migrations():
     """
     Applies Alembic migrations - useful for serverless applications
@@ -108,4 +112,4 @@ async def apply_migrations():
             status_code=500, detail=f"Error while applying Alembic migrations: {str(e)}"
         )
 
-    return {"status": "Done"}
+    return {"status": "Migration successful"}