From 6b803dca6ed2945b3212a4b27856f2b1c0b96322 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 19:56:19 +0800 Subject: [PATCH 01/16] refactor(envs): explicitly add driver instead of removing specified driver --- backend/api/core/config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/api/core/config.py b/backend/api/core/config.py index 8fa4842..2ba0ca3 100644 --- a/backend/api/core/config.py +++ b/backend/api/core/config.py @@ -14,20 +14,24 @@ class Settings(BaseSettings): mcp_server_port: int = 8050 postgres_dsn: PostgresDsn = ( - "postgresql+psycopg://postgres:password@example.supabase.com:6543/postgres" + "postgresql://postgres:password@example.supabase.com:6543/postgres" ) @computed_field @property def orm_conn_str(self) -> str: - return self.postgres_dsn.encoded_string() + # NOTE: Explicitly follow LangGraph AsyncPostgresSaver + # and use psycopg driver for ORM + return self.postgres_dsn.encoded_string().replace( + "postgresql://", "postgresql+psycopg://" + ) @computed_field @property def checkpoint_conn_str(self) -> str: # NOTE: LangGraph AsyncPostgresSaver has some issues # with specifying psycopg driver explicitly - return self.postgres_dsn.encoded_string().replace("+psycopg", "") + return self.postgres_dsn.encoded_string() settings = Settings() From eb27c1cab72e181c630556df05d281a690d0c0fd Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 19:56:42 +0800 Subject: [PATCH 02/16] refactor(mcp): cleanup dockerfile --- backend/mcp/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/mcp/Dockerfile b/backend/mcp/Dockerfile index af1aaab..0eeeddc 100644 --- a/backend/mcp/Dockerfile +++ b/backend/mcp/Dockerfile @@ -3,10 +3,8 @@ FROM ghcr.io/astral-sh/uv:python3.13-bookworm WORKDIR /app COPY ./backend/mcp/uv.lock ./backend/mcp/pyproject.toml . RUN uv sync --frozen && rm ./uv.lock ./pyproject.toml -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl +RUN apt-get update && apt-get install -y --no-install-recommends curl COPY ./backend/mcp ./mcp COPY ./backend/shared_mcp ./shared_mcp ENV PYTHONPATH /app:$PYTHONPATH -ENV PATH /app:$PATH ENTRYPOINT ["uv", "run", "mcp/main.py"] From e9c658859d01a000d9daa19c8dda66b914a2124f Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 19:56:56 +0800 Subject: [PATCH 03/16] refactor(shared mcp): remove unused --- backend/shared_mcp/tools.py | 1 - envs/shared_mcp.env | 2 -- 2 files changed, 3 deletions(-) delete mode 100644 envs/shared_mcp.env diff --git a/backend/shared_mcp/tools.py b/backend/shared_mcp/tools.py index b082adf..b9bd389 100644 --- a/backend/shared_mcp/tools.py +++ b/backend/shared_mcp/tools.py @@ -4,7 +4,6 @@ mcp = FastMCP( "MCP Server", - host=os.environ["MCP_SERVER_HOST"], port=os.environ["MCP_SERVER_PORT"], ) diff --git a/envs/shared_mcp.env b/envs/shared_mcp.env deleted file mode 100644 index 4e2eb5e..0000000 --- a/envs/shared_mcp.env +++ /dev/null @@ -1,2 +0,0 @@ -MCP_SERVER_HOST=0.0.0.0 -MCP_SERVER_PORT=8050 From 14409b9d37a1bd2b1dcea5e94199eee8c829e407 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 19:57:42 +0800 Subject: [PATCH 04/16] feat(envs): youtube env for api key --- envs/youtube.env | 1 + 1 file changed, 1 insertion(+) create mode 100644 envs/youtube.env diff --git a/envs/youtube.env b/envs/youtube.env new file mode 100644 index 0000000..b267e8e --- /dev/null +++ b/envs/youtube.env @@ -0,0 +1 @@ +YOUTUBE_API_KEY= From e292979dfb6c8eb6aaa1d04ac10d4de7470510b6 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 19:58:53 +0800 Subject: [PATCH 05/16] build(inspector): add mcp inspector --- inspector/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 inspector/Dockerfile diff --git a/inspector/Dockerfile b/inspector/Dockerfile new file mode 100644 index 0000000..12c0cb1 --- /dev/null +++ b/inspector/Dockerfile @@ -0,0 +1,6 @@ +FROM node:22.15.0-alpine + +WORKDIR /app +RUN apk update && apk add curl && \ + npm install -g @modelcontextprotocol/inspector +ENTRYPOINT ["npx", "@modelcontextprotocol/inspector"] From 661fdcd2c227a664d467ae3128d38146aa621e4b Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 19:59:43 +0800 Subject: [PATCH 06/16] build(compose-dev): add 2 community mcp examples and inspector --- compose-dev.yaml | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/compose-dev.yaml b/compose-dev.yaml index 9bd9907..f7cc3da 100644 --- a/compose-dev.yaml +++ b/compose-dev.yaml @@ -6,7 +6,7 @@ services: dockerfile: ./backend/api/Dockerfile entrypoint: uv run fastapi run api/main.py --root-path=/api --reload env_file: - - ./envs/shared_mcp.env + - ./envs/backend.env ports: - 8000:8000 volumes: @@ -19,14 +19,41 @@ services: build: context: . dockerfile: ./backend/mcp/Dockerfile - env_file: - - ./envs/shared_mcp.env + environment: + - MCP_SERVER_PORT=${MCP_SERVER_PORT} ports: - - 8050:8050 + - ${MCP_SERVER_PORT}:${MCP_SERVER_PORT} volumes: - ./backend/mcp:/app/mcp - ./backend/shared_mcp:/app/shared_mcp + youtube: + image: youtube-mcp-server + env_file: + - ./envs/youtube.env + environment: + - YOUTUBE_MCP_SERVER_PORT=${MCP_SERVER_PORT} + ports: + - 5000:${MCP_SERVER_PORT} + + dbhub: + image: bytebase/dbhub:0.3.3 + ports: + - 8080:${MCP_SERVER_PORT} + command: > + --transport sse + --port ${MCP_SERVER_PORT} + --dsn ${POSTGRES_DSN} + + inspector: + image: inspector:prod + build: + context: . + dockerfile: ./inspector/Dockerfile + ports: + - 6274:6274 + - 6277:6277 + nginx: image: nginx:1.26.3-alpine ports: From 3289551813e7742f2843c9afc45a1736159d1a09 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 20:00:51 +0800 Subject: [PATCH 07/16] build(envs): add sample env --- .env.sample | 1 + 1 file changed, 1 insertion(+) create mode 100644 .env.sample diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..c08a239 --- /dev/null +++ b/.env.sample @@ -0,0 +1 @@ +MCP_SERVER_PORT=8050 From 5ee9c66872aecc72427a812bec731d16bf32cb9a Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 20:24:00 +0800 Subject: [PATCH 08/16] docs(envs): add comments --- .env.sample | 1 + envs/backend.env | 1 + 2 files changed, 2 insertions(+) diff --git a/.env.sample b/.env.sample index c08a239..f37173d 100644 --- a/.env.sample +++ b/.env.sample @@ -1 +1,2 @@ +# change if required MCP_SERVER_PORT=8050 diff --git a/envs/backend.env b/envs/backend.env index c439a50..4fa624a 100644 --- a/envs/backend.env +++ b/envs/backend.env @@ -1,2 +1,3 @@ OPENAI_API_KEY= +# do not specify driver (do not specify `+psycopg`) POSTGRES_DSN= From 9034d41d702924781ca1e4c3b5a8489ca4c72743 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 20:42:42 +0800 Subject: [PATCH 09/16] docs(readme): add guide --- README.md | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/mcp.md | 0 docs/supabase.md | 0 3 files changed, 79 insertions(+) create mode 100644 docs/mcp.md create mode 100644 docs/supabase.md diff --git a/README.md b/README.md index 3779476..9f340c4 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,11 @@ - [Core Features](#core-features) - [Technology Stack and Features](#technology-stack-and-features) - [Planned Features](#planned-features) + - [Getting Started](#getting-started) + - [Development](#development) + - [VSCode Devcontainer](#vscode-devcontainer) + - [Without VSCode Devcontainer](#without-vscode-devcontainer) + - [Refactored Markdown Files](#refactored-markdown-files) ## Core Features @@ -39,3 +44,77 @@ - :dollar: Deploy live demo to [![Fargate](https://img.shields.io/badge/Fargate-white.svg?logo=awsfargate)](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/AWS_Fargate.html) - Provision with [![Terraform](https://img.shields.io/github/stars/hashicorp/terraform?logo=terraform&label=Terraform)](https://github.com/hashicorp/terraform) IaC - Push built images to ECR and Dockerhub + +## Getting Started + +Build images with: + +```bash +docker compose -f compose-dev.yaml build +``` + +Copy environment file: + +```bash +cp .env.sample .env +``` + +Add your following API keys and value to the respective file: `./envs/backend.env`, `./envs/youtube.env` and `.env`. + +```bash +OPENAI_API_KEY=sk-proj-... +POSTGRES_DSN=postgresql://postgres... +YOUTUBE_API_KEY=... +``` + +Set environment variables in shell: (compatible with `bash` and `zsh`) + +```bash +set -a; for env_file in ./envs/*; do source $env_file; done; set +a +``` + +Start production containers: + +```bash +docker compose up -d +``` + +## Development + +### VSCode Devcontainer + +> [!WARNING] +> Only replace the following if you plan to start debugger for FastAPI server in VSCode. + +Replace `./compose-dev.yaml` entrypoint to allow debugging FastAPI server: + +```yaml +# ... + api: + # ... + # entrypoint: uv run fastapi run api/main.py --root-path=/api --reload + # replace above with: + entrypoint: bash -c "sleep infinity" + # ... +``` + +```bash +code --no-sandbox . +``` + +Press `F1` and type `Dev Containers: Rebuild and Reopen in Container` to open containerized environment with IntelliSense and Debugger for FastAPI. + +### Without VSCode Devcontainer + +Run development environment with: + +```bash +docker compose -f compose-dev.yaml up -d +``` + +## Refactored Markdown Files + +The following markdown files provide additional details on other features: + +- [`./docs/mcp.md`](./docs/mcp.md) +- [`./docs/supabase.md`](./docs/supabase.md) diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/supabase.md b/docs/supabase.md new file mode 100644 index 0000000..e69de29 From aef569adc87dc516a85d96a33fe447ebfb5fa3ae Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 20:48:15 +0800 Subject: [PATCH 10/16] fix(vscode)!: fix debugger for fastapi behind nginx proxy --- backend/api/.vscode/launch.json | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/api/.vscode/launch.json b/backend/api/.vscode/launch.json index e756e17..89d6854 100644 --- a/backend/api/.vscode/launch.json +++ b/backend/api/.vscode/launch.json @@ -12,6 +12,7 @@ "args": [ "run", "api/main.py", + "--root-path=/api", "--reload" ] } From d6955327be50fd7883f977caa2ac8f72099dda11 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 21:28:01 +0800 Subject: [PATCH 11/16] refactor(unused): remove unused --- compose-dev.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/compose-dev.yaml b/compose-dev.yaml index f7cc3da..3c67bec 100644 --- a/compose-dev.yaml +++ b/compose-dev.yaml @@ -2,7 +2,6 @@ services: api: image: api:prod build: - context: . dockerfile: ./backend/api/Dockerfile entrypoint: uv run fastapi run api/main.py --root-path=/api --reload env_file: @@ -17,7 +16,6 @@ services: mcp: image: mcp:prod build: - context: . dockerfile: ./backend/mcp/Dockerfile environment: - MCP_SERVER_PORT=${MCP_SERVER_PORT} @@ -48,7 +46,6 @@ services: inspector: image: inspector:prod build: - context: . dockerfile: ./inspector/Dockerfile ports: - 6274:6274 From 663460359ea12b86da20fe2a308de063dd0b8f46 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 21:38:11 +0800 Subject: [PATCH 12/16] feat(youtube builder): add builder and guide --- README.md | 13 +++++++++++-- community/youtube/build.sh | 11 +++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100755 community/youtube/build.sh diff --git a/README.md b/README.md index 9f340c4..ff48182 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,16 @@ ## Getting Started -Build images with: +Build community youtube MCP image with: + +```bash +./community/youtube/build.sh +``` + +> [!TIP] +> Instead of cloning or submoduling the repository locally, then building the image, this script builds the Docker image inside a temporary Docker-in-Docker container. This approach avoids polluting your local environment with throwaway files by cleaning up everything once the container exits. + +Then build the other images with: ```bash docker compose -f compose-dev.yaml build @@ -83,7 +92,7 @@ docker compose up -d ### VSCode Devcontainer -> [!WARNING] +> [!WARNING] > Only replace the following if you plan to start debugger for FastAPI server in VSCode. Replace `./compose-dev.yaml` entrypoint to allow debugging FastAPI server: diff --git a/community/youtube/build.sh b/community/youtube/build.sh new file mode 100755 index 0000000..721e418 --- /dev/null +++ b/community/youtube/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +docker run --rm -it --entrypoint sh \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --workdir /app \ + docker:dind \ + -c " + git clone https://github.com/Klavis-AI/klavis.git . && \ + touch mcp_servers/youtube/.env && \ + docker build -t youtube-mcp-server -f mcp_servers/youtube/Dockerfile . + " From b0d75dbb82d19446abe36a1ea9048a6f6a98791c Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 21:58:47 +0800 Subject: [PATCH 13/16] ci(actions): build community image --- .github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b3595d9..957c88a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,4 +18,5 @@ jobs: - name: Build production and development docker images run: | + ./community/youtube/build.sh docker compose build From 64e36ea6ed248a132322426370f24b8a9e8ae410 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 22:01:11 +0800 Subject: [PATCH 14/16] ci(secrets): add secrets --- .github/workflows/build.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 957c88a..6e22a69 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,6 +7,11 @@ on: jobs: build: runs-on: ubuntu-latest + env: + MCP_SERVER_PORT: ${{ secrets.MCP_SERVER_PORT }} + YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + POSTGRES_DSN: ${{ secrets.POSTGRES_DSN }} steps: - uses: actions/checkout@v4.2.2 From 111e3e9067cc66e994214e6786c30ad5e49e4dc8 Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 22:08:00 +0800 Subject: [PATCH 15/16] ci(workflow): change order and use secrets --- .github/workflows/build.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6e22a69..519c08a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,12 +16,17 @@ jobs: steps: - uses: actions/checkout@v4.2.2 - - name: Run docker compose - uses: hoverkraft-tech/compose-action@v2.0.1 - with: - compose-file: "compose-dev.yaml" - - name: Build production and development docker images run: | ./community/youtube/build.sh docker compose build + + - name: Run docker compose + uses: hoverkraft-tech/compose-action@v2.0.1 + with: + compose-file: "compose-dev.yaml" + env: + MCP_SERVER_PORT: ${{ secrets.MCP_SERVER_PORT }} + YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + POSTGRES_DSN: ${{ secrets.POSTGRES_DSN }} From 8b6c8087ce6fc315b62a9780a90fd3816d709abb Mon Sep 17 00:00:00 2001 From: NicholasGoh Date: Thu, 24 Apr 2025 22:17:26 +0800 Subject: [PATCH 16/16] ci(github actions): add docker in docker --- .github/workflows/build.yaml | 6 ++++++ community/youtube/build.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 519c08a..3f46dab 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,6 +7,12 @@ on: jobs: build: runs-on: ubuntu-latest + services: + docker: + image: docker:dind + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MCP_SERVER_PORT: ${{ secrets.MCP_SERVER_PORT }} YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} diff --git a/community/youtube/build.sh b/community/youtube/build.sh index 721e418..953bc66 100755 --- a/community/youtube/build.sh +++ b/community/youtube/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -docker run --rm -it --entrypoint sh \ +docker run --rm --entrypoint sh \ --volume /var/run/docker.sock:/var/run/docker.sock \ --workdir /app \ docker:dind \