Supabase Self-Host Implementation

.env.template

@@ -1,50 +1,40 @@
-# ALL SERVICES ARE CONFIGURED HERE. 
-# You can use these defaults, or BYO services to fit your needs.
-# The defaults should work 'out of the box' without any changes.
-# ⚠️ For Security, we recommend changing all variables marked with CHANGE ME.
+# OpenAI API key, required and please add one
+OPENAI_API_KEY=YOUR_OPENAI_API_KEY_HERE
 
-# OpenAI key is REQUIRED for Embeddings during ingest & RAG retrieval
-OPENAI_API_KEY=  # ⚠️ CHANGE ME
+# Main backend Flask app
+FLASK_PORT=3012
 
-# Qdrant Vector DB
-QDRANT_URL=qdrant:6333 # container name
-QDRANT_COLLECTION_NAME=uiuc-chat	
-QDRANT_API_KEY=your-strong-key-here # ⚠️ CHANGE ME
+# Ingest queue state is managed by Redis
+INGEST_REDIS_HOST=redis # container name
+INGEST_REDIS_PASSWORD=your-strong-password-here # ⚠️ CHANGE ME
+INGEST_REDIS_PORT=6379
 
 # Object Storage: You can use either Minio or S3. Choose one, not both. Minio is used by default.
-MINIO_URL=http://minio:9001 # You MUST comment this out if using AWS S3.
-AWS_ACCESS_KEY_ID=minioadmin # ⚠️ CHANGE ME
-AWS_SECRET_ACCESS_KEY=minioadmin # ⚠️ CHANGE ME
-S3_BUCKET_NAME=uiuc-chat
-
-# For SQL, you have 3 options: Supabase, Postgres, or SQLite. Choose only one. 
-# Self-hosted Supabase is used by default, see here for more info on self hosted Supabase: https://supabase.com/docs/guides/self-hosting
-
-# DEFAULT SQL: Supabase
-# TODO: FIGURE OUT THE DEFAULTS HERE
-# SUPABASE_USER=
-# SUPABASE_PASSWORD=
-# SUPABASE_URL=
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin
+DOCKER_INTERNAL_MINIO_API_PORT=10000
+DOCKER_INTERNAL_MINIO_DASHBOARD_PORT=10001
+PUBLIC_MINIO_API_PORT=3013
+PUBLIC_MINIO_DASHBOARD_PORT=3014
+MINIO_URL=http://minio:${DOCKER_INTERNAL_MINIO_API_PORT}
 
-# Or, bring your own Postgres (from outside this repo):
-# POSTGRES_USER=
-# POSTGRES_PASSWORD=
-# POSTGRES_URL=
+# Qdrant Vector DB
+# QDRANT_API_KEY is set in the qdrant_config.yaml
+QDRANT_COLLECTION_NAME=uiuc-chat
+QDRANT_URL=http://qdrant:6333
+S3_BUCKET_NAME=uiuc-chat
 
-# Or, use SQLite. I'm a huge fan of SQLite:
-# SQLITE_DB_NAME=uiuc-chat-backend-sqlite.db
+# Supabase related, should match the values in the Supabase .env file
+POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password  # Must match Supabase's password
+POSTGRES_HOST=supabase-db
+POSTGRES_PORT=5432
+POSTGRES_DB=postgres
+POSTGRES_USER=postgres
 
-# Ingest queue state is managed by Redis
-INGEST_REDIS_HOST=redis # container name
-INGEST_REDIS_PORT=6379
-INGEST_REDIS_PASSWORD=your-strong-password-here # ⚠️ CHANGE ME
-
-# Main backend Flask app
-FLASK_PORT=8000
+# Application variables 
+S3_BUCKET_NAME=uiuc-chat
+OPENAI_API_KEY=YOUR_KEY_HERE # replace with your OpenAI API key
 
-# Optional services. Adds functionality if you want it, but not necessary.	
-# NOMIC_API_KEY=	
-# POSTHOG_API_KEY=	
-# SENTRY_DSN=	
-# EMAIL_SENDER=	
-# N8N_URL=	
+# Optional
+# POSTHOG_API_KEY=OPTIONAL
+# NOMIC_API_KEY=OPTIONAL

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "supabase"]
+	path = supabase
+	url = https://github.com/supabase/supabase
+[submodule "ic_crawlee"]
+	path = ic_crawlee
+	url = https://github.com/UIUC-Chatbot/crawlee.git

README.md

@@ -19,12 +19,26 @@ Failure to obtain a commercial license for commercial use is a violation of the
 
 ## Docker Deployment
 
+### Supabase
+
+1. Duplicate `.env.example` from `supabase/docker/.env.example` and rename it to `.env`. (example: cp ./supabase/docker/.env.example ./supabase/docker/.env)
+2. Customize your env variables as needed in the supabase docker
+
+### Self-host docker
+
 1. Duplicate `.env.template` and rename it to `.env`. E.g. `cp .env.template .env`
 2. Customize your env variables. Your vector database can be either Qdrant and Pinecone. The SQL database can be any of SQLite, Postgres, and Supabase. The object storage can be Minio or AWS S3. 
-3. Run Docker Compose `docker compose up --build`
-4. Navitage to `localhost:8000` (or whatever your `$FLASK_PORT` is)
 
-To customize HTTP port used as the main entrypoint, set the `FLASK_PORT` variabel in your `.env`. It defaults to 8000.
+### Running simultaneously
+
+We've created an `init.sh` file to run both docker-compose files with run command. To do this, first initialize the `init.sh` with right permission.
+
+```bash
+chmod +x init.sh
+./init.sh #runs the script
+```
+
+To customize HTTP port used as the main entrypoint, set the `FLASK_PORT` variabel in your `.env`. It defaults to 8188.
 
 Works on version: `Docker Compose version v2.27.1-desktop.1`
 
@@ -43,7 +57,7 @@ See docs on https://docs.uiuc.chat
 
 1. Rename `.env.template` to `.env` and fill in the required variables
 2. Install Python requirements `pip install -r requirements.txt`
-3. Start the server for development (with live reloads) `cd ai_ta_backend` then `flask --app ai_ta_backend.main:app --debug run --port 8000`
+3. Start the server for development (with live reloads) `cd ai_ta_backend` then `flask --app ai_ta_backend.main:app --debug run --port 8188`
 
 The docs are auto-built and deployed to [our docs website](https://uiuc-chatbot.github.io/ai-ta-backend/) on every push. Or you can build the docs locally when writing:
 
@@ -61,3 +75,8 @@ The docs are auto-built and deployed to [our docs website](https://uiuc-chatbot.
 'url': doc.metadata.get('url'), # wouldn't this error out?
 'base_url': doc.metadata.get('base_url'),
 ```
+
+
+### Note
+
+For Supabase, the current version we are using is v1.24.09 ([link](https://github.com/supabase/supabase/tree/v1.24.09))

ai_ta_backend/database/qdrant.py

@@ -23,10 +23,10 @@ def __init__(self):
     """
     # vector DB
     self.qdrant_client = QdrantClient(
-        url='http://qdrant:6333',
+        url=os.getenv('QDRANT_URL', 'http://qdrant:6333'),  # Default to localhost if not set
         https=False,
-        api_key=os.environ['QDRANT_API_KEY'],
-        timeout=20,  # default is 5 seconds. Getting timeout errors w/ document groups.
+        api_key=os.getenv('QDRANT_API_KEY'),
+        timeout=20,
     )
 
     self.vectorstore = Qdrant(