Merge remote-tracking branch 'origin' into 132-smart-memory-management

Author: minump

.env.template

@@ -0,0 +1,34 @@
+# Supabase SQL
+SUPABASE_URL=
+SUPABASE_API_KEY=
+SUPABASE_READ_ONLY=
+SUPABASE_JWT_SECRET=
+
+MATERIALS_SUPABASE_TABLE=uiuc_chatbot
+NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE=documents
+
+# QDRANT
+QDRANT_COLLECTION_NAME=uiuc-chatbot
+DEV_QDRANT_COLLECTION_NAME=dev
+QDRANT_URL=
+QDRANT_API_KEY=
+
+REFACTORED_MATERIALS_SUPABASE_TABLE=
+
+# AWS
+S3_BUCKET_NAME=uiuc-chatbot
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+OPENAI_API_KEY=
+
+NOMIC_API_KEY=
+LINTRULE_SECRET=
+
+# Github Agent
+GITHUB_APP_ID=<OptionalForGithubApps>
+GITHUB_APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----
+
+-----END RSA PRIVATE KEY-----"
+
+NUMEXPR_MAX_THREADS=2

.github/workflows/rules.yml

@@ -1,25 +0,0 @@
-name: Rules Check
-
-on:
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  rules:
-    runs-on: ubuntu-latest
-    environment: lintrule
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2 # this part is important!
-
-      - name: Install Lint Rules
-        run: |
-          curl -fsSL https://www.lintrule.com/install.sh | bash
-
-      - name: Run Lint Rules Check
-        run: |
-          rules check --secret "${{ secrets.LINTRULE_SECRET }}"

.github/workflows/yapf-format.yml

@@ -3,21 +3,23 @@ name: Format code
 on:
   push:
     branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 jobs:
   format:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - name: pip install yapf
+        run: pip install yapf
       - name: Format code with yapf
-        run: |
-          pip install yapf
-          yapf --in-place --recursive --parallel --style='{based_on_style: google, column_limit: 140, indent_width: 2}' --exclude '*.env' .
+        run: yapf --in-place --recursive --parallel --style='{based_on_style: google, column_limit: 140, indent_width: 2}' --exclude '*.env' .
       - name: Commit changes
         uses: EndBug/add-and-commit@v4
         with:
           author_name: ${{ github.actor }}
           author_email: ${{ github.actor }}@users.noreply.github.com
-          message: "Format code with Yapf & custom options"
+          message: "Format code"
           add: "."
-          branch: ${{ github.ref }}
+          branch: ${{ github.ref }}

.gitignore

@@ -1,7 +1,7 @@
 # don't sync coursera docs
 coursera-dl/
 *parsed.json
-*wandb*
+wandb
 *.ipynb
 *.pem
 

.rules/no-bugs.md

@@ -13,8 +13,22 @@ Automatic [API Reference](https://uiuc-chatbot.github.io/ai-ta-backend/reference
 
 ## 📣 Development
 
-- Install Python requirements `pip install -r requirements.txt`
-- Start the server for development (with live reloads) `cd ai_ta_backend` then `flask --app ai_ta_backend.main:app --debug run --port 8000`
+1. Rename `.env.template` to `.env` and fill in the required variables
+2. Install Python requirements `pip install -r requirements.txt`
+3. Start the server for development (with live reloads) `cd ai_ta_backend` then `flask --app ai_ta_backend.main:app --debug run --port 8000`
 
 The docs are auto-built and deployed to [our docs website](https://uiuc-chatbot.github.io/ai-ta-backend/) on every push. Or you can build the docs locally when writing:
 - `mkdocs serve`
+
+
+### Course metadata structure
+```
+'text': doc.page_content,
+'readable_filename': doc.metadata['readable_filename'],
+'course_name ': doc.metadata['course_name'],
+'s3_path': doc.metadata['s3_path'],
+'pagenumber': doc.metadata['pagenumber_or_timestamp'], # this is the recent breaking change!! 
+# OPTIONAL properties
+'url': doc.metadata.get('url'), # wouldn't this error out?
+'base_url': doc.metadata.get('base_url'),
+```

README.md

@@ -51,4 +51,7 @@ def upload(myfile):
   pool.map(upload, filenames)
 
   print("All data files uploaded to S3 successfully.")
-  return s3_paths
+  return s3_paths
+
+if __name__ == '__main__':
+  pass