more cleanup

backend/arxivsearch/api/routes/papers.py

@@ -8,7 +8,7 @@
 
 from arxivsearch import config
 from arxivsearch.db import redis_helpers
-from arxivsearch.schema.similarity import (
+from arxivsearch.schema.models import (
     PaperSimilarityRequest,
     SearchResponse,
     UserTextSimilarityRequest,

backend/arxivsearch/db/load.py

@@ -10,7 +10,7 @@
 
 from arxivsearch import config
 from arxivsearch.db import redis_helpers
-from arxivsearch.schema.provider import Provider
+from arxivsearch.schema.models import Provider
 
 logger = logging.getLogger(__name__)
 

backend/arxivsearch/db/redis_helpers.py

@@ -23,7 +23,7 @@ def get_schema():
     return IndexSchema.from_yaml(os.path.join(dir_path, "index.yaml"))
 
 
-def get_index():
+def get_test_index():
     dir_path = os.path.dirname(os.path.realpath(__file__))
     index = SearchIndex.from_yaml(os.path.join(dir_path, "index.yaml"))
     index.connect(redis_url=config.REDIS_URL)

backend/arxivsearch/main.py

@@ -56,7 +56,6 @@ def main():
             }
         )
 
-    # uvicorn.run(app, **server_attr)
     uvicorn.run("arxivsearch.main:app", **server_attr)
 
 

backend/arxivsearch/schema/similarity.py → backend/arxivsearch/schema/models.py

@@ -1,6 +1,15 @@
-from pydantic import BaseModel, Field
+from enum import Enum
+from pydantic import BaseModel
 
-from arxivsearch.schema.provider import Provider
+from arxivsearch.schema.models import Provider
+
+
+class Provider(str, Enum):
+    """Embedding model provider"""
+
+    huggingface = "huggingface"
+    openai = "openai"
+    cohere = "cohere"
 
 
 class BaseRequest(BaseModel):
@@ -20,7 +29,7 @@ class UserTextSimilarityRequest(BaseRequest):
 
 
 class Paper(BaseModel):
-    paper_id: str  # = Field(alias="id")
+    paper_id: str
     authors: str
     categories: str
     year: str

backend/arxivsearch/tests/api/routes/test_papers.py

@@ -1,9 +1,8 @@
 import pytest
-import pytest_asyncio
 from httpx import AsyncClient
 
 from arxivsearch.main import app
-from arxivsearch.schema.similarity import (
+from arxivsearch.schema.models import (
     PaperSimilarityRequest,
     UserTextSimilarityRequest,
 )

backend/arxivsearch/tests/utils/seed.py

@@ -20,6 +20,6 @@ def seed_test_db():
         paper["openai"] = np.array(paper["openai"], dtype=np.float32).tobytes()
         paper["cohere"] = np.array(paper["cohere"], dtype=np.float32).tobytes()
 
-    index = redis_helpers.get_index()
+    index = redis_helpers.get_test_index()
     index.load(data=papers, id_field="paper_id")
     return papers

backend/arxivsearch/utils/embeddings.py

@@ -8,7 +8,7 @@
 )
 
 from arxivsearch import config
-from arxivsearch.schema.provider import Provider
+from arxivsearch.schema.models import Provider
 
 
 def preprocess_text(text: str) -> str:
@@ -43,10 +43,7 @@ class Embeddings:
     def __init__(self):
         self.oai_vectorizer = OpenAITextVectorizer(model=config.OPENAI_EMBEDDING_MODEL)
         self.co_vectorizer = CohereTextVectorizer(model=config.COHERE_EMBEDDING_MODEL)
-        self.hf_vectorizer = HFTextVectorizer(
-            model=config.SENTENCE_TRANSFORMER_MODEL
-        )  # resume_download=True
-        # self.hf_vectorizer = None
+        self.hf_vectorizer = HFTextVectorizer(model=config.SENTENCE_TRANSFORMER_MODEL)
 
     async def get(self, provider: str, text: str):
         """

backend/scripts.py

@@ -9,10 +9,6 @@ def start_app():
     # load data
     subprocess.run(["python", "-m", "arxivsearch.db.load"], check=True)
     # start app
-    # subprocess.run(
-    #     ["uvicorn", "arxivsearch.main:app", "--port", "8888", "--host", "0.0.0.0"],
-    #     check=True,
-    # )
     subprocess.run(["python", "-m", "arxivsearch.main"], check=True)
 
 

docker-local-redis.yml

@@ -21,20 +21,9 @@ services:
     ports:
       - "6379:6379"
       - "8001:8001"
-    # volumes:
-    #     - redis-vector-db:/data
     healthcheck:
       test: ["CMD", "redis-cli", "-h", "localhost", "-p", "6379", "ping"]
       interval: 2s
       timeout: 1m30s
       retries: 5
       start_period: 5s
-
-
-# volumes:
-#   redis-vector-db:
-#     driver: local
-#     driver_opts:
-#       type: none
-#       device: data
-#       o: bind

frontend/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "my-app",
+  "name": "redis-arXiv-search",
   "version": "0.1.0",
   "private": true,
   "dependencies": {