A few small fixes for running on latest main

adserver/analyzer/backends/st.py

@@ -42,7 +42,9 @@ def embed_response(self, resp) -> list:
         """Analyze an HTTP response and return a list of keywords/topics for the URL."""
         model = SentenceTransformer(
             "multi-qa-MiniLM-L6-cos-v1",
-            cache_folder=os.getenv("SENTENCE_TRANSFORMERS_HOME", "/model/"),
+            cache_folder=os.getenv(
+                "SENTENCE_TRANSFORMERS_HOME", "/tmp/sentence_transformers"
+            ),
         )
 
         soup = BeautifulSoup(resp.content, features="html.parser")

adserver/analyzer/tasks.py

@@ -53,22 +53,22 @@ def analyze_url(url, publisher_slug, force=False):
 
     log.debug("Analyzing url: %s", normalized_url)
     keywords = set()
-    embeddings = set()
+    embeddings = []
 
     for backend in get_url_analyzer_backends():
         backend_instance = backend(url)
         analyzed_keywords = backend_instance.analyze()  # Can be None
         log.debug("Keywords from '%s': %s", backend.__name__, analyzed_keywords)
 
-        analyzed_embedding = backend.embedding()  # Can be None
+        analyzed_embedding = backend_instance.embedding()  # Can be None
         log.debug("Embedding from '%s': %s", backend.__name__, analyzed_embedding)
 
         if analyzed_keywords:
             for kw in analyzed_keywords:
                 keywords.add(kw)
 
         if analyzed_embedding:
-            embeddings.add(analyzed_embedding)
+            embeddings.append(analyzed_embedding)
 
     log.debug("Keywords found : %s", keywords)
     log.debug("Embeddings found : %s", embeddings)

docker-compose.yml

@@ -44,7 +44,7 @@ services:
     # build:
     #   context: .
     #   dockerfile: ./docker-compose/postgres/Dockerfile
-    image: ankane/pgvector:latest
+    image: pgvector/pgvector:pg15
     volumes:
       - local_postgres_data:/var/lib/postgresql/data
       - local_postgres_data_backups:/backups

docker-compose/django/Dockerfile

@@ -52,6 +52,9 @@ RUN chmod +x /start-celerybeat
 # Ensure that ``python`` is in the PATH so that ``./manage.py`` works
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
+# Load model
+RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('multi-qa-MiniLM-L6-cos-v1', cache_folder='/tmp/sentence_transformers')"
+
 # Setup the locale
 RUN locale-gen en_US.UTF-8
 RUN dpkg-reconfigure locales

docker-compose/django/start

@@ -5,7 +5,7 @@ set -o pipefail
 set -o nounset
 
 # Reinstall dependencies without rebuilding docker image
-# pip install -r /app/requirements/production.txt
+# pip install -r /app/requirements/production.txt -r /app/requirements/analyzer.txt
 
 # Don't auto-migrate locally because this can cause weird issues when testing migrations
 # python manage.py migrate