Merge branch 'longview-labs:main' into main

Author: longviewoor

docs/tooltips.json

@@ -37,12 +37,84 @@
     "definition": "Every single AO process and message is stored and read from Arweave. It acts as the state layer for each process, while AO nodes read from it and perform computation.",
     "link": "https://www.arweave.org/"
   },
+  {
+    "term": "ArDrive",
+    "headline": "Permanent cloud storage platform",
+    "definition": "ArDrive is like your favourite cloud storage app, but pay once, store permanently. Users can upload files, pages, and more.",
+    "link": "https://ardrive.io/"
+  },
+  {
+    "term": "AOX",
+    "headline": "AOX is a cross-chain bridge for AO and Arweave",
+    "definition": "Users can bring wrapped assets in and out of the AO ecosystem, and can bridge assets from both Arweave and Ethereum.",
+    "link": "https://aox.xyz"
+  },
+  {
+    "term": "Astro",
+    "headline": "AstroUSD ($USDA) is AO's first native stablecoin",
+    "definition": "Astro aims to solve liquidity fragementation by acting a a universal conversion layer for different kinds of stablecoins bridged to AO.",
+    "link": "https://astrousd.com"
+  },
+  {
+    "term": "Beacon Wallet",
+    "headline": "Beacon is a native iOS mobile wallet for AO and Arweave",
+    "definition": "Beacon includes features such as 2FA, multi-sigs, built-in app integrations, and social recovery.",
+    "link": "https://beaconwallet.app"
+  },
+  {
+    "term": "BetterIDEa",
+    "headline": "A web-based IDE for developing applications on AO",
+    "definition": "BetterIDEa is a web-based IDE for developing applications on AO. It has built-in support for the AO Package Manager (APM) and other features for developing processes on AO.",
+    "link": "https://betteridea.dev"
+  },
+  {
+    "term": "Botega",
+    "headline": "Botega is a decentralized exchange (DEX) and DeFi platform built on AO",
+    "definition": "It is fully decentralized with the frontend stored on Arweave and backend processes on AO, and is focused on supporting sophisticated strategies and autonomous trading agents.",
+    "link": "https://botega.arweave.net"
+  },
+  {
+    "term": "Dexi",
+    "headline": "Dexi is financial data platform for AO",
+    "definition": "Dexi aggregates data from liquidity pools agents deployed on AO, and siplays data on trading volume, asset prices, fluctuations, and more.",
+    "link": "https://dexi.arweave.net"
+  },
+  {
+    "term": "LiquidOps",
+    "headline": "LiquidOps is a lending and borrowing platform built on AO",
+    "definition": "Users can accrue interest by providing liquidty, and lenders can take overcollateralized loans from the protocol.",
+    "link": "https://liquidops.io"
+  },
+  {
+    "term": "Load Network",
+    "headline": "Load is an \"onchain data center\" focused on scaling EVM storage",
+    "definition": "Load Network is focused on scaling EVM storage via Arweave, and building out other developer solutions and infrastructure for uploading data to the Permaweb.",
+    "link": "https://www.load.network/"
+  },
   {
     "term": "Lua",
     "headline": "Lua is the programming language used for AO processes",
     "definition": "Lua is a dynamically-typed, lightweight programming language used to write processes on AO.",
     "link": "https://learnxinyminutes.com/lua/"
   },
+  {
+    "term": "Permaswap",
+    "headline": "Permaswap is a decentralized exchange (DEX) in the AO ecosystem",
+    "definition": "Users can swap between assets such as AO, wrapped AR, and more, and can place limit orders and set up a custom slippage amount.",
+    "link": "https://permaswap.network"
+  },
+  {
+    "term": "PerplexFi",
+    "headline": "A perpetual trading platform built on AO",
+    "definition": "Perplex is a perpetual futures trading platform built fully onchain, on AO. Users can trade perpetual contracts with leverage, provide liquidty and earn yield from pools, and more.",
+    "link": "https://perplex.finance"
+  },
+  {
+    "term": "ProtocolLand",
+    "headline": "Onchain hosting platform for code repositories",
+    "definition": "ProtocolLand is like GitHub, but built on Arweave and AO. Users can store their code repositories and deploy websites and apps through built-in integrations.",
+    "link": "https://protocol.land"
+  },
   {
     "term": "Wander Wallet",
     "headline": "Browser extension and mobile wallet for Arweave and AO",

sync.py

@@ -2,16 +2,17 @@
 import json
 import logging
 from pathlib import Path
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Set
 import weaviate
 from weaviate.util import generate_uuid5
 from weaviate.classes.init import Auth
 from weaviate.classes.config import Configure, Property, DataType
+from weaviate.exceptions import WeaviateBaseError
 # from dotenv import load_dotenv
 
-# load_dotenv()
+# load_dotenv(override=True)
 
-logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s\n")
 
 class WeaviateSyncer:
     def __init__(self):
@@ -33,59 +34,106 @@ def ensure_collection(self, name: str, properties: List[Property]):
             )
             logging.info(f"Created collection '{name}'")
 
+    def get_existing_uuids(self, collection_name: str, batch_size: int = 100) -> Set[str]:
+        collection = self.client.collections.get(collection_name)
+        uuids = set()
+        offset = 0
+
+        while True:
+            response = collection.query.fetch_objects(limit=batch_size, offset=offset)
+            objects = response.objects
+            if not objects:
+                break
+            for obj in objects:
+                uuids.add(str(obj.uuid))
+            offset += len(objects)
+
+        return uuids
+
     def upsert_entry(self, collection_name: str, properties: Dict[str, Any], uuid_key: str):
         collection = self.client.collections.get(collection_name)
-        uuid = generate_uuid5(properties[uuid_key])
+        stable_id = properties[uuid_key].strip().lower()
+        uuid = generate_uuid5(stable_id)
+
         try:
             collection.data.insert(properties=properties, uuid=uuid)
-            logging.info(f"Inserted: {properties.get(uuid_key, '')[:50]}")
+            logging.info(f"Inserted: {properties.get(uuid_key, '')[:50]} (UUID: {uuid})")
         except weaviate.exceptions.UnexpectedStatusCodeException as e:
             if "already exists" in str(e):
                 collection.data.update(uuid=uuid, properties=properties)
-                logging.info(f"Updated: {properties.get(uuid_key, '')[:50]}")
+                logging.info(f"Updated: {properties.get(uuid_key, '')[:50]} (UUID: {uuid})")
             else:
                 logging.error(f"Error inserting: {e}")
 
-    def sync_from_json(self, collection_name: str, schema_props: List[Property], json_path: Path, uuid_key: str):
-        self.ensure_collection(collection_name, schema_props)
-        if not json_path.exists():
-            logging.warning(f"File not found: {json_path}")
-            return
+        return uuid
+
+    def delete_entry(self, collection_name: str, uuid: str, name_field: str):
+        collection = self.client.collections.get(collection_name)
         try:
-            with json_path.open(encoding="utf-8") as f:
-                entries = json.load(f)
-        except json.JSONDecodeError as e:
-            logging.error(f"Error loading {json_path.name}: {e}")
-            return
+            obj = collection.query.fetch_object_by_id(uuid)
+            name = obj.properties.get(name_field, "<unknown>") if obj else "<missing>"
+            logging.info(f"Deleting: {name} (UUID: {uuid})")
+            collection.data.delete_by_id(uuid)
+        except WeaviateBaseError as e:
+            logging.error(f"Failed to delete {uuid}: {e}")
+
+    def sync_entries(self, collection_name: str, schema_props: List[Property], entries: List[Dict[str, Any]], uuid_key: str):
+        self.ensure_collection(collection_name, schema_props)
+
+        # Upsert all entries and collect current UUIDs
+        current_uuids = set()
         for entry in entries:
-            self.upsert_entry(collection_name, entry, uuid_key)
+            uuid = self.upsert_entry(collection_name, entry, uuid_key)
+            current_uuids.add(uuid)
+
+        # Get what's currently in DB (after upserts)
+        existing_uuids = self.get_existing_uuids(collection_name)
+        logging.info(f"Existing uuids: {existing_uuids}")
+        logging.info(f"Current uuids: {current_uuids}")
+        to_delete = existing_uuids - current_uuids
+        logging.info(f"UUIDs to delete: {to_delete}")
+
+        # Delete what's no longer present
+        for uuid in to_delete:
+            self.delete_entry(collection_name, uuid, uuid_key)
 
     def close(self):
         self.client.close()
 
+
 def main():
     syncer = WeaviateSyncer()
     try:
-        # Q&A sync
+        # --- FAQ Sync ---
         qa_props = [
             Property(name="question", data_type=DataType.TEXT),
             Property(name="answer", data_type=DataType.TEXT),
             Property(name="source", data_type=DataType.TEXT),
             Property(name="tags", data_type=DataType.TEXT_ARRAY)
         ]
-        docs_dir = Path(__file__).resolve().parent / "docs" / "faq"
-        for file_path in docs_dir.glob("*.json"):
-            syncer.sync_from_json("QAEntry", qa_props, file_path, "question")
+        faq_path = Path(__file__).resolve().parent / "docs" / "faq" / "general.json"
+        if faq_path.exists():
+            with faq_path.open(encoding="utf-8") as f:
+                faq_entries = json.load(f)
+            syncer.sync_entries("QAEntry", qa_props, faq_entries, "question")
+        else:
+            logging.warning(f"FAQ file not found: {faq_path}")
 
-        # Tooltip sync
+        # --- Tooltip Sync ---
         tooltip_props = [
             Property(name="term", data_type=DataType.TEXT),
             Property(name="headline", data_type=DataType.TEXT),
             Property(name="definition", data_type=DataType.TEXT),
             Property(name="link", data_type=DataType.TEXT)
         ]
         tooltip_path = Path(__file__).resolve().parent / "docs" / "tooltips.json"
-        syncer.sync_from_json("Tooltip", tooltip_props, tooltip_path, "term")
+        if tooltip_path.exists():
+            with tooltip_path.open(encoding="utf-8") as f:
+                tooltip_entries = json.load(f)
+            syncer.sync_entries("Tooltip", tooltip_props, tooltip_entries, "term")
+        else:
+            logging.warning(f"Tooltip file not found: {tooltip_path}")
+
     finally:
         syncer.close()