Part 1 of major refactor to support plugins

compass/_cli/finalize.py

@@ -8,7 +8,7 @@
 from rich.console import Console
 
 from compass.utilities import Directories
-from compass.utilities.location import Jurisdiction
+from compass.utilities.jurisdictions import Jurisdiction
 from compass.utilities.parsing import load_config
 from compass.utilities.finalize import save_run_meta, doc_infos_to_db, save_db
 from compass.scripts.process import _initialize_model_params
@@ -116,16 +116,7 @@ def _compile_db(jurisdictions, dirs):
 
         doc_info = doc_info[0]
         all_doc_infos.append(
-            {
-                "ord_db_fp": ord_db_fp,
-                "source": doc_info.get("source"),
-                "date": (
-                    doc_info.get("effective_year"),
-                    doc_info.get("effective_month"),
-                    doc_info.get("effective_day"),
-                ),
-                "jurisdiction": jurisdiction,
-            }
+            {"ord_db_fp": ord_db_fp, "jurisdiction": jurisdiction}
         )
 
     db, __ = doc_infos_to_db(all_doc_infos)

compass/common/__init__.py

@@ -2,7 +2,6 @@
 
 from .base import (
     EXTRACT_ORIGINAL_SETBACK_TEXT_PROMPT,
-    BaseTextExtractor,
     empty_output,
     llm_response_starts_with_no,
     llm_response_starts_with_yes,

compass/common/base.py

@@ -1,19 +1,13 @@
 """Common ordinance extraction components"""
 
-import asyncio
 import logging
 from datetime import datetime
 
 import networkx as nx
-from elm import ApiBase
 
 from compass.common.tree import AsyncDecisionTree
 from compass.utilities import llm_response_as_json
-from compass.utilities.enums import LLMUsageCategory
-from compass.utilities.parsing import (
-    merge_overlapping_texts,
-    clean_backticks_from_llm_response,
-)
+
 from compass.exceptions import COMPASSRuntimeError
 
 
@@ -970,64 +964,3 @@ def setup_graph_permitted_use_districts(**kwargs):
         ),
     )
     return G
-
-
-class BaseTextExtractor:
-    """Base implementation for a text extractor"""
-
-    SYSTEM_MESSAGE = (
-        "You are a text extraction assistant. Your job is to extract only "
-        "verbatim, **unmodified** excerpts from provided legal or policy "
-        "documents. Do not interpret or paraphrase. Do not summarize. Only "
-        "return exactly copied segments that match the specified scope. If "
-        "the relevant content appears within a table, return the entire "
-        "table, including headers and footers, exactly as formatted."
-    )
-    """System message for text extraction LLM calls"""
-    _USAGE_LABEL = LLMUsageCategory.DOCUMENT_ORDINANCE_SUMMARY
-
-    def __init__(self, llm_caller):
-        """
-
-        Parameters
-        ----------
-        llm_caller : LLMCaller
-            LLM Caller instance used to extract ordinance info with.
-        """
-        self.llm_caller = llm_caller
-
-    async def _process(self, text_chunks, instructions, is_valid_chunk):
-        """Perform extraction processing"""
-        logger.info(
-            "Extracting summary text from %d text chunks asynchronously...",
-            len(text_chunks),
-        )
-        logger.debug("Model instructions are:\n%s", instructions)
-        outer_task_name = asyncio.current_task().get_name()
-        summaries = [
-            asyncio.create_task(
-                self.llm_caller.call(
-                    sys_msg=self.SYSTEM_MESSAGE,
-                    content=f"{instructions}\n\n# TEXT #\n\n{chunk}",
-                    usage_sub_label=self._USAGE_LABEL,
-                ),
-                name=outer_task_name,
-            )
-            for chunk in text_chunks
-        ]
-        summary_chunks = await asyncio.gather(*summaries)
-        summary_chunks = [
-            clean_backticks_from_llm_response(chunk)
-            for chunk in summary_chunks
-            if is_valid_chunk(chunk)
-        ]
-
-        text_summary = merge_overlapping_texts(summary_chunks)
-        logger.debug(
-            "Final summary contains %d tokens",
-            ApiBase.count_tokens(
-                text_summary,
-                model=self.llm_caller.kwargs.get("model", "gpt-4"),
-            ),
-        )
-        return text_summary

compass/data/tx_water_districts.csv

@@ -38,7 +38,7 @@ Texas,,High Plains,Underground Water Conservation District,36,
 Texas,,Hill Country,Underground Water Conservation District,37,
 Texas,,Hudspeth County,Underground Water Conservation District,38,
 Texas,,Irion County,WConservation District,39,
-Texas,,Jeff Davis,County Underground Water Conservation District,40,
+Texas,,Jeff Davis County,Underground Water Conservation District,40,
 Texas,,Kenedy County,Groundwater Conservation District,41,
 Texas,,Kimble County,Groundwater Conservation District,42,
 Texas,,Kinney County,Groundwater Conservation District,43,

compass/exceptions.py

@@ -21,9 +21,17 @@ class COMPASSNotInitializedError(COMPASSError):
     """COMPASS not initialized error"""
 
 
+class COMPASSTypeError(COMPASSError, TypeError):
+    """COMPASS TypeError"""
+
+
 class COMPASSValueError(COMPASSError, ValueError):
     """COMPASS ValueError"""
 
 
 class COMPASSRuntimeError(COMPASSError, RuntimeError):
     """COMPASS RuntimeError"""
+
+
+class COMPASSPluginConfigurationError(COMPASSRuntimeError):
+    """COMPASS Plugin Configuration Error"""

compass/extraction/__init__.py

@@ -1,9 +1,9 @@
 """Ordinance text extraction tooling"""
 
 from .apply import (
-    check_for_ordinance_info,
+    check_for_relevant_text,
     extract_date,
-    extract_ordinance_text_with_llm,
-    extract_ordinance_text_with_ngram_validation,
+    extract_relevant_text_with_llm,
+    extract_relevant_text_with_ngram_validation,
     extract_ordinance_values,
 )