Update web crawl conditions

compass/pipeline/collection/steps.py

@@ -13,8 +13,7 @@
     find_jurisdiction_website,
     load_known_docs,
 )
-from compass.validation.location import JurisdictionWebsiteValidator
-from compass.utilities.enums import LLMTasks, COMPASSDocumentCollectionStep
+from compass.utilities.enums import COMPASSDocumentCollectionStep
 from compass.utilities.url import base_website_url
 from compass.pb import COMPASS_PB
 
@@ -258,24 +257,21 @@ async def collect(self, workflow):  # noqa: PLR6301
 
         if not workflow.perform_website_search:
             return []
+
+        await _validate_jurisdiction_website(workflow)
         if not workflow.jurisdiction_website:
-            await try_set_website_from_jurisdiction(workflow)
-            if not workflow.jurisdiction_website:
-                logger.debug(
-                    "No jurisdiction website found for %r; skipping "
-                    "website document collection",
-                    workflow.jurisdiction.full_name,
-                )
-                return []
+            logger.debug(
+                "No jurisdiction website found for %r; skipping "
+                "ELM website document collection",
+                workflow.jurisdiction.full_name,
+            )
+            return []
 
         logger.debug(
             "Collecting documents using ELM web crawl for: %s",
             workflow.jurisdiction.full_name,
         )
         try:
-            workflow.jurisdiction_website = await get_redirected_url(
-                workflow.jurisdiction_website, timeout=30
-            )
             out = await download_jurisdiction_ordinances_from_website(
                 workflow.jurisdiction_website,
                 heuristic=await workflow.extractor.get_heuristic(),
@@ -342,15 +338,15 @@ async def collect(self, workflow):  # noqa: PLR6301
         """
         if not workflow.perform_website_search:
             return []
+
+        await _validate_jurisdiction_website(workflow)
         if not workflow.jurisdiction_website:
-            await try_set_website_from_jurisdiction(workflow)
-            if not workflow.jurisdiction_website:
-                logger.debug(
-                    "No jurisdiction website found for %r; skipping "
-                    "website document collection",
-                    workflow.jurisdiction.full_name,
-                )
-                return []
+            logger.debug(
+                "No jurisdiction website found for %r; skipping "
+                "COMPASS website document collection",
+                workflow.jurisdiction.full_name,
+            )
+            return []
 
         logger.debug(
             "Collecting documents using COMPASS web crawl for: %s",
@@ -384,65 +380,20 @@ async def collect(self, workflow):  # noqa: PLR6301
         return docs
 
 
-async def try_set_website_from_jurisdiction(workflow):
-    """Resolve the website URL for this jurisdiction
-
-    Parameters
-    ----------
-    workflow : compass.pipeline.jurisdiction.SingleJurisdictionRun
-        The workflow for the jurisdiction being processed, which may or
-        may not have a user-supplied website URL. If the workflow
-        doesn't have a website URL, this function will attempt to find
-        one.
-    """
-    if workflow.jurisdiction_website:
-        if workflow.validate_user_website_input:
-            await _validate_jurisdiction_website(workflow)
-        else:
-            workflow.jurisdiction_website = await _get_base_website(
-                workflow.jurisdiction_website
-            )
-
-    if not workflow.jurisdiction_website:
-        website = await _find_jurisdiction_website_for_workflow(workflow)
-        if website:
-            workflow.jurisdiction_website = website
-
-
 async def _validate_jurisdiction_website(workflow):
-    """Validate a user-supplied jurisdiction website"""
-    if workflow.jurisdiction_website is None:
-        return
+    """Try to set and resolve the website URL for this jurisdiction"""
+    if workflow.jurisdiction_website:
+        workflow.jurisdiction_website = await _get_base_website(
+            workflow.jurisdiction_website
+        )
 
-    workflow.jurisdiction_website = await _get_base_website(
-        workflow.jurisdiction_website,
-    )
-    if workflow.jurisdiction_website is None:
+    # only try to find a website if we don't have one and we have LLMs
+    # we can use for validation
+    if workflow.jurisdiction_website or not workflow.runtime.models:
         return
 
-    COMPASS_PB.update_jurisdiction_task(
-        workflow.jurisdiction.full_name,
-        description=(
-            f"Validating user input website: {workflow.jurisdiction_website}"
-        ),
-    )
-    model_config = workflow.runtime.models.get(
-        LLMTasks.DOCUMENT_JURISDICTION_VALIDATION,
-        workflow.runtime.models[LLMTasks.DEFAULT],
-    )
-    validator = JurisdictionWebsiteValidator(
-        browser_semaphore=workflow.runtime.browser_semaphore,
-        file_loader_kwargs=workflow.runtime.file_loader_kwargs_no_ocr,
-        usage_tracker=workflow.usage_tracker,
-        llm_service=model_config.llm_service,
-        **model_config.llm_call_kwargs,
-    )
-    is_website_correct = await validator.check(
-        workflow.jurisdiction_website,
-        workflow.jurisdiction,
-    )
-    if not is_website_correct:
-        workflow.jurisdiction_website = None
+    if website := await _find_jurisdiction_website_for_workflow(workflow):
+        workflow.jurisdiction_website = await _get_base_website(website)
 
 
 async def _get_base_website(website):
@@ -468,7 +419,6 @@ async def _find_jurisdiction_website_for_workflow(workflow):
         search_semaphore=workflow.runtime.search_engine_semaphore,
         browser_semaphore=workflow.runtime.browser_semaphore,
         usage_tracker=workflow.usage_tracker,
-        validate=workflow.validate_user_website_input,
         url_ignore_substrings=(
             workflow.runtime.search_params.url_ignore_substrings
         ),

compass/pipeline/coordinator.py

@@ -122,13 +122,7 @@ def __init__(self, runtime):
         """
         self.runtime = runtime
 
-    def _create(
-        self,
-        jurisdiction,
-        *,
-        usage_tracker=None,
-        validate_user_website_input=True,
-    ):
+    def _create(self, jurisdiction, *, usage_tracker=None):
         """Create one configured jurisdiction workflow"""
         extractor = self.runtime.extractor_class(
             jurisdiction=jurisdiction,
@@ -148,7 +142,6 @@ def _create(
             perform_website_search=(
                 self.runtime.request.perform_website_search
             ),
-            validate_user_website_input=validate_user_website_input,
         )
 
     @abstractmethod
@@ -195,11 +188,7 @@ async def run(self, jurisdictions_df):
             usage_tracker = UsageTracker(
                 jurisdiction.full_name, usage_from_response
             )
-            workflow = self._create(
-                jurisdiction,
-                usage_tracker=usage_tracker,
-                validate_user_website_input=True,
-            )
+            workflow = self._create(jurisdiction, usage_tracker=usage_tracker)
             tasks.append(
                 asyncio.create_task(
                     workflow.run_process_with_logging(),
@@ -252,11 +241,7 @@ async def run(self, jurisdictions_df):
         )
         tasks = []
         for jurisdiction in jurisdictions_from_df(jurisdictions_df):
-            workflow = self._create(
-                jurisdiction,
-                usage_tracker=None,
-                validate_user_website_input=False,
-            )
+            workflow = self._create(jurisdiction, usage_tracker=None)
             tasks.append(
                 asyncio.create_task(
                     workflow.run_collection_with_logging(
@@ -342,11 +327,7 @@ async def run(self, jurisdictions_df):
             usage_tracker = UsageTracker(
                 jurisdiction.full_name, usage_from_response
             )
-            workflow = self._create(
-                jurisdiction,
-                usage_tracker=usage_tracker,
-                validate_user_website_input=True,
-            )
+            workflow = self._create(jurisdiction, usage_tracker=usage_tracker)
             tasks.append(
                 asyncio.create_task(
                     workflow.run_extraction_with_logging(collection_info[0]),

compass/pipeline/data_classes.py

@@ -673,7 +673,7 @@ def __init__(  # noqa: PLR0913
     @cached_property
     def models(self):
         """dict: Mapping of LLM task to OpenAIConfig for this request"""
-        if not self.user_model_input or self.MODE == COMPASSRunMode.COLLECT:
+        if not self.user_model_input:
             return {}
         return _build_models(self.user_model_input)
 
@@ -750,13 +750,57 @@ def __init__(  # noqa: PLR0913
             "City", "Township", etc.)
         model : str or list of dict, optional
             Optional model configuration used only for collection-side
-            LLM tasks, such as validating a user-supplied jurisdiction
-            website. If provided as a string, it is treated as the
-            default model name. If provided as a list, each entry
-            should contain keyword arguments used to initialize
-            :class:`~compass.llm.config.OpenAIConfig`, along with a
-            ``tasks`` key describing which LLM tasks that configuration
-            should handle. By default, ``None``.
+            LLM tasks, such as:
+
+                - Searching for and validating a jurisdiction website
+                  before website crawl
+
+            If this key is left out, these steps are skipped completely.
+            If provided as a string, it is assumed to be the name of the
+            default model (e.g., "gpt-5-mini"), and environment
+            variables are used for authentication.
+
+            If a list is provided, it should contain dictionaries of
+            arguments that can initialize instances of
+            :class:`~compass.llm.config.OpenAIConfig`. Each dictionary
+            can specify the model name, client type, and initialization
+            arguments.
+
+            Each dictionary must also include a ``tasks`` key, which
+            maps to a string or list of strings indicating the tasks
+            that instance should handle. Exactly one of the instances
+            **must** include "default" as a task, which will be used
+            when no specific task is matched. For example::
+
+                "model": [
+                    {
+                        "model": "gpt-4o-mini",
+                        "llm_call_kwargs": {
+                            "temperature": 0,
+                            "timeout": 300,
+                        },
+                        "client_kwargs": {
+                            "api_key": "<your_api_key>",
+                            "api_version": "<your_api_version>",
+                            "azure_endpoint": "<your_azure_endpoint>",
+                        },
+                        "tasks": ["default", "date_extraction"],
+                    },
+                    {
+                        "model": "gpt-4o",
+                        "client_type": "openai",
+                        "tasks": ["ordinance_text_extraction"],
+                    }
+                ]
+
+            .. IMPORTANT::
+                You will need to ensure that the model name used here
+                matches your deployment if you are using Azure OpenAI.
+                For example, if you deployed the GPT-4o-mini model under
+                the name ``"gpt-4o-mini-2025-04-11"``, you would want to
+                set ``"model": "gpt-4o-mini-2025-04-11"``.
+
+            By default, ``None``.
         num_urls_to_check_per_jurisdiction : int, default=5
             Number of unique Google search result URLs to check for each
             jurisdiction when attempting to locate ordinance documents.

compass/pipeline/jurisdiction.py

@@ -34,7 +34,6 @@ def __init__(
         known_doc_urls=None,
         perform_se_search=True,
         perform_website_search=True,
-        validate_user_website_input=True,
     ):
         """
 
@@ -69,10 +68,6 @@ def __init__(
         perform_website_search : bool, optional
             Whether website-specific search and crawl steps should be
             performed for this jurisdiction. By default, ``True``.
-        validate_user_website_input : bool, optional
-            Whether user-supplied jurisdiction website inputs should be
-            validated before being used in collection. By default,
-            ``True``.
         """
         self.runtime = runtime
         self.jurisdiction = jurisdiction
@@ -82,7 +77,6 @@ def __init__(
         self.known_doc_urls = known_doc_urls
         self.perform_se_search = perform_se_search
         self.perform_website_search = perform_website_search
-        self.validate_user_website_input = validate_user_website_input
         self.jurisdiction_website = jurisdiction.website_url
         self.last_scrape_results = []
         self.extraction_workflow = DocumentExtraction(self)

compass/scripts/download.py

@@ -173,7 +173,6 @@ async def find_jurisdiction_website(
     browser_semaphore=None,
     usage_tracker=None,
     url_ignore_substrings=None,
-    validate=True,
     **kwargs,
 ):
     """Search for the main landing page of a given jurisdiction
@@ -213,12 +212,6 @@ async def find_jurisdiction_website(
     url_ignore_substrings : list of str, optional
         URL substrings that should be excluded from search results.
         Substrings are applied case-insensitively. By default, ``None``.
-    validate : bool, default=True
-        If ``True``, each potential jurisdiction website will be checked
-        for validity using the
-        :class:`~compass.validation.location.JurisdictionWebsiteValidator`
-        before being returned. If ``False``, the first potential website
-        will be returned without validation. By default, ``True``.
     **kwargs
         Additional arguments forwarded to
         :func:`elm.web.search.run.search_with_fallback`.
@@ -250,9 +243,6 @@ async def find_jurisdiction_website(
     if not potential_website_links:
         return None
 
-    if not validate:
-        return potential_website_links.pop()
-
     model_config = model_configs.get(
         LLMTasks.JURISDICTION_MAIN_WEBSITE_VALIDATION,
         model_configs[LLMTasks.DEFAULT],

support/jurisdictions/update_jur_websites.py

@@ -457,7 +457,6 @@ async def _find_jurisdiction_website_for_jurisdiction(
         search_semaphore=search_semaphore,
         browser_semaphore=browser_semaphore,
         usage_tracker=usage_tracker,
-        validate=True,
         url_ignore_substrings=search_params.url_ignore_substrings,
         **search_params.se_kwargs,
     )

tests/python/unit/scripts/test_download.py

@@ -13,7 +13,7 @@
 async def test_find_jurisdiction_website_returns_base_domain(monkeypatch):
     """Return the canonical root URL for the selected website"""
 
-    async def fake_search_with_fallback(**_kwargs):
+    async def fake_search_with_fallback(**_kwargs):  # noqa
         return [
             "https://prattvilleal.gov/venue/autauga-county-commission/",
             "https://prattvilleal.gov/government/mayor",
@@ -48,9 +48,7 @@ async def check(self, url, jurisdiction):
     )
 
     out = await download_module.find_jurisdiction_website(
-        jurisdiction,
-        {LLMTasks.DEFAULT: model_config},
-        validate=True,
+        jurisdiction, {LLMTasks.DEFAULT: model_config}
     )
 
     assert out == "https://prattvilleal.gov/"