From dfaeee8810beb633880fb3eee0e4f4380f63c687 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:05:56 -0600 Subject: [PATCH 01/37] Deprecate input --- compass/pipeline/coordinator.py | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/compass/pipeline/coordinator.py b/compass/pipeline/coordinator.py index ab5f88f82..f7c3198f3 100644 --- a/compass/pipeline/coordinator.py +++ b/compass/pipeline/coordinator.py @@ -122,13 +122,7 @@ def __init__(self, runtime): """ self.runtime = runtime - def _create( - self, - jurisdiction, - *, - usage_tracker=None, - validate_user_website_input=True, - ): + def _create(self, jurisdiction, *, usage_tracker=None): """Create one configured jurisdiction workflow""" extractor = self.runtime.extractor_class( jurisdiction=jurisdiction, @@ -148,7 +142,6 @@ def _create( perform_website_search=( self.runtime.request.perform_website_search ), - validate_user_website_input=validate_user_website_input, ) @abstractmethod @@ -195,11 +188,7 @@ async def run(self, jurisdictions_df): usage_tracker = UsageTracker( jurisdiction.full_name, usage_from_response ) - workflow = self._create( - jurisdiction, - usage_tracker=usage_tracker, - validate_user_website_input=True, - ) + workflow = self._create(jurisdiction, usage_tracker=usage_tracker) tasks.append( asyncio.create_task( workflow.run_process_with_logging(), @@ -252,11 +241,7 @@ async def run(self, jurisdictions_df): ) tasks = [] for jurisdiction in jurisdictions_from_df(jurisdictions_df): - workflow = self._create( - jurisdiction, - usage_tracker=None, - validate_user_website_input=False, - ) + workflow = self._create(jurisdiction, usage_tracker=None) tasks.append( asyncio.create_task( workflow.run_collection_with_logging( @@ -342,11 +327,7 @@ async def run(self, jurisdictions_df): usage_tracker = UsageTracker( jurisdiction.full_name, usage_from_response ) - workflow = self._create( - jurisdiction, - usage_tracker=usage_tracker, - validate_user_website_input=True, - ) + workflow = self._create(jurisdiction, usage_tracker=usage_tracker) tasks.append( asyncio.create_task( workflow.run_extraction_with_logging(collection_info[0]), From 4f7e124c624e8657cd2284f98c115961da1bc9fb Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:06:07 -0600 Subject: [PATCH 02/37] Update docstring --- compass/pipeline/data_classes.py | 57 ++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/compass/pipeline/data_classes.py b/compass/pipeline/data_classes.py index 2fb341510..76f765155 100644 --- a/compass/pipeline/data_classes.py +++ b/compass/pipeline/data_classes.py @@ -750,13 +750,56 @@ def __init__( # noqa: PLR0913 "City", "Township", etc.) model : str or list of dict, optional Optional model configuration used only for collection-side - LLM tasks, such as validating a user-supplied jurisdiction - website. If provided as a string, it is treated as the - default model name. If provided as a list, each entry - should contain keyword arguments used to initialize - :class:`~compass.llm.config.OpenAIConfig`, along with a - ``tasks`` key describing which LLM tasks that configuration - should handle. By default, ``None``. + LLM tasks, such as: + + - validating a jurisdiction website before website crawl + + If this key is left out, these steps are skipped completely. + If provided as a string, it is assumed to be the name of the + default model (e.g., "gpt-5-mini"), and environment + variables are used for authentication. + + If a list is provided, it should contain dictionaries of + arguments that can initialize instances of + :class:`~compass.llm.config.OpenAIConfig`. Each dictionary + can specify the model name, client type, and initialization + arguments. + + Each dictionary must also include a ``tasks`` key, which + maps to a string or list of strings indicating the tasks + that instance should handle. Exactly one of the instances + **must** include "default" as a task, which will be used + when no specific task is matched. For example:: + + "model": [ + { + "model": "gpt-4o-mini", + "llm_call_kwargs": { + "temperature": 0, + "timeout": 300, + }, + "client_kwargs": { + "api_key": "", + "api_version": "", + "azure_endpoint": "", + }, + "tasks": ["default", "date_extraction"], + }, + { + "model": "gpt-4o", + "client_type": "openai", + "tasks": ["ordinance_text_extraction"], + } + ] + + .. IMPORTANT:: + You will need to ensure that the model name used here + matches your deployment if you are using Azure OpenAI. + For example, if you deployed the GPT-4o-mini model under + the name ``"gpt-4o-mini-2025-04-11"``, you would want to + set ``"model": "gpt-4o-mini-2025-04-11"``. + + By default, ``None``. num_urls_to_check_per_jurisdiction : int, default=5 Number of unique Google search result URLs to check for each jurisdiction when attempting to locate ordinance documents. From dcd97a18df8d358ea1dfdb569fdbc818e5cc98ab Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:06:22 -0600 Subject: [PATCH 03/37] Allow website collection if user provides model config --- compass/pipeline/data_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/pipeline/data_classes.py b/compass/pipeline/data_classes.py index 76f765155..744a3aefa 100644 --- a/compass/pipeline/data_classes.py +++ b/compass/pipeline/data_classes.py @@ -673,7 +673,7 @@ def __init__( # noqa: PLR0913 @cached_property def models(self): """dict: Mapping of LLM task to OpenAIConfig for this request""" - if not self.user_model_input or self.MODE == COMPASSRunMode.COLLECT: + if not self.user_model_input: return {} return _build_models(self.user_model_input) From 687ae6370580e85f9eab5c92b547320eb8ca3217 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:06:43 -0600 Subject: [PATCH 04/37] Update docstring --- compass/pipeline/data_classes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compass/pipeline/data_classes.py b/compass/pipeline/data_classes.py index 744a3aefa..bec8963af 100644 --- a/compass/pipeline/data_classes.py +++ b/compass/pipeline/data_classes.py @@ -752,7 +752,8 @@ def __init__( # noqa: PLR0913 Optional model configuration used only for collection-side LLM tasks, such as: - - validating a jurisdiction website before website crawl + - Searching for and validating a jurisdiction website + before website crawl If this key is left out, these steps are skipped completely. If provided as a string, it is assumed to be the name of the From ac27f8008dbcd84cd66b68daa99b1c96674a0be6 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:06:51 -0600 Subject: [PATCH 05/37] Deprecate input --- compass/pipeline/jurisdiction.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/compass/pipeline/jurisdiction.py b/compass/pipeline/jurisdiction.py index f3a746bf1..06cb9d8ac 100644 --- a/compass/pipeline/jurisdiction.py +++ b/compass/pipeline/jurisdiction.py @@ -34,7 +34,6 @@ def __init__( known_doc_urls=None, perform_se_search=True, perform_website_search=True, - validate_user_website_input=True, ): """ @@ -69,10 +68,6 @@ def __init__( perform_website_search : bool, optional Whether website-specific search and crawl steps should be performed for this jurisdiction. By default, ``True``. - validate_user_website_input : bool, optional - Whether user-supplied jurisdiction website inputs should be - validated before being used in collection. By default, - ``True``. """ self.runtime = runtime self.jurisdiction = jurisdiction @@ -82,7 +77,6 @@ def __init__( self.known_doc_urls = known_doc_urls self.perform_se_search = perform_se_search self.perform_website_search = perform_website_search - self.validate_user_website_input = validate_user_website_input self.jurisdiction_website = jurisdiction.website_url self.last_scrape_results = [] self.extraction_workflow = DocumentExtraction(self) From 58f00c595b711ae8c58426ccc658549ad2225106 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:07:09 -0600 Subject: [PATCH 06/37] Always validate a website from the search engines --- compass/scripts/download.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/compass/scripts/download.py b/compass/scripts/download.py index 9e17577e0..218e766b4 100644 --- a/compass/scripts/download.py +++ b/compass/scripts/download.py @@ -173,7 +173,6 @@ async def find_jurisdiction_website( browser_semaphore=None, usage_tracker=None, url_ignore_substrings=None, - validate=True, **kwargs, ): """Search for the main landing page of a given jurisdiction @@ -213,12 +212,6 @@ async def find_jurisdiction_website( url_ignore_substrings : list of str, optional URL substrings that should be excluded from search results. Substrings are applied case-insensitively. By default, ``None``. - validate : bool, default=True - If ``True``, each potential jurisdiction website will be checked - for validity using the - :class:`~compass.validation.location.JurisdictionWebsiteValidator` - before being returned. If ``False``, the first potential website - will be returned without validation. By default, ``True``. **kwargs Additional arguments forwarded to :func:`elm.web.search.run.search_with_fallback`. @@ -250,9 +243,6 @@ async def find_jurisdiction_website( if not potential_website_links: return None - if not validate: - return potential_website_links.pop() - model_config = model_configs.get( LLMTasks.JURISDICTION_MAIN_WEBSITE_VALIDATION, model_configs[LLMTasks.DEFAULT], From 3e55b0127aa6674ec3ddfe3781daa7fd3601acef Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:12:50 -0600 Subject: [PATCH 07/37] Fix test --- tests/python/unit/scripts/test_download.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/python/unit/scripts/test_download.py b/tests/python/unit/scripts/test_download.py index 1434ef5eb..07f8400b9 100644 --- a/tests/python/unit/scripts/test_download.py +++ b/tests/python/unit/scripts/test_download.py @@ -13,7 +13,7 @@ async def test_find_jurisdiction_website_returns_base_domain(monkeypatch): """Return the canonical root URL for the selected website""" - async def fake_search_with_fallback(**_kwargs): + async def fake_search_with_fallback(**_kwargs): # noqa return [ "https://prattvilleal.gov/venue/autauga-county-commission/", "https://prattvilleal.gov/government/mayor", @@ -48,9 +48,7 @@ async def check(self, url, jurisdiction): ) out = await download_module.find_jurisdiction_website( - jurisdiction, - {LLMTasks.DEFAULT: model_config}, - validate=True, + jurisdiction, {LLMTasks.DEFAULT: model_config} ) assert out == "https://prattvilleal.gov/" From b2c42da1e5a9d7618cc92679c376dff34f00e3ab Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:13:03 -0600 Subject: [PATCH 08/37] website input form user now assumed to be correct --- compass/pipeline/collection/steps.py | 104 +++++++-------------------- 1 file changed, 27 insertions(+), 77 deletions(-) diff --git a/compass/pipeline/collection/steps.py b/compass/pipeline/collection/steps.py index a9ca61c27..ad6b7b67b 100644 --- a/compass/pipeline/collection/steps.py +++ b/compass/pipeline/collection/steps.py @@ -13,8 +13,7 @@ find_jurisdiction_website, load_known_docs, ) -from compass.validation.location import JurisdictionWebsiteValidator -from compass.utilities.enums import LLMTasks, COMPASSDocumentCollectionStep +from compass.utilities.enums import COMPASSDocumentCollectionStep from compass.utilities.url import base_website_url from compass.pb import COMPASS_PB @@ -258,24 +257,21 @@ async def collect(self, workflow): # noqa: PLR6301 if not workflow.perform_website_search: return [] + + await _validate_jurisdiction_website(workflow) if not workflow.jurisdiction_website: - await try_set_website_from_jurisdiction(workflow) - if not workflow.jurisdiction_website: - logger.debug( - "No jurisdiction website found for %r; skipping " - "website document collection", - workflow.jurisdiction.full_name, - ) - return [] + logger.debug( + "No jurisdiction website found for %r; skipping " + "ELM website document collection", + workflow.jurisdiction.full_name, + ) + return [] logger.debug( "Collecting documents using ELM web crawl for: %s", workflow.jurisdiction.full_name, ) try: - workflow.jurisdiction_website = await get_redirected_url( - workflow.jurisdiction_website, timeout=30 - ) out = await download_jurisdiction_ordinances_from_website( workflow.jurisdiction_website, heuristic=await workflow.extractor.get_heuristic(), @@ -342,15 +338,15 @@ async def collect(self, workflow): # noqa: PLR6301 """ if not workflow.perform_website_search: return [] + + await _validate_jurisdiction_website(workflow) if not workflow.jurisdiction_website: - await try_set_website_from_jurisdiction(workflow) - if not workflow.jurisdiction_website: - logger.debug( - "No jurisdiction website found for %r; skipping " - "website document collection", - workflow.jurisdiction.full_name, - ) - return [] + logger.debug( + "No jurisdiction website found for %r; skipping " + "COMPASS website document collection", + workflow.jurisdiction.full_name, + ) + return [] logger.debug( "Collecting documents using COMPASS web crawl for: %s", @@ -384,65 +380,20 @@ async def collect(self, workflow): # noqa: PLR6301 return docs -async def try_set_website_from_jurisdiction(workflow): - """Resolve the website URL for this jurisdiction - - Parameters - ---------- - workflow : compass.pipeline.jurisdiction.SingleJurisdictionRun - The workflow for the jurisdiction being processed, which may or - may not have a user-supplied website URL. If the workflow - doesn't have a website URL, this function will attempt to find - one. - """ - if workflow.jurisdiction_website: - if workflow.validate_user_website_input: - await _validate_jurisdiction_website(workflow) - else: - workflow.jurisdiction_website = await _get_base_website( - workflow.jurisdiction_website - ) - - if not workflow.jurisdiction_website: - website = await _find_jurisdiction_website_for_workflow(workflow) - if website: - workflow.jurisdiction_website = website - - async def _validate_jurisdiction_website(workflow): - """Validate a user-supplied jurisdiction website""" - if workflow.jurisdiction_website is None: - return + """Try to set and resolve the website URL for this jurisdiction""" + if workflow.jurisdiction_website: + workflow.jurisdiction_website = await _get_base_website( + workflow.jurisdiction_website + ) - workflow.jurisdiction_website = await _get_base_website( - workflow.jurisdiction_website, - ) - if workflow.jurisdiction_website is None: + # only try to find a website if we don't have one and we have LLMs + # we can use for validation + if workflow.jurisdiction_website or not workflow.runtime.models: return - COMPASS_PB.update_jurisdiction_task( - workflow.jurisdiction.full_name, - description=( - f"Validating user input website: {workflow.jurisdiction_website}" - ), - ) - model_config = workflow.runtime.models.get( - LLMTasks.DOCUMENT_JURISDICTION_VALIDATION, - workflow.runtime.models[LLMTasks.DEFAULT], - ) - validator = JurisdictionWebsiteValidator( - browser_semaphore=workflow.runtime.browser_semaphore, - file_loader_kwargs=workflow.runtime.file_loader_kwargs_no_ocr, - usage_tracker=workflow.usage_tracker, - llm_service=model_config.llm_service, - **model_config.llm_call_kwargs, - ) - is_website_correct = await validator.check( - workflow.jurisdiction_website, - workflow.jurisdiction, - ) - if not is_website_correct: - workflow.jurisdiction_website = None + if website := await _find_jurisdiction_website_for_workflow(workflow): + workflow.jurisdiction_website = await _get_base_website(website) async def _get_base_website(website): @@ -468,7 +419,6 @@ async def _find_jurisdiction_website_for_workflow(workflow): search_semaphore=workflow.runtime.search_engine_semaphore, browser_semaphore=workflow.runtime.browser_semaphore, usage_tracker=workflow.usage_tracker, - validate=workflow.validate_user_website_input, url_ignore_substrings=( workflow.runtime.search_params.url_ignore_substrings ), From ef1bad117369ac30de096e87692ae076ddd63337 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:13:18 -0600 Subject: [PATCH 09/37] New func signature --- support/jurisdictions/update_jur_websites.py | 1 - 1 file changed, 1 deletion(-) diff --git a/support/jurisdictions/update_jur_websites.py b/support/jurisdictions/update_jur_websites.py index 9a2d2e33f..8e04e59b5 100755 --- a/support/jurisdictions/update_jur_websites.py +++ b/support/jurisdictions/update_jur_websites.py @@ -457,7 +457,6 @@ async def _find_jurisdiction_website_for_jurisdiction( search_semaphore=search_semaphore, browser_semaphore=browser_semaphore, usage_tracker=usage_tracker, - validate=True, url_ignore_substrings=search_params.url_ignore_substrings, **search_params.se_kwargs, ) From 66b614accad522acd619371b03e20552ac6e9f04 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:57:12 -0600 Subject: [PATCH 10/37] Add tests --- .../test_pipeline_collection_steps.py | 128 +++++++++++++++++- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/tests/python/unit/pipeline/test_pipeline_collection_steps.py b/tests/python/unit/pipeline/test_pipeline_collection_steps.py index 0e1c2d17b..ce4e79048 100644 --- a/tests/python/unit/pipeline/test_pipeline_collection_steps.py +++ b/tests/python/unit/pipeline/test_pipeline_collection_steps.py @@ -38,9 +38,14 @@ async def check(self, website, jurisdiction): return True -def _build_workflow(): +def _build_workflow(*, website="https://example.com", models=None): """Build a minimal workflow for collection-step tests""" model_config = SimpleNamespace(llm_service=object(), llm_call_kwargs={}) + if models is None: + models = { + LLMTasks.DEFAULT: model_config, + LLMTasks.DOCUMENT_JURISDICTION_VALIDATION: model_config, + } runtime = SimpleNamespace( file_loader_kwargs={ "pdf_ocr_read_coroutine": object(), @@ -49,14 +54,16 @@ def _build_workflow(): file_loader_kwargs_no_ocr={"loader_mode": "no-ocr"}, crawl_semaphore=None, browser_semaphore=None, - models={ - LLMTasks.DEFAULT: model_config, - LLMTasks.DOCUMENT_JURISDICTION_VALIDATION: model_config, - }, + search_engine_semaphore=None, + search_params=SimpleNamespace( + url_ignore_substrings=(), + se_kwargs={}, + ), + models=models, ) return SimpleNamespace( perform_website_search=True, - jurisdiction_website="https://example.com", + jurisdiction_website=website, jurisdiction=SimpleNamespace(full_name="Example Township"), extractor=_DummyExtractor(), runtime=runtime, @@ -121,5 +128,114 @@ async def fake_download(url, **kwargs): # noqa assert captured["already_visited"] == {"https://seen.example"} +@pytest.mark.asyncio +async def test_elm_website_crawl_uses_provided_website_without_discovery( + monkeypatch, +): + """Provided jurisdiction websites should bypass discovery""" + workflow = _build_workflow(website="https://user-provided.example/path") + captured = {} + + async def fake_get_base_website(url): # noqa + return "https://user-provided.example" + + async def fail_if_discovery_called(workflow): # noqa + raise AssertionError("website discovery should not be attempted") + + async def fake_download(url, **kwargs): # noqa + captured["url"] = url + return [], [] + + monkeypatch.setattr( + steps_module, "_get_base_website", fake_get_base_website + ) + monkeypatch.setattr( + steps_module, + "_find_jurisdiction_website_for_workflow", + fail_if_discovery_called, + ) + monkeypatch.setattr( + steps_module, + "download_jurisdiction_ordinances_from_website", + fake_download, + ) + + docs = await ElmWebsiteCrawlStep().collect(workflow) + + assert docs == [] + assert captured["url"] == "https://user-provided.example" + assert workflow.jurisdiction_website == "https://user-provided.example" + + +@pytest.mark.asyncio +async def test_elm_website_crawl_attempts_discovery_when_models_present( + monkeypatch, +): + """Missing jurisdiction websites should be discovered when models exist""" + workflow = _build_workflow(website=None) + calls = {"discover": 0} + captured = {} + + async def fake_discover(workflow): # noqa + calls["discover"] += 1 + return "https://discovered.example/home" + + async def fake_get_base_website(url): # noqa + return "https://discovered.example" + + async def fake_download(url, **kwargs): # noqa + captured["url"] = url + return [], [] + + monkeypatch.setattr( + steps_module, + "_find_jurisdiction_website_for_workflow", + fake_discover, + ) + monkeypatch.setattr( + steps_module, "_get_base_website", fake_get_base_website + ) + monkeypatch.setattr( + steps_module, + "download_jurisdiction_ordinances_from_website", + fake_download, + ) + + docs = await ElmWebsiteCrawlStep().collect(workflow) + + assert docs == [] + assert calls["discover"] == 1 + assert captured["url"] == "https://discovered.example" + assert workflow.jurisdiction_website == "https://discovered.example" + + +@pytest.mark.asyncio +async def test_elm_website_crawl_skips_discovery_without_models(monkeypatch): + """Missing websites should short-circuit when no models are available""" + workflow = _build_workflow(website=None, models={}) + + async def fail_if_discovery_called(workflow): # noqa + raise AssertionError("website discovery should not be attempted") + + async def fail_if_download_called(url, **kwargs): # noqa + raise AssertionError("website crawl should not be attempted") + + monkeypatch.setattr( + steps_module, + "_find_jurisdiction_website_for_workflow", + fail_if_discovery_called, + ) + monkeypatch.setattr( + steps_module, + "download_jurisdiction_ordinances_from_website", + fail_if_download_called, + ) + + docs = await ElmWebsiteCrawlStep().collect(workflow) + + assert docs == [] + assert workflow.jurisdiction_website is None + + if __name__ == "__main__": pytest.main(["-q", "--show-capture=all", Path(__file__), "-rapP"]) From f7f5d9f12de7217fde09464017c563f05aee005c Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 16:58:04 -0600 Subject: [PATCH 11/37] Update func name --- compass/pipeline/collection/steps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compass/pipeline/collection/steps.py b/compass/pipeline/collection/steps.py index ad6b7b67b..a4f3a42e8 100644 --- a/compass/pipeline/collection/steps.py +++ b/compass/pipeline/collection/steps.py @@ -258,7 +258,7 @@ async def collect(self, workflow): # noqa: PLR6301 if not workflow.perform_website_search: return [] - await _validate_jurisdiction_website(workflow) + await _resolve_jurisdiction_website(workflow) if not workflow.jurisdiction_website: logger.debug( "No jurisdiction website found for %r; skipping " @@ -339,7 +339,7 @@ async def collect(self, workflow): # noqa: PLR6301 if not workflow.perform_website_search: return [] - await _validate_jurisdiction_website(workflow) + await _resolve_jurisdiction_website(workflow) if not workflow.jurisdiction_website: logger.debug( "No jurisdiction website found for %r; skipping " @@ -380,7 +380,7 @@ async def collect(self, workflow): # noqa: PLR6301 return docs -async def _validate_jurisdiction_website(workflow): +async def _resolve_jurisdiction_website(workflow): """Try to set and resolve the website URL for this jurisdiction""" if workflow.jurisdiction_website: workflow.jurisdiction_website = await _get_base_website( From cf94b6ab5807e5954b0d644de73aea8b5a711bba Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 16 Jul 2026 17:28:59 -0600 Subject: [PATCH 12/37] Fix a jur --- compass/data/conus_jurisdictions.csv | 2 +- .../jurisdictions/compile_jurisdiction_gpkg.ipynb | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/compass/data/conus_jurisdictions.csv b/compass/data/conus_jurisdictions.csv index 6f0f09344..0a3469ae0 100644 --- a/compass/data/conus_jurisdictions.csv +++ b/compass/data/conus_jurisdictions.csv @@ -6559,7 +6559,7 @@ Florida,Santa Rosa,Skyline,city,1211393127, Florida,Santa Rosa,Whiting Field,city,1211393659, Florida,Santa Rosa,,county,12113,https://www.santarosa.fl.gov/ Florida,Sarasota,Englewood,city,1211590975, -Florida,Sarasota,Interior,county,1211591599,https://www.scgov.net/ +Florida,Sarasota,Interior,region,1211591599,https://www.scgov.net/ Florida,Sarasota,Longboat Key,town,1241150,https://www.longboatkey.org/ Florida,Sarasota,North Port,city,1249675,https://www.northportfl.gov/ Florida,Sarasota,Osprey-Laurel-Nokomis,city,1211592542, diff --git a/support/jurisdictions/compile_jurisdiction_gpkg.ipynb b/support/jurisdictions/compile_jurisdiction_gpkg.ipynb index 6cdf579d6..da4008ec2 100755 --- a/support/jurisdictions/compile_jurisdiction_gpkg.ipynb +++ b/support/jurisdictions/compile_jurisdiction_gpkg.ipynb @@ -921,7 +921,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "64ce222a", "metadata": {}, "outputs": [], @@ -940,7 +940,15 @@ " & (ord_areas.county_name == \"Washoe\")\n", " & (ord_areas.subd_name == \"Verdi\"),\n", " \"subd_fips\"\n", - "] = \"0603194840\"\n" + "] = \"0603194840\"\n", + "\n", + "\n", + "ord_areas.loc[\n", + " (ord_areas.state_name == \"Florida\")\n", + " & (ord_areas.county_name == \"Sarasota\")\n", + " & (ord_areas.subd_name == \"Interior\"),\n", + " \"subd_type\"\n", + "] = \"region\"\n" ] }, { From 042b6bf452132d95049741c6375ef3bad756aaff Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 16:02:23 -0600 Subject: [PATCH 13/37] Stricter de-duplication --- compass/pipeline/collection/dedupe.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compass/pipeline/collection/dedupe.py b/compass/pipeline/collection/dedupe.py index 2c5f2475c..5af43b5a1 100644 --- a/compass/pipeline/collection/dedupe.py +++ b/compass/pipeline/collection/dedupe.py @@ -39,10 +39,9 @@ def add_docs(self, docs, *, step_name, jurisdiction_name): except KeyError: key = _collection_doc_key(doc, use_fallback=True) - if key not in self._docs: - self._docs[key] = {"doc": doc, "from_steps": []} - - self._docs[key]["from_steps"].append(step_name) + entry = self._docs.setdefault(key, {"doc": doc, "from_steps": []}) + if step_name not in entry["from_steps"]: + entry["from_steps"].append(step_name) @property def values(self): From 17a8a1fee0f8d45798f6c699b8fb2abfdd0e4407 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 16:03:13 -0600 Subject: [PATCH 14/37] Minor cleanup --- compass/pipeline/collection/dedupe.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/compass/pipeline/collection/dedupe.py b/compass/pipeline/collection/dedupe.py index 5af43b5a1..89bb3abb1 100644 --- a/compass/pipeline/collection/dedupe.py +++ b/compass/pipeline/collection/dedupe.py @@ -34,11 +34,7 @@ def add_docs(self, docs, *, step_name, jurisdiction_name): logger.debug("Adding %d doc(s) to collection", len(docs)) for doc in docs: doc.attrs.setdefault("jurisdiction_name", jurisdiction_name) - try: - key = _collection_doc_key(doc) - except KeyError: - key = _collection_doc_key(doc, use_fallback=True) - + key = _collection_doc_key(doc) entry = self._docs.setdefault(key, {"doc": doc, "from_steps": []}) if step_name not in entry["from_steps"]: entry["from_steps"].append(step_name) @@ -52,9 +48,11 @@ def __bool__(self): return bool(self._docs) -def _collection_doc_key(doc, use_fallback=False): +def _collection_doc_key(doc): """Build the deduplication key for a collected document""" - if use_fallback: + try: + return str(doc.attrs["checksum"]) + except KeyError: return str( doc.attrs.get("checksum") or doc.attrs.get("source_fp") @@ -62,4 +60,3 @@ def _collection_doc_key(doc, use_fallback=False): or doc.attrs.get("cache_fn") or id(doc) ) - return str(doc.attrs["checksum"]) From 3c130ce7d2fb8b8609da3af96d7624d2a5e7e6a7 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 16:09:58 -0600 Subject: [PATCH 15/37] Don't count inputs that have no docs --- compass/pipeline/collection/persistence.py | 13 +++++++++---- compass/pipeline/coordinator.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/compass/pipeline/collection/persistence.py b/compass/pipeline/collection/persistence.py index 5acef889d..f0ce21e80 100644 --- a/compass/pipeline/collection/persistence.py +++ b/compass/pipeline/collection/persistence.py @@ -33,10 +33,10 @@ def build_collection_manifest( tech : str Technology specified in the pipeline request, included in the manifest for compatibility validation when loading. - jurisdictions : dict - Dictionary mapping jurisdiction full names to serialized - collection metadata for each jurisdiction, including - jurisdiction identifiers and the persisted document records. + jurisdictions : list + List of serialized collection metadata for each jurisdiction, + including jurisdiction identifiers and the persisted document + records. time_start_utc : datetime.datetime UTC datetime when the collection process started, used to calculate elapsed time for the manifest metadata. @@ -52,6 +52,11 @@ def build_collection_manifest( """ time_end_utc = datetime.now(UTC) time_elapsed = time_end_utc - time_start_utc + jurisdictions = [ + info + for info in jurisdictions + if info is not None and info.get("documents") + ] return { "tech": tech, "time_start_utc": time_start_utc.isoformat(), diff --git a/compass/pipeline/coordinator.py b/compass/pipeline/coordinator.py index f7c3198f3..7b890d3d7 100644 --- a/compass/pipeline/coordinator.py +++ b/compass/pipeline/coordinator.py @@ -254,7 +254,7 @@ async def run(self, jurisdictions_df): collection_infos = await asyncio.gather(*tasks) manifest = build_collection_manifest( self.runtime.tech, - list(filter(None, collection_infos)), + collection_infos, start_date, len(jurisdictions_df), ) From e0985a225bac28bf27d509865caf3e65c26b5c1d Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 18:11:27 -0600 Subject: [PATCH 16/37] Add debug logging --- compass/web/website_crawl.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index fb1b4bf04..51e8c97a0 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -244,6 +244,11 @@ async def _run( """Recursive web crawl function""" if link is None: base_url, link = self._reset_crawl(base_url) + logger.debug( + "Starting COMPASS crawl for base URL: %s\nLink: %r", + base_url, + link, + ) if link in self._already_visited: return From b7a0a41de8e60f5bb2c4adcf82e3102559ca5b81 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 18:34:12 -0600 Subject: [PATCH 17/37] Fix bug that broke crawl --- compass/web/website_crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index 51e8c97a0..fbae17543 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -332,7 +332,7 @@ async def _website_link_is_doc(self, link, depth, score): # at this point the page is NOT a PDF. However, it could still # just be a normal webpage on the main domain that we haven't # visited before. In that case, just return False - if not link.consistent_domain: + if link.consistent_domain: return False # now we are on an external page that we either have not visited From 66cdaff194287b5376ec2e5fa33afdf332a7820a Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 18:34:25 -0600 Subject: [PATCH 18/37] Fix error message --- compass/web/website_crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index fbae17543..e81caeccf 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -475,7 +475,7 @@ async def _should_terminate_crawl( def _log_crawl_stats(self): """Log statistics about crawled pages and depths""" logger.info("Crawled %d pages", len(self._already_visited)) - logger.info("Found %d potential documents", len(self._out_docs)) + logger.info("Found %d potential document(s)", len(self._out_docs)) logger.debug("Average score: %.2f", self._compute_avg_link_score()) logger.debug("Pages crawled by depth:") From b520bfd449b469baffea2b0b140dac6a750b4b82 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 18:34:52 -0600 Subject: [PATCH 19/37] Log the docs that were found --- compass/web/website_crawl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index e81caeccf..3f320bad6 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -475,7 +475,11 @@ async def _should_terminate_crawl( def _log_crawl_stats(self): """Log statistics about crawled pages and depths""" logger.info("Crawled %d pages", len(self._already_visited)) - logger.info("Found %d potential document(s)", len(self._out_docs)) + logger.info( + "Found %d potential document(s):\n%r", + len(self._out_docs), + self._out_docs, + ) logger.debug("Average score: %.2f", self._compute_avg_link_score()) logger.debug("Pages crawled by depth:") From 339547623ae8970cadc9344b4f686fc650105141 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 18:49:13 -0600 Subject: [PATCH 20/37] Only keep documents where persistence was successful --- compass/pipeline/collection/persistence.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compass/pipeline/collection/persistence.py b/compass/pipeline/collection/persistence.py index f0ce21e80..2d9c637f6 100644 --- a/compass/pipeline/collection/persistence.py +++ b/compass/pipeline/collection/persistence.py @@ -217,7 +217,7 @@ async def persist_documents(jurisdiction, collected_docs, *, relative_to=None): "subdivision": jurisdiction.subdivision_name, "jurisdiction_type": jurisdiction.type, "FIPS": jurisdiction.code, - "documents": documents, + "documents": [doc for doc in documents if doc is not None], } @@ -315,6 +315,8 @@ def _make_relative(fp, relative_to): def _serialize_collection_doc_info(doc, from_steps): """Serialize a collected document for manifest storage""" serialized = dict(doc.attrs) + if not serialized or serialized.get("parsed_fp") is None: + return None serialized.pop("cache_fn", None) serialized.pop("cleaned_fps", None) serialized.setdefault("check_correct_jurisdiction", True) From f80c9b99dc3cd84f59fe4e2742c7cd3f794b1129 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:20:40 -0600 Subject: [PATCH 21/37] Explicitly pass stems to file moves --- compass/extraction/context.py | 4 ++-- compass/pipeline/collection/persistence.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/compass/extraction/context.py b/compass/extraction/context.py index f70aaa28e..d437b8f03 100644 --- a/compass/extraction/context.py +++ b/compass/extraction/context.py @@ -157,9 +157,9 @@ def multi_doc_context(self, attr_text_key=None): return f"## MULTI-DOCUMENT CONTEXT ##\n\n{serialized}" -async def _move_file_to_out_dir(doc, out_fn): +async def _move_file_to_out_dir(doc, out_stem): """Move PDF or HTML text file to output directory""" - out_fp = await FileMover.call(doc, out_fn) + out_fp = await FileMover.call(doc, out_stem) doc.attrs["out_fp"] = out_fp return doc diff --git a/compass/pipeline/collection/persistence.py b/compass/pipeline/collection/persistence.py index 2d9c637f6..6e787c169 100644 --- a/compass/pipeline/collection/persistence.py +++ b/compass/pipeline/collection/persistence.py @@ -201,7 +201,7 @@ async def persist_documents(jurisdiction, collected_docs, *, relative_to=None): task = asyncio.create_task( _persist_doc( info["doc"], - out_fn=f"{jurisdiction.full_name}_{index}", + out_stem=f"{jurisdiction.full_name}_{index}", from_steps=info["from_steps"], relative_to=relative_to, ), @@ -276,24 +276,24 @@ async def _load_single_doc(doc_info): return doc -async def _persist_doc(doc, out_fn, from_steps, relative_to): +async def _persist_doc(doc, out_stem, from_steps, relative_to): """Persist one collected document and its parsed text""" - await _move_file_to_collection_dir(doc, out_fn, relative_to) - await _persist_parsed_text(doc, out_fn, relative_to) + await _move_file_to_collection_dir(doc, out_stem, relative_to) + await _persist_parsed_text(doc, out_stem, relative_to) return _serialize_collection_doc_info(doc, from_steps) -async def _move_file_to_collection_dir(doc, out_fn, relative_to): +async def _move_file_to_collection_dir(doc, out_stem, relative_to): """Move a source file to the collection output directory""" - out_fp = await FileMover.call(doc, out_fn, "downloaded") + out_fp = await FileMover.call(doc, out_stem, "downloaded") if relative_to is not None and out_fp is not None: out_fp = _make_relative(out_fp, relative_to) doc.attrs["source_fp"] = out_fp -async def _persist_parsed_text(doc, out_fn, relative_to): +async def _persist_parsed_text(doc, out_stem, relative_to): """Write parsed text for a collected document""" - out_fp = await ParsedFileWriter.call(doc, out_fn) + out_fp = await ParsedFileWriter.call(doc, out_stem) if relative_to is not None and out_fp is not None: out_fp = _make_relative(out_fp, relative_to) doc.attrs["parsed_fp"] = out_fp From 0d51d487df6c70f347e0576527347b551b228017 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:20:54 -0600 Subject: [PATCH 22/37] Fix tests --- .../unit/services/test_services_threaded.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/python/unit/services/test_services_threaded.py b/tests/python/unit/services/test_services_threaded.py index 506fa52f9..66cc220c3 100644 --- a/tests/python/unit/services/test_services_threaded.py +++ b/tests/python/unit/services/test_services_threaded.py @@ -201,6 +201,32 @@ def test_move_file_uses_jurisdiction_name(tmp_path): assert not cached_fp.exists() +def test_output_filenames_preserve_index_after_st_abbreviation(tmp_path): + """Periods in names like ``St.`` should not drop indexed suffixes""" + + cached_dir = tmp_path / "cached" + cached_dir.mkdir() + out_dir = tmp_path / "output" + out_dir.mkdir() + + cached_fp = cached_dir / "download.md" + cached_fp.write_text("content", encoding="utf-8") + + doc = HTMLDocument(["payload"]) + doc.attrs["cache_fn"] = cached_fp + + date = datetime.now().strftime("%Y_%m_%d") + out_stem = "City of St. Paul, Alaska_1" + + moved_fp = threaded._move_file(doc, out_dir, out_stem=out_stem) + parsed_fp = threaded._write_parsed_text(doc, out_dir, out_stem=out_stem) + + assert moved_fp.name == f"City_of_St._Paul_Alaska_1_processed_{date}.md" + assert parsed_fp.name == "City_of_St._Paul_Alaska_1.txt" + assert moved_fp.read_text(encoding="utf-8") == "content" + assert parsed_fp.read_text(encoding="utf-8").strip() == "payload" + + def test_move_file_handles_extensionless_cached_file(tmp_path): """Verify `_move_file` handles cached files without an extension""" @@ -243,7 +269,7 @@ def test_write_cleaned_file_with_debug(tmp_path): } CLEANED_FP_REGISTRY["cleaned_file_test"] = fp_names - try: # noqa + try: outputs = threaded._write_cleaned_file( doc, tmp_path, From ea531a9672c45970f7dec95ad1df51f768f1ede9 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:21:22 -0600 Subject: [PATCH 23/37] Remove periods from file stem --- compass/services/threaded.py | 37 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/compass/services/threaded.py b/compass/services/threaded.py index 71858537e..619fc5a49 100644 --- a/compass/services/threaded.py +++ b/compass/services/threaded.py @@ -51,7 +51,23 @@ def _compute_sha256(file_path): return f"sha256:{m.hexdigest()}" -def _move_file(doc, out_dir, out_fn=None, verb="processed"): +def _normalize_output_stem(out_stem): + """Normalize an output file name while preserving the full stem""" + return ( + out_stem.replace(".", "") + .replace(",", "") + .replace("/", "_") + .replace(" ", "_") + ) + + +def _ensure_output_suffix(out_dir, out_stem, suffix): + """Build output path""" + out_stem = _normalize_output_stem(out_stem) + return Path(out_dir) / f"{out_stem}{suffix}" + + +def _move_file(doc, out_dir, out_stem=None, verb="processed"): """Move a file from a temp directory to an output directory""" cached_fp = doc.attrs.get("cache_fn") if cached_fp is None: @@ -59,13 +75,9 @@ def _move_file(doc, out_dir, out_fn=None, verb="processed"): cached_fp = Path(cached_fp) date = datetime.now().strftime("%Y_%m_%d") - out_fn = out_fn or cached_fp.stem - out_fn = out_fn.replace(",", "").replace("/", "_").replace(" ", "_") - out_fn = f"{out_fn}_{verb}_{date}" - out_fp = Path(out_dir) / out_fn - - if out_fp.suffix != cached_fp.suffix: - out_fp = out_fp.with_suffix(cached_fp.suffix) + out_stem = out_stem or cached_fp.stem + out_stem = f"{out_stem}_{verb}_{date}" + out_fp = _ensure_output_suffix(out_dir, out_stem, cached_fp.suffix) shutil.move(cached_fp, out_fp) return out_fp @@ -92,15 +104,12 @@ def _write_cleaned_file(doc, out_dir, tech, jurisdiction_name=None): return out_paths -def _write_parsed_text(doc, out_dir, out_fn=None): +def _write_parsed_text(doc, out_dir, out_stem=None): """Write parsed document text to directory""" - if not doc.text or out_fn is None: + if not doc.text or out_stem is None: return None - out_fn = out_fn.replace(",", "").replace("/", "_").replace(" ", "_") - out_fp = Path(out_dir) / out_fn - if out_fp.suffix != ".txt": - out_fp = out_fp.with_suffix(".txt") + out_fp = _ensure_output_suffix(out_dir, out_stem, ".txt") out_fp.write_text(doc.text, encoding="utf-8") return out_fp From 05fd2ab98ccdd972f482872a190e6c89362a960e Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:23:28 -0600 Subject: [PATCH 24/37] Add test --- .../test_pipeline_collection_dedupe.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/python/unit/pipeline/test_pipeline_collection_dedupe.py diff --git a/tests/python/unit/pipeline/test_pipeline_collection_dedupe.py b/tests/python/unit/pipeline/test_pipeline_collection_dedupe.py new file mode 100644 index 000000000..c00c2467d --- /dev/null +++ b/tests/python/unit/pipeline/test_pipeline_collection_dedupe.py @@ -0,0 +1,36 @@ +"""Tests for collection document de-duplication""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from compass.pipeline.collection.dedupe import DocumentDeDuplicator + + +def test_add_docs_keeps_from_steps_unique_for_same_doc_and_step(): + """Repeated docs from one step should only record that step once""" + deduplicator = DocumentDeDuplicator() + doc = SimpleNamespace(attrs={"checksum": "abc123"}) + + deduplicator.add_docs( + [doc, doc], + step_name="Look for document on jurisdiction website", + jurisdiction_name="Example Township", + ) + deduplicator.add_docs( + [doc], + step_name="Look for document on jurisdiction website", + jurisdiction_name="Example Township", + ) + + values = list(deduplicator.values) + + assert len(values) == 1 + assert values[0]["from_steps"] == [ + "Look for document on jurisdiction website" + ] + + +if __name__ == "__main__": + pytest.main(["-q", "--show-capture=all", Path(__file__), "-rapP"]) From f3b6976ef96cbdb5383f7abfcf1af3fc8271e59e Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:28:17 -0600 Subject: [PATCH 25/37] Add phrases --- compass/extraction/ghp/plugin_config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compass/extraction/ghp/plugin_config.yaml b/compass/extraction/ghp/plugin_config.yaml index 577c8eb15..834ad84cc 100644 --- a/compass/extraction/ghp/plugin_config.yaml +++ b/compass/extraction/ghp/plugin_config.yaml @@ -93,6 +93,9 @@ heuristic_keywords: - "geothermal production project" - "exploratory well" - "injection well" + - "wellbeing" + - "well-being" + - "as well as" collection_prompts: True From 52162a84cf5d9c8211c3d969621f0270ea0416f0 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:32:34 -0600 Subject: [PATCH 26/37] HTML docs properly cached --- compass/web/website_crawl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index 3f320bad6..29d9c457b 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -394,8 +394,13 @@ async def _website_link_as_html_doc(self, link, depth, score): logger.debug("Loading Link as HTML: %s", link) html_text = await self._get_text_no_err(link.href) - attrs = {_DEPTH_KEY: depth, _SCORE_KEY: score} + attrs = {_DEPTH_KEY: depth, _SCORE_KEY: score, "source": link.href} doc = HTMLDocument([html_text], attrs=attrs) + + cache_fn = await TempFileCache.call(doc, doc.text) + if cache_fn is not None: + doc.attrs["cache_fn"] = cache_fn + self._out_docs.append(doc) return True From c517db1265f37cfd34b01920fd4ee7f1a9d19254 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:37:40 -0600 Subject: [PATCH 27/37] Use fast vs final AFL --- compass/web/website_crawl.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index 29d9c457b..b57f4cb49 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -18,9 +18,11 @@ from playwright._impl._errors import Error as PlaywrightError # noqa: PLC2701 from elm.web.utilities import pw_page from elm.web.document import HTMLDocument +from elm.web.file_loader import AsyncWebFileLoader from elm.web.website_crawl import ELMLinkScorer, _SCORE_KEY # noqa: PLC2701 -from compass.utilities.url import sanitize_url +from compass.utilities.url import sanitize_url +from compass.services.threaded import TempFileCache from compass.web.file_loader import COMPASSWebFileLoader from compass.utilities.parsing import is_pdf_doc @@ -176,7 +178,13 @@ def __init__( file_loader_kwargs = file_loader_kwargs or {} flk = {"verify_ssl": False} flk.update(file_loader_kwargs or {}) - self.afl = COMPASSWebFileLoader(**flk) + + # Fast file loader that always uses poppler + self.fast_afl = AsyncWebFileLoader(**flk) + + # best parsing file loader selected by user + self.final_afl = COMPASSWebFileLoader(**flk) + self.pw_launch_kwargs = ( file_loader_kwargs.get("pw_launch_kwargs") or {} ) @@ -287,6 +295,7 @@ async def _run( if doc_was_just_found: if await self.validator(self._out_docs[-1]): logger.debug(" - Document passed validation check!") + self._load_last_doc_with_final_afl(next_link["href"]) else: self._out_docs = self._out_docs[:-1] elif ( @@ -308,6 +317,26 @@ async def _run( return + async def _load_last_doc_with_final_afl(self, link): + """Load the last document with the final file loader""" + old_doc = self._out_docs[-1] + link = old_doc.attrs.get("source", link) + try: + doc = await self.final_afl.fetch(link) + doc.attrs[_DEPTH_KEY] = old_doc.attrs[_DEPTH_KEY] + doc.attrs[_SCORE_KEY] = old_doc.attrs[_SCORE_KEY] + if not doc.empty: + self._out_docs[-1] = doc + except KeyboardInterrupt: + raise + except Exception as e: + msg = ( + "Encountered error of type %r while trying to fetch " + "content from %s" + ) + err_type = type(e) + logger.exception(msg, err_type, link) + def _reset_crawl(self, base_url): """Reset crawl state and initialize crawling link""" self._out_docs = [] @@ -366,7 +395,7 @@ async def _website_link_is_pdf(self, link, depth, score): logger.debug("Loading Link: %s", link) try: - doc = await self.afl.fetch(link.href) + doc = await self.fast_afl.fetch(link.href) except KeyboardInterrupt: raise except Exception as e: From 5730f0039e08d4286022204502c13209ebbe369f Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:42:48 -0600 Subject: [PATCH 28/37] Add logging --- compass/pipeline/collection/steps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compass/pipeline/collection/steps.py b/compass/pipeline/collection/steps.py index a4f3a42e8..01b79a8e6 100644 --- a/compass/pipeline/collection/steps.py +++ b/compass/pipeline/collection/steps.py @@ -292,6 +292,8 @@ async def collect(self, workflow): # noqa: PLR6301 return [] docs, scrape_results = out + logger.debug("Found the following docs with ELM crawl:\n%r", docs) + workflow.last_scrape_results = scrape_results for doc in docs: doc.attrs["compass_crawl"] = False From 57cb76e12dbdd9774d1c30c4efa72037e3079c94 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:43:21 -0600 Subject: [PATCH 29/37] Add logging --- compass/pipeline/collection/steps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compass/pipeline/collection/steps.py b/compass/pipeline/collection/steps.py index 01b79a8e6..fcb9fbe22 100644 --- a/compass/pipeline/collection/steps.py +++ b/compass/pipeline/collection/steps.py @@ -376,6 +376,7 @@ async def collect(self, workflow): # noqa: PLR6301 ) return [] + logger.debug("Found the following docs with COMPASS crawl:\n%r", docs) for doc in docs: doc.attrs["compass_crawl"] = True doc.attrs["check_correct_jurisdiction"] = True From b7213fdc5f0b7c4743d0505da3c0c4c2504ba8a4 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:43:49 -0600 Subject: [PATCH 30/37] Revert logging call --- compass/web/website_crawl.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index b57f4cb49..f69c79ff3 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -509,11 +509,7 @@ async def _should_terminate_crawl( def _log_crawl_stats(self): """Log statistics about crawled pages and depths""" logger.info("Crawled %d pages", len(self._already_visited)) - logger.info( - "Found %d potential document(s):\n%r", - len(self._out_docs), - self._out_docs, - ) + logger.info("Found %d potential document(s)", len(self._out_docs)) logger.debug("Average score: %.2f", self._compute_avg_link_score()) logger.debug("Pages crawled by depth:") From 11d06826efb59158c4deabc852a66730fb3819d8 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:44:13 -0600 Subject: [PATCH 31/37] ELM crawl can now reload documents using fast and good file loaders --- compass/scripts/download.py | 49 +++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/compass/scripts/download.py b/compass/scripts/download.py index 218e766b4..0abe23b83 100644 --- a/compass/scripts/download.py +++ b/compass/scripts/download.py @@ -10,6 +10,7 @@ ELMWebsiteCrawler, ELMLinkScorer, ) +from elm.web.file_loader import AsyncWebFileLoader from elm.web.utilities import filter_documents from compass.web.search import search_single_jurisdiction @@ -363,10 +364,16 @@ async def _crawl_hook(*__, **___): # noqa: RUF029 "kwargs for COMPASSWebFileLoader:\n%s", pprint.PrettyPrinter().pformat(flk), ) - afl = COMPASSWebFileLoader(**flk) + + # Fast file loader that always uses poppler + fast_afl = AsyncWebFileLoader(**flk) + + # best parsing file loader selected by user + final_afl = COMPASSWebFileLoader(**flk) + crawler = ELMWebsiteCrawler( validator=_doc_heuristic, - async_file_loader=afl, + async_file_loader=fast_afl, url_scorer=ELMLinkScorer(keyword_points).score, browser_config_kwargs=browser_config_kwargs, crawler_config_kwargs=crawler_config_kwargs, @@ -395,11 +402,10 @@ async def _crawl_hook(*__, **___): # noqa: RUF029 if return_c4ai_results: docs, c4ai_results = docs_or_pair - _sanitize_doc_sources(docs) + docs = await _finalize_doc_sources(docs, final_afl) return docs, c4ai_results - _sanitize_doc_sources(docs_or_pair) - return docs_or_pair + return await _finalize_doc_sources(docs_or_pair, final_afl) async def download_jurisdiction_ordinances_from_website_compass_crawl( @@ -867,8 +873,8 @@ async def _contains_relevant_text( return found_text -def _sanitize_doc_sources(docs): - """Rewrite source attrs on documents returned by ELMWebsiteCrawler +async def _finalize_doc_sources(docs, final_afl): + """Finalize documents returned by ELMWebsiteCrawler crawl4ai can surface PDF URLs containing raw spaces (e.g. filenames like "Land Use Code.pdf"). These fail when the file loader issues @@ -881,6 +887,35 @@ def _sanitize_doc_sources(docs): if source and " " in source: doc.attrs["source"] = sanitize_url(source) + return await _reload_using_final_afl(docs, final_afl) + + +async def _reload_using_final_afl(docs, final_afl): + """Reload documents using the final AsyncFileLoader""" + out_docs = [] + for old_doc in docs: + link = old_doc.attrs.get("source") + if not link: + out_docs.append(old_doc) + continue + + try: + doc = await final_afl.fetch(link) + doc.attrs[_SCORE_KEY] = old_doc.attrs[_SCORE_KEY] + out_docs.append(doc) + except KeyboardInterrupt: + raise + except Exception as e: + msg = ( + "Encountered error of type %r while trying " + "to fetch content from %s" + ) + err_type = type(e) + logger.exception(msg, err_type, link) + out_docs.append(old_doc) + + return out_docs + def _sort_final_ord_docs(all_ord_docs): """Sort ordinance documents by desirability heuristics""" From adb7884f7546e81387cde6fad7d125b8d77d54f5 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:45:07 -0600 Subject: [PATCH 32/37] Fix tests --- tests/python/unit/web/test_web_crawl.py | 93 +++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/tests/python/unit/web/test_web_crawl.py b/tests/python/unit/web/test_web_crawl.py index 6f2f1b558..ebee501cf 100644 --- a/tests/python/unit/web/test_web_crawl.py +++ b/tests/python/unit/web/test_web_crawl.py @@ -459,8 +459,39 @@ async def test_website_link_is_doc_skips_pre_checked(crawler_setup): @pytest.mark.asyncio -async def test_website_link_is_doc_external_returns_false(crawler_setup): - """External domains should return false and not create docs""" +async def test_website_link_is_doc_same_domain_non_pdf_returns_false( + crawler_setup, monkeypatch +): + """Same-domain HTML pages must stay on the recursive crawl path""" + + crawler = crawler_setup["crawler"] + link = _Link( + title="Internal", + href="https://example.com/page", + base_domain="https://example.com", + ) + html_doc_calls = 0 + + async def fake_is_pdf(_link, _depth, _score): # noqa + return False + + async def fake_as_html_doc(_link, _depth, _score): # noqa + nonlocal html_doc_calls + html_doc_calls += 1 + return True + + monkeypatch.setattr(crawler, "_website_link_is_pdf", fake_is_pdf) + monkeypatch.setattr(crawler, "_website_link_as_html_doc", fake_as_html_doc) + + assert not await crawler._website_link_is_doc(link, 0, 0) + assert html_doc_calls == 0 + + +@pytest.mark.asyncio +async def test_website_link_is_doc_external_collects_html_doc( + crawler_setup, monkeypatch +): + """External non-PDF pages should be collected as HTML docs""" crawler = crawler_setup["crawler"] link = _Link( @@ -468,7 +499,51 @@ async def test_website_link_is_doc_external_returns_false(crawler_setup): href="https://other.com/file", base_domain="https://example.com", ) - assert not await crawler._website_link_is_doc(link, 0, 0) + + async def fake_get_text_no_err(_url): # noqa + return "external" + + async def fake_tempfile_call(_doc, _text): # noqa + return None + + monkeypatch.setattr(crawler, "_get_text_no_err", fake_get_text_no_err) + monkeypatch.setattr( + website_crawl.TempFileCache, "call", fake_tempfile_call + ) + + assert await crawler._website_link_is_doc(link, 0, 0) + assert len(crawler._out_docs) == 1 + assert crawler._out_docs[0].attrs["source"] == link.href + + +@pytest.mark.asyncio +async def test_website_link_is_doc_external_sets_cache_fn( + crawler_setup, monkeypatch +): + """External HTML docs should store returned cache filename""" + + crawler = crawler_setup["crawler"] + link = _Link( + title="External", + href="https://other.com/file", + base_domain="https://example.com", + ) + cache_fn = Path("/tmp/external-cache.html") # noqa + + async def fake_get_text_no_err(_url): # noqa + return "external" + + async def fake_tempfile_call(_doc, _text): # noqa + return cache_fn + + monkeypatch.setattr(crawler, "_get_text_no_err", fake_get_text_no_err) + monkeypatch.setattr( + website_crawl.TempFileCache, "call", fake_tempfile_call + ) + + assert await crawler._website_link_is_doc(link, 0, 0) + assert len(crawler._out_docs) == 1 + assert crawler._out_docs[0].attrs["cache_fn"] == cache_fn @pytest.mark.asyncio @@ -488,7 +563,9 @@ def __init__(self, source): href="https://example.com/doc.pdf", base_domain="https://example.com", ) - crawler.afl.loader_docs[link.href] = DummyDoclingPDFDocument(link.href) + crawler.fast_afl.loader_docs[link.href] = DummyDoclingPDFDocument( + link.href + ) assert await crawler._website_link_is_pdf(link, 2, 88) assert crawler._out_docs[-1].attrs[_DEPTH_KEY] == 2 @@ -554,6 +631,13 @@ async def fake_get_text(self, url): types.MethodType(fake_get_text, crawler), ) + async def fake_tempfile_call(_doc, _text): # noqa + return None + + monkeypatch.setattr( + website_crawl.TempFileCache, "call", fake_tempfile_call + ) + link = _Link( title="HTML", href="https://example.com/page", @@ -564,6 +648,7 @@ async def fake_get_text(self, url): assert doc.attrs[_DEPTH_KEY] == 2 assert doc.attrs[_SCORE_KEY] == 9 assert "keep" in doc.text + assert doc.attrs["source"] == "https://example.com/page" @pytest.mark.asyncio From c9772f830fba4b48f6a1bfe4154c5cd80edeebaa Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:50:20 -0600 Subject: [PATCH 33/37] Fix tests --- tests/python/unit/services/test_services_threaded.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/python/unit/services/test_services_threaded.py b/tests/python/unit/services/test_services_threaded.py index 66cc220c3..192a1341f 100644 --- a/tests/python/unit/services/test_services_threaded.py +++ b/tests/python/unit/services/test_services_threaded.py @@ -193,7 +193,7 @@ def test_move_file_uses_jurisdiction_name(tmp_path): doc.attrs.update({"cache_fn": cached_fp}) date = datetime.now().strftime("%Y_%m_%d") - moved_fp = threaded._move_file(doc, out_dir, out_fn="Test County, ST") + moved_fp = threaded._move_file(doc, out_dir, out_stem="Test County, ST") expected_name = f"Test_County_ST_processed_{date}.pdf" assert moved_fp.name == expected_name @@ -221,8 +221,8 @@ def test_output_filenames_preserve_index_after_st_abbreviation(tmp_path): moved_fp = threaded._move_file(doc, out_dir, out_stem=out_stem) parsed_fp = threaded._write_parsed_text(doc, out_dir, out_stem=out_stem) - assert moved_fp.name == f"City_of_St._Paul_Alaska_1_processed_{date}.md" - assert parsed_fp.name == "City_of_St._Paul_Alaska_1.txt" + assert moved_fp.name == f"City_of_St_Paul_Alaska_1_processed_{date}.md" + assert parsed_fp.name == "City_of_St_Paul_Alaska_1.txt" assert moved_fp.read_text(encoding="utf-8") == "content" assert parsed_fp.read_text(encoding="utf-8").strip() == "payload" From e2e6f88e3410593a9fcabb662c371f3e8e7bc67e Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:54:36 -0600 Subject: [PATCH 34/37] FIx tests --- tests/python/unit/web/test_web_crawl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/unit/web/test_web_crawl.py b/tests/python/unit/web/test_web_crawl.py index ebee501cf..17b2e2cc6 100644 --- a/tests/python/unit/web/test_web_crawl.py +++ b/tests/python/unit/web/test_web_crawl.py @@ -154,6 +154,7 @@ async def fetch(self, url): ) monkeypatch.setattr(website_crawl, "HTMLDocument", DummyHTMLDocument) + monkeypatch.setattr(website_crawl, "AsyncWebFileLoader", DummyLoader) monkeypatch.setattr(website_crawl, "COMPASSWebFileLoader", DummyLoader) async def validator(doc): @@ -870,7 +871,7 @@ def test_log_crawl_stats_emits_messages( crawler._log_crawl_stats() assert_message_was_logged("Crawled 1 pages", log_level="INFO") - assert_message_was_logged("Found 1 potential documents", log_level="INFO") + assert_message_was_logged("Found 1 potential document", log_level="INFO") @pytest.mark.asyncio From 45165900db08f616c2b23acbd19570e2dd299123 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:56:44 -0600 Subject: [PATCH 35/37] Add versions --- compass/pipeline/collection/persistence.py | 4 ++++ compass/utilities/finalize.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compass/pipeline/collection/persistence.py b/compass/pipeline/collection/persistence.py index 6e787c169..d5cca3497 100644 --- a/compass/pipeline/collection/persistence.py +++ b/compass/pipeline/collection/persistence.py @@ -6,6 +6,9 @@ from warnings import warn from datetime import datetime, UTC +from elm.version import __version__ as elm_version + +from compass import __version__ as compass_version from compass.services.threaded import ( FileMover, ParsedFileWriter, @@ -59,6 +62,7 @@ def build_collection_manifest( ] return { "tech": tech, + "versions": {"compass": compass_version, "elm": elm_version}, "time_start_utc": time_start_utc.isoformat(), "time_end_utc": time_end_utc.isoformat(), "total_time": time_elapsed.total_seconds(), diff --git a/compass/utilities/finalize.py b/compass/utilities/finalize.py index f1b38f234..ca830956c 100644 --- a/compass/utilities/finalize.py +++ b/compass/utilities/finalize.py @@ -98,7 +98,7 @@ def save_run_meta( time_elapsed = end_date - start_date meta_data = { "username": username, - "versions": {"elm": elm_version, "compass": compass_version}, + "versions": {"compass": compass_version, "elm": elm_version}, "technology": tech, "models": _extract_model_info_from_all_models(models), "time_start_utc": start_date.isoformat(), From 9c65796c1e61314d85812f8a4d887ad5332494ad Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:56:53 -0600 Subject: [PATCH 36/37] Update lockfile --- pixi.lock | 75 +++++++++++++++++++++++++++----------------------- pixi.toml | 2 +- pyproject.toml | 2 +- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/pixi.lock b/pixi.lock index d82040005..58ca9c932 100644 --- a/pixi.lock +++ b/pixi.lock @@ -4028,7 +4028,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/regex-2026.5.9-py313h07c4f96_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/rpds-py-2026.5.1-py313hafbe609_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/ruff-0.15.20-h6a952e8_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ruff-0.15.22-h462bb3b_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/scikit-learn-1.8.0-np2py313h16d504d_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/scipy-1.17.1-py313h4b8bb8b_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/secretstorage-3.4.1-py313h78bf25f_0.conda @@ -4585,7 +4585,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/readline-8.3-hb682ff5_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/regex-2026.5.9-py313h6194ac5_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rpds-py-2026.5.1-py313hc72d3b0_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ruff-0.15.20-h88be79b_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ruff-0.15.22-hd82de44_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rust-1.89.0-h6cf38e9_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/scikit-learn-1.8.0-np2py313ha4ab095_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/scipy-1.17.1-py313h6b7a087_1.conda @@ -5464,7 +5464,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.3-h68b038d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/regex-2026.5.9-py313hf59fe81_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/rpds-py-2026.5.1-py313he4ad68c_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/ruff-0.15.20-h1ddadc8_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/ruff-0.15.22-hcca44e8_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/safetensors-0.7.0-py313ha265c4a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/scikit-learn-1.8.0-np2py313he2891f2_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/scipy-1.17.1-py313h9cbb6b6_1.conda @@ -6039,7 +6039,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/regex-2026.5.9-py313h0997733_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/rpds-py-2026.5.1-py313hb9d2816_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ruff-0.15.20-h80928e0_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ruff-0.15.22-h828de30_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/safetensors-0.7.0-py313h0b74987_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/scikit-learn-1.8.0-np2py313h3b23316_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/scipy-1.17.1-py313h52f5312_1.conda @@ -6539,7 +6539,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/re2-2025.06.26-h3dd2b4f_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/regex-2026.5.9-py313h5ea7bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/rpds-py-2026.5.1-py313ha9ea572_0.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/ruff-0.15.20-h45713df_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/ruff-0.15.22-h6b72154_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/safetensors-0.7.0-py313hf61f64f_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/scikit-learn-1.8.0-np2py313h4ce4a18_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/scipy-1.17.1-py313he51e9a2_1.conda @@ -13846,10 +13846,10 @@ packages: - pkg:pypi/rpds-py?source=compressed-mapping size: 311167 timestamp: 1779976991134 -- conda: https://conda.anaconda.org/conda-forge/linux-64/ruff-0.15.20-h6a952e8_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/ruff-0.15.22-h462bb3b_0.conda noarch: python - sha256: 4c10593eb248ae3b626ebfc2991bd67df96cb98a51816939188576237c54deee - md5: d9b134cef9bc26a9ed9a0fba1eff3356 + sha256: 2002c7869a95ed94b03303a033fcc24b608597bdd01c8f01d90c47175616edb9 + md5: ad7d8b2d37a1098a04ddd8b34b962481 depends: - python - __glibc >=2.17,<3.0.a0 @@ -13857,11 +13857,12 @@ packages: constrains: - __glibc >=2.17 license: MIT + license_family: MIT purls: - pkg:pypi/ruff?source=compressed-mapping run_exports: {} - size: 9342178 - timestamp: 1782428525856 + size: 9333318 + timestamp: 1784237314764 - conda: https://conda.anaconda.org/conda-forge/linux-64/rust-1.91.1-h53717f1_0.conda sha256: fb5d544cac6a15ddbc7c47fddc812407713fd220f64716928f0ccf13c8655de4 md5: 5a2d92eacdea9d7ffb895c3fd9c761e6 @@ -17472,21 +17473,22 @@ packages: - pkg:pypi/rpds-py?source=compressed-mapping size: 306112 timestamp: 1779976992450 -- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ruff-0.15.20-h88be79b_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ruff-0.15.22-hd82de44_0.conda noarch: python - sha256: 4dcde033d77e17ae8e15669177f8550ef2155ce32443ac3a4f241238f9dab4db - md5: 762455b53e3cd6c35c04259cfa2fa499 + sha256: 7db6db19bf42622fbb4a52bec37a57962453b45347ffffde78beef2cffc3db2e + md5: 7d7a99174374aee86591f23078a2c315 depends: - python - libgcc >=14 constrains: - __glibc >=2.17 license: MIT + license_family: MIT purls: - - pkg:pypi/ruff?source=hash-mapping + - pkg:pypi/ruff?source=compressed-mapping run_exports: {} - size: 8974657 - timestamp: 1782428525763 + size: 9107871 + timestamp: 1784237315859 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/rust-1.89.0-h6cf38e9_0.conda sha256: 7073c5cc353cfc4c1c7ab136a68fc6153b17ea6fcaf98469dc42e66d55f4694f md5: dd5ec0e57839733d74d8c7fe1e744b7f @@ -26076,21 +26078,22 @@ packages: - pkg:pypi/rpds-py?source=compressed-mapping size: 311909 timestamp: 1779977312121 -- conda: https://conda.anaconda.org/conda-forge/osx-64/ruff-0.15.20-h1ddadc8_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-64/ruff-0.15.22-hcca44e8_0.conda noarch: python - sha256: c09f4f7cb6f116fe2c37b6fe90e234beab754f223e11e7368e272fde890d7bac - md5: 4abaf1414e448ab88712f10a67610410 + sha256: 0da8234f5fc805e0993aa26f371f5804bd9d08c85c2c89c7150a78d726fbef68 + md5: e07419cb1a54f1dc3e380754becf7659 depends: - python - __osx >=11.0 constrains: - - __osx >=10.13 + - __osx >=11.0 license: MIT + license_family: MIT purls: - - pkg:pypi/ruff?source=hash-mapping + - pkg:pypi/ruff?source=compressed-mapping run_exports: {} - size: 9318935 - timestamp: 1782428757092 + size: 9376975 + timestamp: 1784237606334 - conda: https://conda.anaconda.org/conda-forge/osx-64/rust-1.91.1-h34a2095_0.conda sha256: 6100c56165de7a7ae364b31c5aaa6ab767c1df93d8f4727588c4c1e67578598e md5: 12a95d7b58a607bb4733ccbad692bbd5 @@ -30038,21 +30041,22 @@ packages: - pkg:pypi/rpds-py?source=compressed-mapping size: 293990 timestamp: 1779977082789 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ruff-0.15.20-h80928e0_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ruff-0.15.22-h828de30_0.conda noarch: python - sha256: 49c7cb7fa8bc6ad2e3448d3ea48e4e939df69690a984e768261ddc0728e40f7f - md5: 60e90a1347592b61888a3f26ad93035c + sha256: 56362f7c5150de1f01450fcb668dbd9c3f163e9f0f69f1f7c9da5bebd0d75172 + md5: 81fe5e5807d471fcd624bfb6bdb98968 depends: - python - __osx >=11.0 constrains: - __osx >=11.0 license: MIT + license_family: MIT purls: - pkg:pypi/ruff?source=compressed-mapping run_exports: {} - size: 8539333 - timestamp: 1782428897543 + size: 8651168 + timestamp: 1784237480462 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/rust-1.91.1-h4ff7c5d_0.conda sha256: 7a6bb008bd61465de2da9a4bbe8b6698d457a134e6c91470a2b90f9fc030cadf md5: e7f3b1b4506cd0583e1e51de8fe608b8 @@ -33576,21 +33580,22 @@ packages: - pkg:pypi/rpds-py?source=compressed-mapping size: 230648 timestamp: 1779977048910 -- conda: https://conda.anaconda.org/conda-forge/win-64/ruff-0.15.20-h45713df_0.conda +- conda: https://conda.anaconda.org/conda-forge/win-64/ruff-0.15.22-h6b72154_0.conda noarch: python - sha256: 63cc41a9acf4d4bcc2273df322e0cb6d08311e9e7425fffe3bbe01e7c496fbb7 - md5: 7e18b4c765bf080576d39b177167b3f2 + sha256: 47c20477fb35f1e9748852dbe21d1c53ff5a9639ac0c99f52e651ffa30fced44 + md5: 10f708a322c6dd1c412848817ad6c968 depends: - python - vc >=14.3,<15 - vc14_runtime >=14.44.35208 - ucrt >=10.0.20348.0 license: MIT + license_family: MIT purls: - - pkg:pypi/ruff?source=hash-mapping + - pkg:pypi/ruff?source=compressed-mapping run_exports: {} - size: 9813687 - timestamp: 1782428572744 + size: 9849056 + timestamp: 1784237363067 - conda: https://conda.anaconda.org/conda-forge/win-64/rust-1.91.1-hf8d6059_0.conda sha256: 8d534ac516635aa5e60d92b662cbb8a46e8538b7c59ac19cfb4bc67fdfba695d md5: 07784b15b3aba5e4b95e708ab64017e5 @@ -34270,7 +34275,7 @@ packages: - pytesseract>=0.3.13,<0.4 ; extra == 'ocr' - jupyter>=1.0.0,<1.1 ; extra == 'dev' - pipreqs>=0.4.13,<0.5 ; extra == 'dev' - - ruff>=0.15.19,<0.16 ; extra == 'dev' + - ruff>=0.15.22,<0.16 ; extra == 'dev' - ruff-lsp>=0.0.62,<0.0.63 ; extra == 'dev' - flaky>=3.8.1,<4 ; extra == 'test' - pytest>=9.0.3,<9.1 ; extra == 'test' diff --git a/pixi.toml b/pixi.toml index f26a7ee96..e37b8bf14 100644 --- a/pixi.toml +++ b/pixi.toml @@ -144,7 +144,7 @@ geopandas = ">=1.0.1,<2" ipykernel = ">=7.1.0,<8" jupyter = ">=1.0.0,<1.1" pipreqs = ">=0.4.13,<0.5" -ruff = ">=0.15.19,<0.16" +ruff = ">=0.15.22,<0.16" ruff-lsp = ">=0.0.62,<0.0.63" seaborn = ">=0.13.2,<0.14" diff --git a/pyproject.toml b/pyproject.toml index 3abfe6ffd..5c28c9e10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ ocr = [ dev = [ "jupyter>=1.0.0,<1.1", "pipreqs>=0.4.13,<0.5", - "ruff>=0.15.19,<0.16", + "ruff>=0.15.22,<0.16", "ruff-lsp>=0.0.62,<0.0.63", ] test = [ From f0e72db1551506e76c3bc7301b3e2f13dacfa4c7 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 20 Jul 2026 19:58:02 -0600 Subject: [PATCH 37/37] Update linter ignores --- compass/_cli/common.py | 6 ++--- compass/common/base.py | 28 +++++++++++----------- compass/extraction/small_wind/graphs.py | 8 +++---- compass/extraction/solar/graphs.py | 4 ++-- compass/extraction/water/graphs.py | 32 ++++++++++++------------- compass/extraction/water/plugin.py | 12 +++++----- compass/extraction/wind/graphs.py | 8 +++---- compass/llm/config.py | 2 +- compass/pb.py | 4 ++-- compass/pipeline/collection/steps.py | 12 +++++----- compass/pipeline/data_classes.py | 6 ++--- compass/pipeline/runtime.py | 2 +- compass/plugin/base.py | 4 ++-- compass/plugin/interface.py | 16 ++++++------- compass/plugin/noop.py | 4 ++-- compass/plugin/one_shot/base.py | 4 ++-- compass/plugin/one_shot/components.py | 14 +++++------ compass/plugin/one_shot/generators.py | 2 +- compass/plugin/ordinance.py | 24 +++++++++---------- compass/scripts/download.py | 10 ++++---- compass/services/base.py | 6 ++--- compass/services/cpu.py | 10 ++++---- compass/services/threaded.py | 2 +- compass/utilities/enums.py | 2 +- compass/utilities/finalize.py | 2 +- compass/utilities/io.py | 2 +- compass/utilities/logs.py | 18 +++++++------- compass/utilities/parsing.py | 2 +- compass/validation/content.py | 8 +++---- compass/validation/graphs.py | 6 ++--- compass/validation/location.py | 4 ++-- compass/validation/utilities.py | 6 ++--- compass/web/file_loader.py | 2 +- compass/web/website_crawl.py | 8 +++---- 34 files changed, 140 insertions(+), 140 deletions(-) diff --git a/compass/_cli/common.py b/compass/_cli/common.py index ee1c16226..7c3e2a104 100644 --- a/compass/_cli/common.py +++ b/compass/_cli/common.py @@ -97,11 +97,11 @@ def setup_cli_logging(console, verbosity_level, log_level="INFO"): libs = [] if verbosity_level >= 1: libs.append("compass") - if verbosity_level >= 2: # noqa: PLR2004 + if verbosity_level >= 2: # ruff:ignore[magic-value-comparison] libs.extend(("elm", "docling")) - if verbosity_level >= 3: # noqa: PLR2004 + if verbosity_level >= 3: # ruff:ignore[magic-value-comparison] libs.append("openai") - if verbosity_level >= 4: # noqa: PLR2004 + if verbosity_level >= 4: # ruff:ignore[magic-value-comparison] libs.extend(("networkx", "pytesseract", "pdf2image", "pdftotext")) for lib in libs: diff --git a/compass/common/base.py b/compass/common/base.py index 308e816f9..74c3356ca 100644 --- a/compass/common/base.py +++ b/compass/common/base.py @@ -150,7 +150,7 @@ def setup_async_decision_tree( The function asserts that the tree has recorded at least the system prompt before returning the constructed wrapper. """ - G = graph_setup_func(**kwargs) # noqa: N806 + G = graph_setup_func(**kwargs) # ruff:ignore[non-lowercase-variable-in-function] tree = AsyncDecisionTree(G, usage_sub_label=usage_sub_label) assert len(tree.chat_llm_caller.messages) == 1 return tree @@ -241,7 +241,7 @@ def setup_base_setback_graph(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Base setback questions", **kwargs ) @@ -315,7 +315,7 @@ def setup_participating_owner(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Participating owner", **kwargs ) @@ -434,7 +434,7 @@ def setup_graph_extra_restriction(is_numerical=True, **kwargs): kwargs.setdefault("unit_clarification", "") kwargs.setdefault("feature_clarifications", "") feature_id = kwargs.get("feature_id", "") - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Extra restriction", **kwargs ) @@ -571,7 +571,7 @@ def setup_graph_extra_restriction(is_numerical=True, **kwargs): return G -def _add_other_system_setback_clarification_nodes(G): # noqa: N803 +def _add_other_system_setback_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify "other system" setbacks""" G.add_edge("init", "is_intra_farm", condition=llm_response_starts_with_yes) G.add_node( @@ -592,7 +592,7 @@ def _add_other_system_setback_clarification_nodes(G): # noqa: N803 return G -def _add_coverage_clarification_nodes(G): # noqa: N803 +def _add_coverage_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify "coverage" extraction""" G.add_edge("init", "is_area", condition=llm_response_starts_with_yes) G.add_node( @@ -603,7 +603,7 @@ def _add_coverage_clarification_nodes(G): # noqa: N803 return G -def _add_land_density_clarification_nodes(G): # noqa: N803 +def _add_land_density_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify "land density" extraction""" G.add_edge( "init", "correct_density_units", condition=llm_response_starts_with_yes @@ -623,7 +623,7 @@ def _add_land_density_clarification_nodes(G): # noqa: N803 return G -def _add_minimum_lot_size_clarification_nodes(G): # noqa: N803 +def _add_minimum_lot_size_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify "minimum lot size" extraction""" G.add_edge( "init", "correct_min_ls_units", condition=llm_response_starts_with_yes @@ -641,7 +641,7 @@ def _add_minimum_lot_size_clarification_nodes(G): # noqa: N803 return G -def _add_maximum_lot_size_clarification_nodes(G): # noqa: N803 +def _add_maximum_lot_size_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify "maximum lot size" extraction""" G.add_edge( "init", "correct_max_ls_units", condition=llm_response_starts_with_yes @@ -659,7 +659,7 @@ def _add_maximum_lot_size_clarification_nodes(G): # noqa: N803 return G -def _add_maximum_project_size_clarification_nodes(G): # noqa: N803 +def _add_maximum_project_size_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify "max project size" extraction""" G.add_edge("init", "is_mps_area", condition=llm_response_starts_with_yes) G.add_node( @@ -691,7 +691,7 @@ def _add_maximum_project_size_clarification_nodes(G): # noqa: N803 return G -def _add_maximum_turbine_height_clarification_nodes(G): # noqa: N803 +def _add_maximum_turbine_height_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify max turbine height extraction""" G.add_edge("init", "has_relative", condition=llm_response_starts_with_yes) G.add_node( @@ -726,7 +726,7 @@ def _add_maximum_turbine_height_clarification_nodes(G): # noqa: N803 return G -def _add_value_and_units_clarification_nodes(G): # noqa: N803 +def _add_value_and_units_clarification_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to clarify value and units extraction""" G.add_node( @@ -796,7 +796,7 @@ def _add_value_and_units_clarification_nodes(G): # noqa: N803 return G -def _add_prohibitions_extraction_nodes(G): # noqa: N803 +def _add_prohibitions_extraction_nodes(G): # ruff:ignore[invalid-argument-name] """Add nodes and edges to extract 'prohibitions'""" G.add_edge("init", "is_proposed", condition=llm_response_starts_with_yes) @@ -888,7 +888,7 @@ def setup_graph_permitted_use_districts(**kwargs): `elm.tree.DecisionTree`. """ feature_id = kwargs.get("feature_id", "") - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Permitted use districts", **kwargs ) diff --git a/compass/extraction/small_wind/graphs.py b/compass/extraction/small_wind/graphs.py index 533f45b5f..9074fec58 100644 --- a/compass/extraction/small_wind/graphs.py +++ b/compass/extraction/small_wind/graphs.py @@ -21,7 +21,7 @@ def setup_graph_wes_types(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Wind Energy Farm types", **kwargs ) @@ -154,7 +154,7 @@ def setup_multiplier(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Setback distance", **kwargs ) @@ -362,7 +362,7 @@ def setup_conditional_min(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Minimum setback distance", **kwargs ) @@ -440,7 +440,7 @@ def setup_conditional_max(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Maximum setback distance", **kwargs ) diff --git a/compass/extraction/solar/graphs.py b/compass/extraction/solar/graphs.py index 3f0a35550..a4d10309b 100644 --- a/compass/extraction/solar/graphs.py +++ b/compass/extraction/solar/graphs.py @@ -21,7 +21,7 @@ def setup_graph_sef_types(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Solar Energy Farm types", **kwargs ) @@ -156,7 +156,7 @@ def setup_multiplier(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Setback distance", **kwargs ) diff --git a/compass/extraction/water/graphs.py b/compass/extraction/water/graphs.py index f39406649..a23eab1a0 100644 --- a/compass/extraction/water/graphs.py +++ b/compass/extraction/water/graphs.py @@ -21,7 +21,7 @@ def setup_graph_permits(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Permit Requirements", **kwargs ) @@ -91,7 +91,7 @@ def setup_graph_extraction(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Water Extraction Requirements", **kwargs ) @@ -149,7 +149,7 @@ def setup_graph_geothermal(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Geothermal Policies", **kwargs ) @@ -213,7 +213,7 @@ def setup_graph_oil_and_gas(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Oil and Gas Policies", **kwargs ) @@ -277,7 +277,7 @@ def setup_graph_limits(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Extraction Limits", **kwargs ) @@ -387,7 +387,7 @@ def setup_graph_well_spacing(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Well Spacing", **kwargs ) @@ -481,7 +481,7 @@ def setup_graph_time(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Drilling Window", **kwargs ) @@ -546,7 +546,7 @@ def setup_graph_metering_device(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Metering Device", **kwargs ) @@ -605,7 +605,7 @@ def setup_graph_drought(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Drought Management Plan", **kwargs ) @@ -669,7 +669,7 @@ def setup_graph_contingency(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Contingency Plan Requirements", **kwargs ) @@ -733,7 +733,7 @@ def setup_graph_plugging_reqs(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Plugging Requirements", **kwargs ) @@ -793,7 +793,7 @@ def setup_graph_external_transfer(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="External Transfer Restrictions", **kwargs ) @@ -902,7 +902,7 @@ def setup_graph_production_reporting(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Water Well Production Reporting", **kwargs ) @@ -962,7 +962,7 @@ def setup_graph_production_cost(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Water Well Production Cost", **kwargs ) @@ -1045,7 +1045,7 @@ def setup_graph_setback_features(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Setback Features", **kwargs ) @@ -1109,7 +1109,7 @@ def setup_graph_redrilling(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Redrilling Restrictions", **kwargs ) diff --git a/compass/extraction/water/plugin.py b/compass/extraction/water/plugin.py index 9dd5c3f0b..e4db9bc3b 100644 --- a/compass/extraction/water/plugin.py +++ b/compass/extraction/water/plugin.py @@ -55,7 +55,7 @@ class WaterRightsHeuristic: """NoOp heuristic check""" - def check(self, *__, **___): # noqa: PLR6301 + def check(self, *__, **___): # ruff:ignore[no-self-use] """Always return ``True`` for water rights documents""" return True @@ -73,7 +73,7 @@ class TexasWaterRightsExtractor(BaseExtractionPlugin): ) """:term:`path-like `: Path to Texas GCW names""" - async def get_query_templates(self): # noqa: PLR6301 + async def get_query_templates(self): # ruff:ignore[no-self-use] """Get a list of search engine query templates for extraction Query templates can contain the placeholder ``{jurisdiction}`` @@ -82,7 +82,7 @@ async def get_query_templates(self): # noqa: PLR6301 """ return WATER_RIGHTS_QUERY_TEMPLATES - async def get_website_keywords(self): # noqa: PLR6301 + async def get_website_keywords(self): # ruff:ignore[no-self-use] """Get a dict of website search keyword scores Dictionary mapping keywords to scores that indicate links which @@ -91,7 +91,7 @@ async def get_website_keywords(self): # noqa: PLR6301 """ return BEST_WATER_RIGHTS_ORDINANCE_WEBSITE_URL_KEYWORDS - async def get_heuristic(self): # noqa: PLR6301 + async def get_heuristic(self): # ruff:ignore[no-self-use] """Get a `BaseHeuristic` instance with a `check()` method The ``check()`` method should accept a string of text and return @@ -140,7 +140,7 @@ async def filter_docs(self, extraction_context, __): "Embeddings are ``None`` when building corpus for " "water rights extraction!" ) - raise COMPASSRuntimeError(msg) # noqa: TRY301 + raise COMPASSRuntimeError(msg) # ruff:ignore[raise-within-try] corpus.append( pd.DataFrame( @@ -151,7 +151,7 @@ async def filter_docs(self, extraction_context, __): ) ) - except Exception as e: # noqa: BLE001 + except Exception as e: # ruff:ignore[blind-except] logger.info("could not embed %r with error: %s", url, e) continue diff --git a/compass/extraction/wind/graphs.py b/compass/extraction/wind/graphs.py index b30ddf52a..0ba188069 100644 --- a/compass/extraction/wind/graphs.py +++ b/compass/extraction/wind/graphs.py @@ -21,7 +21,7 @@ def setup_graph_wes_types(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Wind Energy Farm types", **kwargs ) @@ -148,7 +148,7 @@ def setup_multiplier(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Setback distance", **kwargs ) @@ -356,7 +356,7 @@ def setup_conditional_min(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Minimum setback distance", **kwargs ) @@ -434,7 +434,7 @@ def setup_conditional_max(**kwargs): Graph instance that can be used to initialize an `elm.tree.DecisionTree`. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Maximum setback distance", **kwargs ) diff --git a/compass/llm/config.py b/compass/llm/config.py index e79e11c15..c208726bc 100644 --- a/compass/llm/config.py +++ b/compass/llm/config.py @@ -81,7 +81,7 @@ def __init__( @cached_property def text_splitter(self): - """`TextSplitter `_: Text splitter for ordinance text""" # noqa: W505, E501 + """`TextSplitter `_: Text splitter for ordinance text""" # ruff:ignore[doc-line-too-long, line-too-long] return _PrintableRecursiveCharacterTextSplitter( RTS_SEPARATORS, chunk_size=self.text_splitter_chunk_size, diff --git a/compass/pb.py b/compass/pb.py index 317e9a921..beae09c12 100644 --- a/compass/pb.py +++ b/compass/pb.py @@ -25,7 +25,7 @@ class _TimeElapsedColumn(ProgressColumn): """Renders time elapsed""" - def render(self, task): # noqa: PLR6301 + def render(self, task): # ruff:ignore[no-self-use] """Show time elapsed""" elapsed = task.finished_time if task.finished else task.elapsed if elapsed is None: @@ -66,7 +66,7 @@ def render(self, task): class _TotalCostColumn(ProgressColumn): """Renders total cost '($1.23)'""" - def render(self, task): # noqa: PLR6301 + def render(self, task): # ruff:ignore[no-self-use] """Show completed/total""" total_cost = task.fields.get("total_cost", 0) if not total_cost: diff --git a/compass/pipeline/collection/steps.py b/compass/pipeline/collection/steps.py index fcb9fbe22..c060040b3 100644 --- a/compass/pipeline/collection/steps.py +++ b/compass/pipeline/collection/steps.py @@ -26,7 +26,7 @@ class CollectionStep(ABC): @property @abstractmethod - def STEP_NAME(self): # noqa: N802 + def STEP_NAME(self): # ruff:ignore[invalid-function-name] """Identifier for step (e.g. "known_local_docs")""" raise NotImplementedError @@ -42,7 +42,7 @@ class KnownLocalDocumentsStep(CollectionStep): STEP_NAME = COMPASSDocumentCollectionStep.KNOWN_LOCAL_DOCS """Identifier for step""" - async def collect(self, workflow): # noqa: PLR6301 + async def collect(self, workflow): # ruff:ignore[no-self-use] """Collect known local documents for this jurisdiction Parameters @@ -98,7 +98,7 @@ class KnownUrlDocumentsStep(CollectionStep): STEP_NAME = COMPASSDocumentCollectionStep.KNOWN_DOC_URLS """Identifier for step""" - async def collect(self, workflow): # noqa: PLR6301 + async def collect(self, workflow): # ruff:ignore[no-self-use] """Collect documents from known URL's for this jurisdiction Parameters @@ -152,7 +152,7 @@ class SearchEngineDocumentsStep(CollectionStep): STEP_NAME = COMPASSDocumentCollectionStep.SEARCH_ENGINE """Identifier for step""" - async def collect(self, workflow): # noqa: PLR6301 + async def collect(self, workflow): # ruff:ignore[no-self-use] """Collect documents based on a search engine search Parameters @@ -223,7 +223,7 @@ class ElmWebsiteCrawlStep(CollectionStep): STEP_NAME = COMPASSDocumentCollectionStep.WEBSITE_SEARCH_ELM """Identifier for step""" - async def collect(self, workflow): # noqa: PLR6301 + async def collect(self, workflow): # ruff:ignore[no-self-use] """Collect documents based on an ELM website crawl Parameters @@ -307,7 +307,7 @@ class CompassWebsiteCrawlStep(CollectionStep): STEP_NAME = COMPASSDocumentCollectionStep.WEBSITE_SEARCH_COMPASS """Identifier for step""" - async def collect(self, workflow): # noqa: PLR6301 + async def collect(self, workflow): # ruff:ignore[no-self-use] """Collect documents based on a COMPASS website crawl Parameters diff --git a/compass/pipeline/data_classes.py b/compass/pipeline/data_classes.py index bec8963af..37728695a 100644 --- a/compass/pipeline/data_classes.py +++ b/compass/pipeline/data_classes.py @@ -321,7 +321,7 @@ class BaseRequest: MODE = None """COMPASSRunMode associated with this request type""" - def __init__( # noqa: PLR0913 + def __init__( # ruff:ignore[too-many-arguments] self, out_dir, tech, @@ -691,7 +691,7 @@ class CollectionRequest(BaseRequest): MODE = COMPASSRunMode.COLLECT """COMPASSRunMode associated with this request type""" - def __init__( # noqa: PLR0913 + def __init__( # ruff:ignore[too-many-arguments] self, out_dir, tech, @@ -1025,7 +1025,7 @@ class ExtractionRequest(BaseRequest): MODE = COMPASSRunMode.EXTRACT """COMPASSRunMode associated with this request type""" - def __init__( # noqa: PLR0913 + def __init__( # ruff:ignore[too-many-arguments] self, out_dir, tech, diff --git a/compass/pipeline/runtime.py b/compass/pipeline/runtime.py index 63203896f..ada408854 100644 --- a/compass/pipeline/runtime.py +++ b/compass/pipeline/runtime.py @@ -265,7 +265,7 @@ def _setup_pytesseract(self): if self._pytesseract_was_set_up: return - import pytesseract # noqa: PLC0415 + import pytesseract # ruff:ignore[import-outside-top-level] logger.debug( "Setting `tesseract_cmd` to %s", diff --git a/compass/plugin/base.py b/compass/plugin/base.py index 06c8bb195..1ad36bd53 100644 --- a/compass/plugin/base.py +++ b/compass/plugin/base.py @@ -68,7 +68,7 @@ def __init__(self, jurisdiction, model_configs, usage_tracker=None): @property @abstractmethod - def IDENTIFIER(self): # noqa: N802 + def IDENTIFIER(self): # ruff:ignore[invalid-function-name] """str: Identifier for extraction task (e.g. "water rights")""" raise NotImplementedError @@ -176,5 +176,5 @@ async def record_usage(self): total_cost = compute_total_cost_from_usage(total_usage) COMPASS_PB.update_total_cost(total_cost, replace=True) - def validate_plugin_configuration(self): # noqa: B027 + def validate_plugin_configuration(self): # ruff:ignore[empty-method-without-abstract-decorator] """[NOT PUBLIC API] Validate plugin is properly configured""" diff --git a/compass/plugin/interface.py b/compass/plugin/interface.py index 6b775d086..c9244fbda 100644 --- a/compass/plugin/interface.py +++ b/compass/plugin/interface.py @@ -42,7 +42,7 @@ class BaseTextCollector(BaseLLMCaller, ABC): @property @abstractmethod - def OUT_LABEL(self): # noqa: N802 + def OUT_LABEL(self): # ruff:ignore[invalid-function-name] """str: Identifier for text collected by this class""" raise NotImplementedError @@ -115,13 +115,13 @@ class FilteredExtractionPlugin(BaseExtractionPlugin): @property @abstractmethod - def IDENTIFIER(self): # noqa: N802 + def IDENTIFIER(self): # ruff:ignore[invalid-function-name] """str: Identifier for extraction task (e.g. "water rights")""" raise NotImplementedError @property @abstractmethod - def QUERY_TEMPLATES(self): # noqa: N802 + def QUERY_TEMPLATES(self): # ruff:ignore[invalid-function-name] """list: List of search engine query templates for extraction Query templates can contain the placeholder ``{jurisdiction}`` @@ -132,7 +132,7 @@ def QUERY_TEMPLATES(self): # noqa: N802 @property @abstractmethod - def WEBSITE_KEYWORDS(self): # noqa: N802 + def WEBSITE_KEYWORDS(self): # ruff:ignore[invalid-function-name] """list: List of keywords List of keywords that indicate links which should be prioritized @@ -142,7 +142,7 @@ def WEBSITE_KEYWORDS(self): # noqa: N802 @property @abstractmethod - def TEXT_COLLECTORS(self): # noqa: N802 + def TEXT_COLLECTORS(self): # ruff:ignore[invalid-function-name] """list of BaseTextCollector: Classes to collect text Should be an iterable of one or more classes to collect text @@ -152,7 +152,7 @@ def TEXT_COLLECTORS(self): # noqa: N802 @property @abstractmethod - def HEURISTIC(self): # noqa: N802 + def HEURISTIC(self): # ruff:ignore[invalid-function-name] """BaseHeuristic: Class with a ``check()`` method The ``check()`` method should accept a string of text and @@ -188,7 +188,7 @@ def save_structured_data(cls, doc_infos, out_dir): save_db(db, out_dir) return num_docs_found - async def pre_filter_docs_hook(self, extraction_context): # noqa: PLR6301 + async def pre_filter_docs_hook(self, extraction_context): # ruff:ignore[no-self-use] """Pre-process documents before running them through the filter Parameters @@ -203,7 +203,7 @@ async def pre_filter_docs_hook(self, extraction_context): # noqa: PLR6301 """ return extraction_context - async def post_filter_docs_hook(self, extraction_context): # noqa: PLR6301 + async def post_filter_docs_hook(self, extraction_context): # ruff:ignore[no-self-use] """Post-process documents after running them through the filter Parameters diff --git a/compass/plugin/noop.py b/compass/plugin/noop.py index bdf44629e..96c6b0222 100644 --- a/compass/plugin/noop.py +++ b/compass/plugin/noop.py @@ -13,7 +13,7 @@ class NoOpHeuristic(BaseHeuristic): """NoOp heuristic check""" - def check(self, *__, **___): # noqa: PLR6301 + def check(self, *__, **___): # ruff:ignore[no-self-use] """Always return ``True``""" return True @@ -76,7 +76,7 @@ def _store_chunk(self, parser, chunk_ind): class NoOpTextExtractor(BaseTextExtractor): """NoOp text extractor that returns the full text""" - async def return_original(self, text_chunks): # noqa: PLR6301 + async def return_original(self, text_chunks): # ruff:ignore[no-self-use] """No processing, just return original text Parameters diff --git a/compass/plugin/one_shot/base.py b/compass/plugin/one_shot/base.py index e2f9dd77b..d2670cd4e 100644 --- a/compass/plugin/one_shot/base.py +++ b/compass/plugin/one_shot/base.py @@ -51,7 +51,7 @@ class _CacheKey(StrEnum): HEURISTIC_KEYWORDS = auto() -def create_schema_based_one_shot_extraction_plugin(config, tech): # noqa: C901 +def create_schema_based_one_shot_extraction_plugin(config, tech): # ruff:ignore[complex-structure] """Create a one-shot extraction plugin based on a configuration Parameters @@ -621,7 +621,7 @@ def _normalize_heuristic_keywords(raw): ) raise COMPASSPluginConfigurationError(msg) - if num_good_kw < 10: # noqa: PLR2004 + if num_good_kw < 10: # ruff:ignore[magic-value-comparison] msg = ( 'It is recommended to provide at least 10 total "Good" ' "heuristic values across the GOOD_TECH_KEYWORDS, " diff --git a/compass/plugin/one_shot/components.py b/compass/plugin/one_shot/components.py index 235371082..41d34a2df 100644 --- a/compass/plugin/one_shot/components.py +++ b/compass/plugin/one_shot/components.py @@ -146,19 +146,19 @@ class SchemaBasedTextCollector(SchemaOutputLLMCaller, BaseTextCollector, ABC): @property @abstractmethod - def SCHEMA(self): # noqa: N802 + def SCHEMA(self): # ruff:ignore[invalid-function-name] """dict: Extraction schema""" raise NotImplementedError @property @abstractmethod - def SCOPE_VALIDATION_OUTPUT_SCHEMA(self): # noqa: N802 + def SCOPE_VALIDATION_OUTPUT_SCHEMA(self): # ruff:ignore[invalid-function-name] """dict: Scope validation output schema""" raise NotImplementedError @property @abstractmethod - def CONTENT_VALIDATION_OUTPUT_SCHEMA(self): # noqa: N802 + def CONTENT_VALIDATION_OUTPUT_SCHEMA(self): # ruff:ignore[invalid-function-name] """dict: Content validation output schema""" raise NotImplementedError @@ -324,13 +324,13 @@ class SchemaBasedTextExtractor(SchemaOutputLLMCaller, BaseTextExtractor): @property @abstractmethod - def SCHEMA(self): # noqa: N802 + def SCHEMA(self): # ruff:ignore[invalid-function-name] """dict: Extraction schema""" raise NotImplementedError @property @abstractmethod - def OUTPUT_SCHEMA(self): # noqa: N802 + def OUTPUT_SCHEMA(self): # ruff:ignore[invalid-function-name] """dict: Validation output schema""" raise NotImplementedError @@ -414,13 +414,13 @@ class SchemaOrdinanceParser(SchemaOutputLLMCaller, BaseParser): @property @abstractmethod - def SCHEMA(self): # noqa: N802 + def SCHEMA(self): # ruff:ignore[invalid-function-name] """dict: Extraction schema""" raise NotImplementedError @property @abstractmethod - def QUALITATIVE_FEATURES(self): # noqa: N802 + def QUALITATIVE_FEATURES(self): # ruff:ignore[invalid-function-name] """set: **Lowercase** feature names of qualitative features""" raise NotImplementedError diff --git a/compass/plugin/one_shot/generators.py b/compass/plugin/one_shot/generators.py index a8fa50af6..d1f35edad 100644 --- a/compass/plugin/one_shot/generators.py +++ b/compass/plugin/one_shot/generators.py @@ -354,7 +354,7 @@ def _is_formattable(q): """True if the query template is formattable with a jurisdiction""" try: q.format(jurisdiction="test") - except Exception: # noqa: BLE001 + except Exception: # ruff:ignore[blind-except] return False return True diff --git a/compass/plugin/ordinance.py b/compass/plugin/ordinance.py index e12468740..08ff7b017 100644 --- a/compass/plugin/ordinance.py +++ b/compass/plugin/ordinance.py @@ -133,13 +133,13 @@ class BaseTextExtractor(BaseLLMCaller, ABC): @property @abstractmethod - def IN_LABEL(self): # noqa: N802 + def IN_LABEL(self): # ruff:ignore[invalid-function-name] """str: Identifier for text ingested by this class""" raise NotImplementedError @property @abstractmethod - def OUT_LABEL(self): # noqa: N802 + def OUT_LABEL(self): # ruff:ignore[invalid-function-name] """str: Identifier for final text extracted by this class""" raise NotImplementedError @@ -166,13 +166,13 @@ class BaseParser(ABC): @property @abstractmethod - def IN_LABEL(self): # noqa: N802 + def IN_LABEL(self): # ruff:ignore[invalid-function-name] """str: Identifier for text ingested by this class""" raise NotImplementedError @property @abstractmethod - def OUT_LABEL(self): # noqa: N802 + def OUT_LABEL(self): # ruff:ignore[invalid-function-name] """str: Identifier for final structured data output""" raise NotImplementedError @@ -312,25 +312,25 @@ def _count_phrase_matches(self, heuristics_text): @property @abstractmethod - def NOT_TECH_WORDS(self): # noqa: N802 + def NOT_TECH_WORDS(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Not tech keywords""" raise NotImplementedError @property @abstractmethod - def GOOD_TECH_KEYWORDS(self): # noqa: N802 + def GOOD_TECH_KEYWORDS(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Tech keywords""" raise NotImplementedError @property @abstractmethod - def GOOD_TECH_ACRONYMS(self): # noqa: N802 + def GOOD_TECH_ACRONYMS(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Tech acronyms""" raise NotImplementedError @property @abstractmethod - def GOOD_TECH_PHRASES(self): # noqa: N802 + def GOOD_TECH_PHRASES(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Tech phrases""" raise NotImplementedError @@ -340,7 +340,7 @@ class PromptBasedTextCollector(JSONFromTextLLMCaller, BaseTextCollector, ABC): @property @abstractmethod - def PROMPTS(self): # noqa: N802 + def PROMPTS(self): # ruff:ignore[invalid-function-name] """list: List of dicts defining the prompts for text extraction Each dict in the list should have the following keys: @@ -523,7 +523,7 @@ class PromptBasedTextExtractor(LLMCaller, BaseTextExtractor, ABC): @property @abstractmethod - def PROMPTS(self): # noqa: N802 + def PROMPTS(self): # ruff:ignore[invalid-function-name] """list: List of dicts defining the prompts for text extraction Each dict in the list should have the following keys: @@ -670,7 +670,7 @@ class OrdinanceExtractionPlugin(FilteredExtractionPlugin): @property @abstractmethod - def TEXT_EXTRACTORS(self): # noqa: N802 + def TEXT_EXTRACTORS(self): # ruff:ignore[invalid-function-name] """list of BaseTextExtractor: Classes to condense text Should be an iterable of one or more classes to condense text in @@ -680,7 +680,7 @@ def TEXT_EXTRACTORS(self): # noqa: N802 @property @abstractmethod - def PARSERS(self): # noqa: N802 + def PARSERS(self): # ruff:ignore[invalid-function-name] """list of BaseParser: Classes to extract structured data Should be an iterable of one or more classes to extract diff --git a/compass/scripts/download.py b/compass/scripts/download.py index 0abe23b83..82f0793c2 100644 --- a/compass/scripts/download.py +++ b/compass/scripts/download.py @@ -6,7 +6,7 @@ from elm.web.search.run import load_docs, search_with_fallback from elm.web.website_crawl import ( - _SCORE_KEY, # noqa: PLC2701 + _SCORE_KEY, # ruff:ignore[import-private-name] ELMWebsiteCrawler, ELMLinkScorer, ) @@ -340,7 +340,7 @@ async def download_jurisdiction_ordinances_from_website( if crawl_semaphore is None: crawl_semaphore = AsyncExitStack() - async def _doc_heuristic(doc): # noqa: RUF029 + async def _doc_heuristic(doc): # ruff:ignore[unused-async] """Heuristic check for wind ordinance documents""" is_valid_document = heuristic.check(doc.text.lower()) if is_valid_document and pb_jurisdiction_name: @@ -348,7 +348,7 @@ async def _doc_heuristic(doc): # noqa: RUF029 return is_valid_document - async def _crawl_hook(*__, **___): # noqa: RUF029 + async def _crawl_hook(*__, **___): # ruff:ignore[unused-async] """Update progress bar as pages are searched""" COMPASS_PB.update_website_crawl_task(pb_jurisdiction_name, advance=1) @@ -475,7 +475,7 @@ async def download_jurisdiction_ordinances_from_website_compass_crawl( if crawl_semaphore is None: crawl_semaphore = AsyncExitStack() - async def _doc_heuristic(doc): # noqa: RUF029 + async def _doc_heuristic(doc): # ruff:ignore[unused-async] """Heuristic check for wind ordinance documents""" is_valid_document = heuristic.check(doc.text.lower()) if is_valid_document and pb_jurisdiction_name: @@ -484,7 +484,7 @@ async def _doc_heuristic(doc): # noqa: RUF029 ) return is_valid_document - async def _crawl_hook(*__, **___): # noqa: RUF029 + async def _crawl_hook(*__, **___): # ruff:ignore[unused-async] """Update progress bar as pages are searched""" COMPASS_PB.update_compass_website_crawl_task( pb_jurisdiction_name, advance=1 diff --git a/compass/services/base.py b/compass/services/base.py index f47b1473b..539346914 100644 --- a/compass/services/base.py +++ b/compass/services/base.py @@ -96,16 +96,16 @@ async def process_using_futures(self, fut, *args, **kwargs): try: response = await self.process(*args, **kwargs) - except Exception as e: # noqa: BLE001 + except Exception as e: # ruff:ignore[blind-except] fut.set_exception(e) return fut.set_result(response) - def acquire_resources(self): # noqa: B027 + def acquire_resources(self): # ruff:ignore[empty-method-without-abstract-decorator] """Use this method to allocate resources, if needed""" - def release_resources(self): # noqa: B027 + def release_resources(self): # ruff:ignore[empty-method-without-abstract-decorator] """Use this method to clean up resources, if needed""" @property diff --git a/compass/services/cpu.py b/compass/services/cpu.py index 70c0bd965..935a4c22b 100644 --- a/compass/services/cpu.py +++ b/compass/services/cpu.py @@ -207,7 +207,7 @@ async def read_pdf_doc_ocr(pdf_bytes, **kwargs): elm.web.document.PDFDocument PDFDocument instances with pages loaded as text. """ - import pytesseract # noqa: PLC0415 + import pytesseract # ruff:ignore[import-outside-top-level] return await OCRPDFLoader.call( _read_pdf_ocr, @@ -239,7 +239,7 @@ async def read_pdf_file_ocr(pdf_fp, **kwargs): bytes Raw bytes of the PDF file. """ - import pytesseract # noqa: PLC0415 + import pytesseract # ruff:ignore[import-outside-top-level] return await OCRPDFLoader.call( _read_pdf_file_ocr, @@ -444,7 +444,7 @@ def _read_file_docling(fp, **kwargs): def _configure_pytesseract(tesseract_cmd): """Set the tesseract_cmd""" - import pytesseract # noqa: PLC0415 + import pytesseract # ruff:ignore[import-outside-top-level] pytesseract.pytesseract.tesseract_cmd = tesseract_cmd @@ -459,7 +459,7 @@ def _try_decode_ocr_pages(pages): decoded_pages = [] for page in pages: with contextlib.suppress(Exception): - page = ast.literal_eval(page).decode("utf-8") # noqa: PLW2901 + page = ast.literal_eval(page).decode("utf-8") # ruff:ignore[redefined-loop-name] decoded_pages.append(page) return decoded_pages @@ -573,6 +573,6 @@ def flush(self): self.logger.log(self.level, self._buffer) self._buffer = "" - def isatty(self): # noqa: PLR6301 + def isatty(self): # ruff:ignore[no-self-use] """bool: Redirected subprocess streams are never TTYs""" return False diff --git a/compass/services/threaded.py b/compass/services/threaded.py index 619fc5a49..465dd205d 100644 --- a/compass/services/threaded.py +++ b/compass/services/threaded.py @@ -372,7 +372,7 @@ async def process(self, doc, *args): @property @abstractmethod - def _PROCESS(self): # noqa: N802 + def _PROCESS(self): # ruff:ignore[invalid-function-name] """str: Key in `_PROCESSING_FUNCTIONS` defining the doc func""" raise NotImplementedError diff --git a/compass/utilities/enums.py b/compass/utilities/enums.py index 4fd1922c6..1d702a15b 100644 --- a/compass/utilities/enums.py +++ b/compass/utilities/enums.py @@ -31,7 +31,7 @@ def _missing_(cls, value): return None @classmethod - def _new_post_hook(cls, obj, value): # noqa: ARG003 + def _new_post_hook(cls, obj, value): # ruff:ignore[unused-class-method-argument] """Hook for post-processing after __new__""" return obj diff --git a/compass/utilities/finalize.py b/compass/utilities/finalize.py index ca830956c..69c6a2116 100644 --- a/compass/utilities/finalize.py +++ b/compass/utilities/finalize.py @@ -253,7 +253,7 @@ def _empirical_adjustments(db): """ if "adder" in db.columns: - db.loc[db["adder"] > 250, "adder"] = None # noqa: PLR2004 + db.loc[db["adder"] > 250, "adder"] = None # ruff:ignore[magic-value-comparison] return db diff --git a/compass/utilities/io.py b/compass/utilities/io.py index df489ac1c..f2e166c8f 100644 --- a/compass/utilities/io.py +++ b/compass/utilities/io.py @@ -74,7 +74,7 @@ def loads(cls, config_str): @property @abstractmethod - def FILE_EXTENSION(self): # noqa: N802 + def FILE_EXTENSION(self): # ruff:ignore[invalid-function-name] """str: Enum name to use""" diff --git a/compass/utilities/logs.py b/compass/utilities/logs.py index e71227f76..cd0ebef89 100644 --- a/compass/utilities/logs.py +++ b/compass/utilities/logs.py @@ -53,7 +53,7 @@ class LQ: class NoLocationFilter(logging.Filter): """Filter that catches all records without a location attribute""" - def filter(self, record): # noqa: PLR6301 + def filter(self, record): # ruff:ignore[no-self-use] """Filter logging record. Parameters @@ -116,7 +116,7 @@ def filter(self, record): class AddLocationFilter(logging.Filter): """Filter that injects location information into the log record""" - def filter(self, record): # noqa: PLR6301 + def filter(self, record): # ruff:ignore[no-self-use] """Add location to record Parameters @@ -165,7 +165,7 @@ def emit(self, record): ) except asyncio.CancelledError: raise - except Exception: # noqa: BLE001 + except Exception: # ruff:ignore[blind-except] self.handleError(record) @@ -239,7 +239,7 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): self.__exit__(exc_type, exc, tb) - def addHandler(self, handler): # noqa: N802 + def addHandler(self, handler): # ruff:ignore[invalid-function-name] """Add a handler to the queue listener Logs that are sent to the queue will be emitted to the handler. @@ -252,7 +252,7 @@ def addHandler(self, handler): # noqa: N802 if handler not in self._listener.handlers: self._listener.handlers.append(handler) - def removeHandler(self, handler): # noqa: N802 + def removeHandler(self, handler): # ruff:ignore[invalid-function-name] """Remove a handler from the queue listener Logs that are sent to the queue will no longer be emitted to the @@ -406,7 +406,7 @@ async def __aexit__(self, exc_type, exc, tb): class ExceptionOnlyFilter(logging.Filter): """Filter to only pass through Exception logging (errors)""" - def filter(self, record): # noqa: D102, PLR6301 + def filter(self, record): # ruff:ignore[undocumented-public-method, no-self-use] return bool(record.exc_info or getattr(record, "exc_type", None)) @@ -417,7 +417,7 @@ def format(self, record): exc_info, exc_text = _extract_exc_info_from_record(record) message = record.getMessage() - if message and len(message) > 103: # noqa: PLR2004 + if message and len(message) > 103: # ruff:ignore[magic-value-comparison] message = message[:103] return { @@ -604,12 +604,12 @@ def _extract_exc_info_from_record(record): try: exc_text = exc_info[1].args[0] - except Exception: # noqa: BLE001 + except Exception: # ruff:ignore[blind-except] exc_text = None try: exc_info = exc_info[0].__name__ - except Exception: # noqa: BLE001 + except Exception: # ruff:ignore[blind-except] exc_info = None return exc_info, exc_text diff --git a/compass/utilities/parsing.py b/compass/utilities/parsing.py index 6ddc733b3..22635b702 100644 --- a/compass/utilities/parsing.py +++ b/compass/utilities/parsing.py @@ -281,7 +281,7 @@ def convert_paths_to_strings(obj): if isinstance(obj, Path): out = os.fspath(obj) if not obj.is_absolute(): - out = os.path.join(".", out) # noqa PTH118 + out = os.path.join(".", out) # ruff:ignore[os-path-join] PTH118 return out if isinstance(obj, dict): return { diff --git a/compass/validation/content.py b/compass/validation/content.py index 5ec130914..39025840c 100644 --- a/compass/validation/content.py +++ b/compass/validation/content.py @@ -223,25 +223,25 @@ def _count_phrase_matches(self, heuristics_text): @property @abstractmethod - def NOT_TECH_WORDS(self): # noqa: N802 + def NOT_TECH_WORDS(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Not tech keywords""" raise NotImplementedError @property @abstractmethod - def GOOD_TECH_KEYWORDS(self): # noqa: N802 + def GOOD_TECH_KEYWORDS(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Tech keywords""" raise NotImplementedError @property @abstractmethod - def GOOD_TECH_ACRONYMS(self): # noqa: N802 + def GOOD_TECH_ACRONYMS(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Tech acronyms""" raise NotImplementedError @property @abstractmethod - def GOOD_TECH_PHRASES(self): # noqa: N802 + def GOOD_TECH_PHRASES(self): # ruff:ignore[invalid-function-name] """:class:`~collections.abc.Iterable`: Tech phrases""" raise NotImplementedError diff --git a/compass/validation/graphs.py b/compass/validation/graphs.py index e860976a0..f1bc940f7 100644 --- a/compass/validation/graphs.py +++ b/compass/validation/graphs.py @@ -36,7 +36,7 @@ def setup_graph_correct_document_type(**kwargs): """ doc_is_from_ocr = kwargs.pop("doc_is_from_ocr", False) - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Correct document type", **kwargs ) G.add_node( @@ -327,7 +327,7 @@ def setup_graph_correct_jurisdiction_type(jurisdiction, **kwargs): JSON payload keyed by ``correct_jurisdiction`` plus a human-readable explanation summarizing the reasoning. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Correct jurisdiction type", **kwargs ) @@ -564,7 +564,7 @@ def setup_graph_correct_jurisdiction_from_url(jurisdiction, **kwargs): ``correct_county``. The final prompt instructs the LLM to emit a JSON document describing each match plus an explanatory string. """ - G = setup_graph_no_nodes( # noqa: N806 + G = setup_graph_no_nodes( # ruff:ignore[non-lowercase-variable-in-function] d_tree_name="Correct jurisdiction type from URL", **kwargs ) diff --git a/compass/validation/location.py b/compass/validation/location.py index cfd00ce01..f5850bd9c 100644 --- a/compass/validation/location.py +++ b/compass/validation/location.py @@ -112,7 +112,7 @@ async def check(self, url): out = await run_async_tree(tree, response_as_json=True) return self._parse_output(out) - def _parse_output(self, props): # noqa: PLR6301 + def _parse_output(self, props): # ruff:ignore[no-self-use] """Parse LLM response and return boolean validation result""" logger.debug( "Parsing URL jurisdiction validation output:\n\t%s", props @@ -193,7 +193,7 @@ async def check(self, content): out = await run_async_tree(tree, response_as_json=True) return self._parse_output(out) - def _parse_output(self, props): # noqa: PLR6301 + def _parse_output(self, props): # ruff:ignore[no-self-use] """Parse LLM response and return boolean validation result""" logger.debug( "Parsing county jurisdiction validation output:\n\t%s", props diff --git a/compass/validation/utilities.py b/compass/validation/utilities.py index 46d141a31..a0c79fc85 100644 --- a/compass/validation/utilities.py +++ b/compass/validation/utilities.py @@ -20,10 +20,10 @@ def step_based_threshold(num_chunks): indicate a stricter requirement for the fraction of chunks that must pass the validation. """ - if num_chunks <= 2: # noqa: PLR2004 + if num_chunks <= 2: # ruff:ignore[magic-value-comparison] return min(1 / 1, 1 / 2) - if num_chunks <= 6: # noqa: PLR2004 + if num_chunks <= 6: # ruff:ignore[magic-value-comparison] return min(2 / 3, 3 / 4, 3 / 5, 4 / 6) - if num_chunks <= 9: # noqa: PLR2004 + if num_chunks <= 9: # ruff:ignore[magic-value-comparison] return min(5 / 7, 6 / 8, 7 / 9) return 0.8 diff --git a/compass/web/file_loader.py b/compass/web/file_loader.py index e9b696461..8ae6b3afb 100644 --- a/compass/web/file_loader.py +++ b/compass/web/file_loader.py @@ -86,7 +86,7 @@ async def _fetch_doc(self, url): class AsyncDoclingWebFileLoader(BaseAsyncFileLoader): """Async web file loader using Docling""" - def __init__( # noqa: PLR0913, PLR0917 + def __init__( # ruff:ignore[too-many-arguments, too-many-positional-arguments] self, header_template=None, verify_ssl=True, diff --git a/compass/web/website_crawl.py b/compass/web/website_crawl.py index f69c79ff3..ec703bcfb 100644 --- a/compass/web/website_crawl.py +++ b/compass/web/website_crawl.py @@ -15,11 +15,11 @@ from bs4 import BeautifulSoup from rebrowser_playwright.async_api import async_playwright from rebrowser_playwright.async_api import Error as RBPlaywrightError -from playwright._impl._errors import Error as PlaywrightError # noqa: PLC2701 +from playwright._impl._errors import Error as PlaywrightError # ruff:ignore[import-private-name] from elm.web.utilities import pw_page from elm.web.document import HTMLDocument from elm.web.file_loader import AsyncWebFileLoader -from elm.web.website_crawl import ELMLinkScorer, _SCORE_KEY # noqa: PLC2701 +from elm.web.website_crawl import ELMLinkScorer, _SCORE_KEY # ruff:ignore[import-private-name] from compass.utilities.url import sanitize_url from compass.services.threaded import TempFileCache @@ -529,7 +529,7 @@ def _crawl_depth_counts(self): return depth_counts -async def _default_found_enough_docs(out_docs): # noqa: RUF029 +async def _default_found_enough_docs(out_docs): # ruff:ignore[unused-async] """Check if a predetermined # of documents has been found The number to check is set by the module-level constant @@ -551,7 +551,7 @@ def _debug_info_on_links(links): logger.debug( " - %d: %s (%s)", link["score"], link["title"], link["href"] ) - if num_links > 3: # noqa: PLR2004 + if num_links > 3: # ruff:ignore[magic-value-comparison] logger.debug(" ...")