Update web crawl conditions

compass/_cli/common.py

@@ -97,11 +97,11 @@ def setup_cli_logging(console, verbosity_level, log_level="INFO"):
     libs = []
     if verbosity_level >= 1:
         libs.append("compass")
-    if verbosity_level >= 2:  # noqa: PLR2004
+    if verbosity_level >= 2:  # ruff:ignore[magic-value-comparison]
         libs.extend(("elm", "docling"))
-    if verbosity_level >= 3:  # noqa: PLR2004
+    if verbosity_level >= 3:  # ruff:ignore[magic-value-comparison]
         libs.append("openai")
-    if verbosity_level >= 4:  # noqa: PLR2004
+    if verbosity_level >= 4:  # ruff:ignore[magic-value-comparison]
         libs.extend(("networkx", "pytesseract", "pdf2image", "pdftotext"))
 
     for lib in libs:

compass/common/base.py

@@ -150,7 +150,7 @@ def setup_async_decision_tree(
     The function asserts that the tree has recorded at least the system
     prompt before returning the constructed wrapper.
     """
-    G = graph_setup_func(**kwargs)  # noqa: N806
+    G = graph_setup_func(**kwargs)  # ruff:ignore[non-lowercase-variable-in-function]
     tree = AsyncDecisionTree(G, usage_sub_label=usage_sub_label)
     assert len(tree.chat_llm_caller.messages) == 1
     return tree
@@ -241,7 +241,7 @@ def setup_base_setback_graph(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Base setback questions", **kwargs
     )
 
@@ -315,7 +315,7 @@ def setup_participating_owner(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Participating owner", **kwargs
     )
 
@@ -434,7 +434,7 @@ def setup_graph_extra_restriction(is_numerical=True, **kwargs):
     kwargs.setdefault("unit_clarification", "")
     kwargs.setdefault("feature_clarifications", "")
     feature_id = kwargs.get("feature_id", "")
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Extra restriction", **kwargs
     )
 
@@ -571,7 +571,7 @@ def setup_graph_extra_restriction(is_numerical=True, **kwargs):
     return G
 
 
-def _add_other_system_setback_clarification_nodes(G):  # noqa: N803
+def _add_other_system_setback_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify "other system" setbacks"""
     G.add_edge("init", "is_intra_farm", condition=llm_response_starts_with_yes)
     G.add_node(
@@ -592,7 +592,7 @@ def _add_other_system_setback_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_coverage_clarification_nodes(G):  # noqa: N803
+def _add_coverage_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify "coverage" extraction"""
     G.add_edge("init", "is_area", condition=llm_response_starts_with_yes)
     G.add_node(
@@ -603,7 +603,7 @@ def _add_coverage_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_land_density_clarification_nodes(G):  # noqa: N803
+def _add_land_density_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify "land density" extraction"""
     G.add_edge(
         "init", "correct_density_units", condition=llm_response_starts_with_yes
@@ -623,7 +623,7 @@ def _add_land_density_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_minimum_lot_size_clarification_nodes(G):  # noqa: N803
+def _add_minimum_lot_size_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify "minimum lot size" extraction"""
     G.add_edge(
         "init", "correct_min_ls_units", condition=llm_response_starts_with_yes
@@ -641,7 +641,7 @@ def _add_minimum_lot_size_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_maximum_lot_size_clarification_nodes(G):  # noqa: N803
+def _add_maximum_lot_size_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify "maximum lot size" extraction"""
     G.add_edge(
         "init", "correct_max_ls_units", condition=llm_response_starts_with_yes
@@ -659,7 +659,7 @@ def _add_maximum_lot_size_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_maximum_project_size_clarification_nodes(G):  # noqa: N803
+def _add_maximum_project_size_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify "max project size" extraction"""
     G.add_edge("init", "is_mps_area", condition=llm_response_starts_with_yes)
     G.add_node(
@@ -691,7 +691,7 @@ def _add_maximum_project_size_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_maximum_turbine_height_clarification_nodes(G):  # noqa: N803
+def _add_maximum_turbine_height_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify max turbine height extraction"""
     G.add_edge("init", "has_relative", condition=llm_response_starts_with_yes)
     G.add_node(
@@ -726,7 +726,7 @@ def _add_maximum_turbine_height_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_value_and_units_clarification_nodes(G):  # noqa: N803
+def _add_value_and_units_clarification_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to clarify value and units extraction"""
 
     G.add_node(
@@ -796,7 +796,7 @@ def _add_value_and_units_clarification_nodes(G):  # noqa: N803
     return G
 
 
-def _add_prohibitions_extraction_nodes(G):  # noqa: N803
+def _add_prohibitions_extraction_nodes(G):  # ruff:ignore[invalid-argument-name]
     """Add nodes and edges to extract 'prohibitions'"""
 
     G.add_edge("init", "is_proposed", condition=llm_response_starts_with_yes)
@@ -888,7 +888,7 @@ def setup_graph_permitted_use_districts(**kwargs):
         `elm.tree.DecisionTree`.
     """
     feature_id = kwargs.get("feature_id", "")
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Permitted use districts", **kwargs
     )
 

compass/data/conus_jurisdictions.csv

@@ -6559,7 +6559,7 @@ Florida,Santa Rosa,Skyline,city,1211393127,
 Florida,Santa Rosa,Whiting Field,city,1211393659,
 Florida,Santa Rosa,,county,12113,https://www.santarosa.fl.gov/
 Florida,Sarasota,Englewood,city,1211590975,
-Florida,Sarasota,Interior,county,1211591599,https://www.scgov.net/
+Florida,Sarasota,Interior,region,1211591599,https://www.scgov.net/
 Florida,Sarasota,Longboat Key,town,1241150,https://www.longboatkey.org/
 Florida,Sarasota,North Port,city,1249675,https://www.northportfl.gov/
 Florida,Sarasota,Osprey-Laurel-Nokomis,city,1211592542,

compass/extraction/context.py

@@ -157,9 +157,9 @@ def multi_doc_context(self, attr_text_key=None):
         return f"## MULTI-DOCUMENT CONTEXT ##\n\n{serialized}"
 
 
-async def _move_file_to_out_dir(doc, out_fn):
+async def _move_file_to_out_dir(doc, out_stem):
     """Move PDF or HTML text file to output directory"""
-    out_fp = await FileMover.call(doc, out_fn)
+    out_fp = await FileMover.call(doc, out_stem)
     doc.attrs["out_fp"] = out_fp
     return doc
 

compass/extraction/ghp/plugin_config.yaml

@@ -93,6 +93,9 @@ heuristic_keywords:
     - "geothermal production project"
     - "exploratory well"
     - "injection well"
+    - "wellbeing"
+    - "well-being"
+    - "as well as"
 
 collection_prompts: True
 

compass/extraction/small_wind/graphs.py

@@ -21,7 +21,7 @@ def setup_graph_wes_types(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Wind Energy Farm types", **kwargs
     )
 
@@ -154,7 +154,7 @@ def setup_multiplier(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Setback distance", **kwargs
     )
 
@@ -362,7 +362,7 @@ def setup_conditional_min(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Minimum setback distance", **kwargs
     )
 
@@ -440,7 +440,7 @@ def setup_conditional_max(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Maximum setback distance", **kwargs
     )
 

compass/extraction/solar/graphs.py

@@ -21,7 +21,7 @@ def setup_graph_sef_types(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Solar Energy Farm types", **kwargs
     )
 
@@ -156,7 +156,7 @@ def setup_multiplier(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Setback distance", **kwargs
     )
 

compass/extraction/water/graphs.py

@@ -21,7 +21,7 @@ def setup_graph_permits(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Permit Requirements", **kwargs
     )
 
@@ -91,7 +91,7 @@ def setup_graph_extraction(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Water Extraction Requirements", **kwargs
     )
 
@@ -149,7 +149,7 @@ def setup_graph_geothermal(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Geothermal Policies", **kwargs
     )
 
@@ -213,7 +213,7 @@ def setup_graph_oil_and_gas(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Oil and Gas Policies", **kwargs
     )
 
@@ -277,7 +277,7 @@ def setup_graph_limits(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Extraction Limits", **kwargs
     )
 
@@ -387,7 +387,7 @@ def setup_graph_well_spacing(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Well Spacing", **kwargs
     )
 
@@ -481,7 +481,7 @@ def setup_graph_time(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Drilling Window", **kwargs
     )
 
@@ -546,7 +546,7 @@ def setup_graph_metering_device(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Metering Device", **kwargs
     )
 
@@ -605,7 +605,7 @@ def setup_graph_drought(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Drought Management Plan", **kwargs
     )
 
@@ -669,7 +669,7 @@ def setup_graph_contingency(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Contingency Plan Requirements", **kwargs
     )
 
@@ -733,7 +733,7 @@ def setup_graph_plugging_reqs(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Plugging Requirements", **kwargs
     )
 
@@ -793,7 +793,7 @@ def setup_graph_external_transfer(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="External Transfer Restrictions", **kwargs
     )
 
@@ -902,7 +902,7 @@ def setup_graph_production_reporting(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Water Well Production Reporting", **kwargs
     )
 
@@ -962,7 +962,7 @@ def setup_graph_production_cost(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Water Well Production Cost", **kwargs
     )
 
@@ -1045,7 +1045,7 @@ def setup_graph_setback_features(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Setback Features", **kwargs
     )
 
@@ -1109,7 +1109,7 @@ def setup_graph_redrilling(**kwargs):
         Graph instance that can be used to initialize an
         `elm.tree.DecisionTree`.
     """
-    G = setup_graph_no_nodes(  # noqa: N806
+    G = setup_graph_no_nodes(  # ruff:ignore[non-lowercase-variable-in-function]
         d_tree_name="Redrilling Restrictions", **kwargs
     )