diff --git a/compass/common/base.py b/compass/common/base.py index e779fd669..75954004b 100644 --- a/compass/common/base.py +++ b/compass/common/base.py @@ -24,11 +24,13 @@ "and `null` otherwise." ) _SUMMARY_PROMPT = ( - "The value of the 'summary' key should be a short summary " - "of the ordinance, using direct text excerpts as much as possible." + "The value of the 'summary' key should be a short summary of the relevant " + "ordinance, **using direct text excerpts as much as possible.** " + "If you had to make a selection when reporting the ordinance, be sure to " + "list out all the other options and their conditions in the summary." ) _UNITS_IN_SUMMARY_PROMPT = ( - "Include any clarifications about the units in the summary." + "Also include any clarifications about the units in the summary." ) EXTRACT_ORIGINAL_SETBACK_TEXT_PROMPT = ( "Extract all portions of the text (with original formatting) " @@ -189,7 +191,7 @@ def setup_base_setback_graph(**kwargs): "{feature_clarifications}" # expected to end in space "Please consider only setbacks from {feature}. " "Please also only consider setbacks that would apply for " - "{system_size_reminder}" + "{system_size_reminder}" # expected to end in space "Don't forget to pay extra attention to clarifying text found " "in parentheses and footnotes. " "Please start your response with either 'Yes' or 'No' and briefly " @@ -465,6 +467,54 @@ def setup_graph_extra_restriction(is_numerical=True, **kwargs): ), ) + G.add_edge("init", "enr", condition=llm_response_starts_with_no) + G.add_node( + "enr", + prompt=( + "Does the legal text **directly** mention not regulating " + "{restriction} for {tech}? " + "As before, focus only on {restriction} specifically for " + "{system_size_reminder}" + "Please start your response with either 'Yes' or 'No' and " + "briefly explain your answer." + ), + ) + G.add_edge("enr", "enr_text", condition=llm_response_starts_with_yes) + G.add_node( + "enr_text", + prompt=( + "What is the **exact text excerpt** that explicitly states there " + "are no regulations around {restriction} for {tech}? " + ), + ) + G.add_edge("enr_text", "enr_validate") + G.add_node( + "enr_validate", + prompt=( + "Based on your response, are you still confident that the text " + "**directly** states that there are no regulations around " + "{restriction} for {tech}? " + "Please start your response with either 'Yes' or 'No' and " + "briefly explain your answer." + ), + ) + G.add_edge( + "enr_validate", "final_enr", condition=llm_response_starts_with_yes + ) + G.add_node( + "final_enr", + prompt=( + "Please respond based on our entire conversation so far. " + "Return your answer as a dictionary in JSON format (not " + "markdown). Your JSON file must include exactly three keys. " + "The keys are 'value', 'summary', and 'section'. The value of " + "the 'value' key should be the string 'ENR' if the text " + "clearly states that the jurisdiction does not regulate " + "{restriction} for {tech} or `null` otherwise. " + "{SUMMARY_PROMPT} {SECTION_PROMPT}" + ), + ) + return G diff --git a/compass/extraction/features.py b/compass/extraction/features.py index 8761f8ef2..7cb8830d1 100644 --- a/compass/extraction/features.py +++ b/compass/extraction/features.py @@ -32,6 +32,11 @@ class SetbackFeatures: "transmission lines", ], "water": ["lakes", "reservoirs", "streams", "rivers", "wetlands"], + "public conservation lands": [ + "public conservation lands", + "natural resource protection areas", + "preservation areas", + ], } """Aliases for mutually-exclusive setback features""" FEATURES_AS_IGNORE = { @@ -41,6 +46,7 @@ class SetbackFeatures: "railroads": "railroads", "transmission": "transmission lines", "water": "wetlands", + "public conservation lands": "public conservation lands", } """Features as they should appear in ignore phrases""" FEATURE_CLARIFICATIONS = { @@ -50,6 +56,11 @@ class SetbackFeatures: "parcel boundaries unless the text **explicitly** makes that " "connection. " ), + "water": ( + "Public conservation lands (or similar) **are not equivalent** to " + "wetlands (or similar) unless the text **explicitly** makes that " + "connection. " + ), "roads": "Roads may also be labeled as rights-of-way. ", } """Clarifications to add to feature prompts""" diff --git a/compass/extraction/accessory_wind/__init__.py b/compass/extraction/small_wind/__init__.py similarity index 58% rename from compass/extraction/accessory_wind/__init__.py rename to compass/extraction/small_wind/__init__.py index 90d16a535..df63ae613 100644 --- a/compass/extraction/accessory_wind/__init__.py +++ b/compass/extraction/small_wind/__init__.py @@ -1,29 +1,29 @@ -"""Accessory wind ordinance extraction utilities""" +"""Small wind ordinance extraction utilities""" from .ordinance import ( - AccessoryWindHeuristic, - AccessoryWindOrdinanceTextCollector, - AccessoryWindOrdinanceTextExtractor, - AccessoryWindPermittedUseDistrictsTextCollector, - AccessoryWindPermittedUseDistrictsTextExtractor, + SmallWindHeuristic, + SmallWindOrdinanceTextCollector, + SmallWindOrdinanceTextExtractor, + SmallWindPermittedUseDistrictsTextCollector, + SmallWindPermittedUseDistrictsTextExtractor, ) from .parse import ( - StructuredAccessoryWindOrdinanceParser, - StructuredAccessoryWindPermittedUseDistrictsParser, + StructuredSmallWindOrdinanceParser, + StructuredSmallWindPermittedUseDistrictsParser, ) -ACCESSORY_WIND_QUESTION_TEMPLATES = [ +SMALL_WIND_QUESTION_TEMPLATES = [ "filetype:pdf {jurisdiction} wind energy conversion system ordinances", "wind energy conversion system ordinances {jurisdiction}", "{jurisdiction} wind WECS ordinance", - "Where can I find the legal text for accessory wind energy " + "Where can I find the legal text for small wind energy " "turbine zoning ordinances in {jurisdiction}?", "What is the specific legal information regarding zoning " - "ordinances for accessory wind turbines in {jurisdiction}?", + "ordinances for small wind turbines in {jurisdiction}?", ] -BEST_ACCESSORY_WIND_ORDINANCE_WEBSITE_URL_KEYWORDS = { +BEST_SMALL_WIND_ORDINANCE_WEBSITE_URL_KEYWORDS = { "pdf": 92160, "wecs": 46080, "wind": 23040, diff --git a/compass/extraction/accessory_wind/graphs.py b/compass/extraction/small_wind/graphs.py similarity index 92% rename from compass/extraction/accessory_wind/graphs.py rename to compass/extraction/small_wind/graphs.py index 8f17a67c1..4091ffa23 100644 --- a/compass/extraction/accessory_wind/graphs.py +++ b/compass/extraction/small_wind/graphs.py @@ -8,7 +8,7 @@ def setup_graph_wes_types(**kwargs): - """Setup graph to get the accessory turbine size in the text + """Setup graph to get the small turbine size in the text Parameters ---------- @@ -49,23 +49,25 @@ def setup_graph_wes_types(**kwargs): "include generic types or other energy system types." ), ) - G.add_edge("get_text", "get_accessory") + G.add_edge("get_text", "get_small_label") G.add_node( - "get_accessory", + "get_small_label", prompt=( "Based on your list, what is the wind energy system size that is " - "**closest to the definition of accessory wind energy systems** " - "that is also **regulated by this ordinance**?" + "**closest to the definition of small, medium, or non-commercial " + "wind energy systems** that is also **regulated by this " + "ordinance**?" ), ) - G.add_edge("get_accessory", "check_matches_definition") + G.add_edge("get_small_label", "check_matches_definition") G.add_node( "check_matches_definition", prompt=( "Does the ordinance explicitly define this system as small, " - "medium, private, accessory, or something akin to that (i.e. " - "**not** large, commercial, or utility-scale)? " + "medium, non-commercial, or something akin to that (i.e. " + "**not** private, micro, building-mounted and **not** large, " + "commercial, or utility-scale)? " "Please start your response with either 'Yes' or 'No' and briefly " "explain your answer." ), @@ -87,7 +89,7 @@ def setup_graph_wes_types(**kwargs): "check_scale_reason", prompt=( "Would a reasonable person classify this kind of system as a " - "**small, private, medium, accessory, or even residential** wind " + "**small, medium, or non-commercial** wind " "energy generation system (e.g. with the primary purpose of " "generating electricity for use on-site, as opposed to large, " "commercial, utility-scale, or other kinds of 'large' systems)? " @@ -112,14 +114,14 @@ def setup_graph_wes_types(**kwargs): "Respond based on our entire conversation so far. Return your " "answer as a dictionary in JSON format (not markdown). Your JSON " "file must include exactly three keys. The keys are " - "'wes_type', 'explanation', and 'is_accessory'. The value of " + "'wes_type', 'explanation', and 'is_small'. The value of " "the 'wes_type' key should be a string that labels the system " - "size **closest to the definition of accessory wind energy " - "systems** that is also **regulated by this ordinance**. " - "The value of the 'explanation' key should be a " + "size **closest to the definition of small, medium, or " + "non-commercial wind energy systems** that is also **regulated by " + "this ordinance**. The value of the 'explanation' key should be a " "string containing a short explanation for your choice. The value " - "of the 'is_accessory' key should be the boolean value `true`, " - "since we determined this is an accessory system." + "of the 'is_small' key should be the boolean value `true`, " + "since we determined this is a small or non-commercial system." ), ) G.add_node( @@ -128,14 +130,14 @@ def setup_graph_wes_types(**kwargs): "Respond based on our entire conversation so far. Return your " "answer as a dictionary in JSON format (not markdown). Your JSON " "file must include exactly three keys. The keys are " - "'wes_type', 'explanation', and 'is_accessory'. The value of " + "'wes_type', 'explanation', and 'is_small'. The value of " "the 'wes_type' key should be a string that labels the system " - "size **closest to the definition of accessory wind energy " - "systems** that is also **regulated by this ordinance**. " - "The value of the 'explanation' key should be a " + "size **closest to the definition of small, medium, or " + "non-commercial wind energy systems** that is also **regulated by " + "this ordinance**. The value of the 'explanation' key should be a " "string containing a short explanation for your choice. The value " - "of the 'is_accessory' key should be the boolean value `false`, " - "since we determined this is not an accessory system." + "of the 'is_small' key should be the boolean value `false`, " + "since we determined this is not a small or non-commercial system." ), ) return G diff --git a/compass/extraction/accessory_wind/ordinance.py b/compass/extraction/small_wind/ordinance.py similarity index 82% rename from compass/extraction/accessory_wind/ordinance.py rename to compass/extraction/small_wind/ordinance.py index c88389597..fc838465e 100644 --- a/compass/extraction/accessory_wind/ordinance.py +++ b/compass/extraction/small_wind/ordinance.py @@ -1,7 +1,7 @@ -"""Accessory wind ordinance document content collection and extraction +"""Small wind ordinance document content collection and extraction These methods help filter down the document text to only the portions -relevant to accessory wind ordinances. +relevant to small wind ordinances. """ import logging @@ -16,25 +16,23 @@ logger = logging.getLogger(__name__) -_ACCESSORY_WES_SYNONYMS = ( - "wind turbines, wind energy conversion systems (WECS), wind energy " - "turbines (WET), small wind energy turbines (SWET), private wind energy " - "turbines (PWET), on-site wind energy systems, distributed wind energy " - "systems (DWES), medium wind energy systems (MWES), agricultural wind " - "energy systems (AWES), residential wind energy systems (RWES), personal " - "wind energy systems (PWES), private wind turbines (PWT), micro turbines, " - "small wind turbines (SWT), accessory wind energy conversion systems " - "(AWECS), alternate energy systems (AES), or similar" +_SMALL_WES_SYNONYMS = ( + "small wind energy turbines (SWET), non-commercial wind energy systems, " + "on-site wind energy systems, distributed wind energy systems, medium " + "wind energy systems (MWES), agricultural wind energy systems (AWES), " + "residential wind energy systems, small wind turbines (SWT), " + "or similar" ) _SEARCH_TERMS_AND = ( "zoning, siting, setback, system design, and operational " "requirements/restrictions" ) _SEARCH_TERMS_OR = _SEARCH_TERMS_AND.replace("and", "or") -_IGNORE_TYPES = "large, utility-scale, for-sale, or commercial" +_IGNORE_TYPES_MICRO = "private, micro, personal, building-mounted" +_IGNORE_TYPES_LARGE = "large, utility-scale, for-sale, commercial" -class AccessoryWindHeuristic(Heuristic): +class SmallWindHeuristic(Heuristic): """Perform a heuristic check for mention of wind turbines in text""" NOT_TECH_WORDS = [ @@ -73,27 +71,15 @@ class AccessoryWindHeuristic(Heuristic): GOOD_TECH_ACRONYMS = ["wecs", "wes", "swet", "pwet", "wef"] """Acronyms for WECS that we want to capture""" GOOD_TECH_PHRASES = [ - "micro wecs", - "micro turbine", - "micro wind", "small wecs", "small turbine", "small wind", - "mini wecs", - "mini turbine", - "mini wind", "medium wecs", "medium turbine", "medium wind", - "private wecs", - "private turbine", - "private wind", "accessory wecs", "accessory turbine", "accessory wind", - "personal wecs", - "personal turbine", - "personal wind", "on-site wecs", "on-site turbine", "on-site wind", @@ -131,7 +117,7 @@ class AccessoryWindHeuristic(Heuristic): """Phrases that indicate text is about WECS""" -class AccessoryWindOrdinanceTextCollector(StructuredLLMCaller): +class SmallWindOrdinanceTextCollector(StructuredLLMCaller): """Check text chunks for ordinances and collect them if they do""" CONTAINS_ORD_PROMPT = ( @@ -155,24 +141,26 @@ class AccessoryWindOrdinanceTextCollector(StructuredLLMCaller): ) """Prompt to check if chunk contains WES ordinance info""" - IS_ACCESSORY_PROMPT = ( + IS_SMALL_PROMPT = ( "You are a legal scholar that reads ordinance text and determines " - f"whether any of it applies to {_SEARCH_TERMS_OR} for " - "**accessory wind energy systems**. Accessory wind energy systems " - f"(AWES) may also be referred to as {_ACCESSORY_WES_SYNONYMS}. " - "Your client is a private resident that does not " - f"care about ordinances related to {_IGNORE_TYPES} wind energy " - "systems. Ignore any text related to such systems. " + f"whether any of it applies to {_SEARCH_TERMS_OR} for **small, " + "medium, or non-commercial wind energy systems**. Small, medium, or " + "non-commercial energy systems may also be referred to as " + f"{_SMALL_WES_SYNONYMS}. " + "Your client is a private resident that does not care about " + f"ordinances related to {_IGNORE_TYPES_MICRO} or " + f"{_IGNORE_TYPES_LARGE} wind energy systems. Ignore any text " + "related to such systems. " "Return your answer as a dictionary in JSON format (not markdown). " "Your JSON file must include exactly two keys. The first key is " "'summary' which contains a string that lists all of the types of " "wind energy systems the text applies to (if any). The second key is " "'{key}', which is a boolean that is set to True if any part of the " - f"text excerpt details {_SEARCH_TERMS_OR} for the **accessory wind " - "energy conversion systems** (or similar) that the client is " - "interested in and False otherwise." + f"text excerpt details {_SEARCH_TERMS_OR} for the **small, medium, or " + "non-commercial wind energy conversion systems** (or similar) that " + "the client is interested in and False otherwise." ) - """Prompt to check if chunk is for accessory WES""" + """Prompt to check if chunk is for small WES""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -192,8 +180,7 @@ async def check_chunk(self, chunk_parser, ind): ------- bool Boolean flag indicating whether or not the text in the chunk - contains accessory wind energy conversion system ordinance - text. + contains small wind energy conversion system ordinance text. """ contains_ord_info = await chunk_parser.parse_from_ind( ind, @@ -206,16 +193,16 @@ async def check_chunk(self, chunk_parser, ind): logger.debug("Text at ind %d does contain ordinance info", ind) - is_accessory_scale = await chunk_parser.parse_from_ind( + is_small_scale = await chunk_parser.parse_from_ind( ind, key="x", - llm_call_callback=self._check_chunk_is_for_accessory_scale, + llm_call_callback=self._check_chunk_is_for_small_scale, ) - if not is_accessory_scale: - logger.debug("Text at ind %d is not for accessory WECS", ind) + if not is_small_scale: + logger.debug("Text at ind %d is not for small WECS", ind) return False - logger.debug("Text at ind %d is for accessory WECS", ind) + logger.debug("Text at ind %d is for small WECS", ind) _store_chunk(chunk_parser, ind, self._ordinance_chunks) logger.debug("Added text at ind %d to ordinances", ind) @@ -253,10 +240,10 @@ async def _check_chunk_contains_ord(self, key, text_chunk): logger.debug("LLM response: %s", content) return content.get(key, False) - async def _check_chunk_is_for_accessory_scale(self, key, text_chunk): - """Call LLM on a chunk of text to check for accessory scale""" + async def _check_chunk_is_for_small_scale(self, key, text_chunk): + """Call LLM on a chunk of text to check for small scale""" content = await self.call( - sys_msg=self.IS_ACCESSORY_PROMPT.format(key=key), + sys_msg=self.IS_SMALL_PROMPT.format(key=key), content=text_chunk, usage_sub_label=(LLMUsageCategory.DOCUMENT_CONTENT_VALIDATION), ) @@ -264,28 +251,29 @@ async def _check_chunk_is_for_accessory_scale(self, key, text_chunk): return content.get(key, False) -class AccessoryWindPermittedUseDistrictsTextCollector(StructuredLLMCaller): +class SmallWindPermittedUseDistrictsTextCollector(StructuredLLMCaller): """Check text chunks for permitted wind districts; collect them""" DISTRICT_PROMPT = ( "You are a legal scholar that reads ordinance text and determines " "whether the text explicitly contains relevant information to " "determine the districts (and especially the district names) where " - "accessory wind energy systems are a permitted use " - "(i.e. accessory use), as well as the districts where wind energy" - "systems are prohibited entirely. Accessory wind energy systems " - f"(AWES) may also be referred to as {_ACCESSORY_WES_SYNONYMS}. " + "small, medium, or non-commercial wind energy systems are a permitted " + "use (i.e. accessory use), as well as the districts where wind energy" + "systems are prohibited entirely. Small wind energy systems " + f"(SWES) may also be referred to as {_SMALL_WES_SYNONYMS}. " "Do not make any inferences; only answer based on information that " "is explicitly stated in the text. " "Note that relevant information may sometimes be found in tables. " "Return your answer as a dictionary in JSON format (not markdown). " "Your JSON file must include exactly two keys. The first key is " "'districts' which contains a string that lists all of the district " - "names for which the text explicitly permits **accessory wind energy " - "systems** (if any). The last key is '{key}', which is a boolean that " - "is set to True if any part of the text excerpt provides information " - "on districts where **accessory wind energy systems** (or similar) " - "are a permitted use (i.e. accessory use) in and False otherwise." + "names for which the text explicitly permits **small, medium, or " + "non-commercial wind energy systems** (if any). The last key is " + "'{key}', which is a boolean that is set to True if any part of the " + "text excerpt provides information on districts where **small, " + "medium, or non-commercial wind energy systems** (or similar) are a " + "permitted use (i.e. accessory use) in and False otherwise." ) """Prompt to check if chunk contains info on permitted districts""" @@ -307,8 +295,8 @@ async def check_chunk(self, chunk_parser, ind): ------- bool Boolean flag indicating whether or not the text in the chunk - contains accessory wind energy conversion system permitted - use text. + contains small wind energy conversion system permitted use + text. """ key = "contains_district_info" @@ -351,14 +339,14 @@ def permitted_use_district_text(self): return merge_overlapping_texts(text) -class AccessoryWindOrdinanceTextExtractor(BaseTextExtractor): +class SmallWindOrdinanceTextExtractor(BaseTextExtractor): """Extract succinct ordinance text from input Purpose: Extract relevant ordinance text from document. Responsibilities: 1. Extract portions from chunked document text relevant to - particular ordinance type (e.g. wind zoning for accessory + particular ordinance type (e.g. wind zoning for small wind systems). Key Relationships: Uses a StructuredLLMCaller for LLM queries. @@ -406,27 +394,28 @@ class AccessoryWindOrdinanceTextExtractor(BaseTextExtractor): ) """Prompt to extract ordinance text for WECS""" - ACCESSORY_WIND_ENERGY_SYSTEM_SECTION_FILTER_PROMPT = ( + SMALL_WIND_ENERGY_SYSTEM_SECTION_FILTER_PROMPT = ( "# CONTEXT #\n" "We want to reduce the provided excerpt to only contain information " - "about **accessory wind energy systems**. The extracted text will be " - "used for structured data extraction, so it must be both " - "**comprehensive** (retaining all relevant details) and **focused** " - "(excluding unrelated content), with **zero rewriting or " + "about **small, medium, or non-commercial wind energy systems**. The " + "extracted text will be used for structured data extraction, so it " + "must be both **comprehensive** (retaining all relevant details) and " + "**focused** (excluding unrelated content), with **zero rewriting or " "paraphrasing**. Ensure that all retained information " - "is **directly applicable** to accessory wind energy systems while " - "preserving full context and accuracy.\n" + "is **directly applicable** to small, medium, or non-commercial wind " + "energy systems while preserving full context and accuracy.\n" "\n# OBJECTIVE #\n" - "Extract all text **pertaining to accessory wind energy systems** " - "from the provided excerpt.\n" + "Extract all text **pertaining to small, medium or non-commercial " + "wind energy systems** from the provided excerpt.\n" "\n# RESPONSE #\n" "Follow these guidelines carefully:\n" "\n1. ## Scope of Extraction ##:\n" - "- Include all text that pertains to **accessory wind energy " - "systems**, even if they are referred to by different names such as:\n" - f"\t{_ACCESSORY_WES_SYNONYMS.capitalize()}.\n" + "- Include all text that pertains to **small, medium, or " + "non-commercial wind energy systems**, even if they are referred to " + "by different names such as:\n" + f"\t{_SMALL_WES_SYNONYMS.capitalize()}.\n" "- Explicitly include any text related to **bans or prohibitions** " - "on accessory wind energy systems.\n" + "on small, medium, or non-commercial wind energy systems.\n" "- Explicitly include any text related to the adoption or enactment " "date of the ordinance (if any).\n" "- **Retain all relevant technical, design, operational, safety, " @@ -441,7 +430,8 @@ class AccessoryWindOrdinanceTextExtractor(BaseTextExtractor): "\t- All other **closely related provisions**.\n" "\n2. ## Exclusions ##:\n" "- Do **not** include text that explicitly applies **only** to " - f"{_IGNORE_TYPES} wind energy systems.\n" + f"{_IGNORE_TYPES_MICRO} or {_IGNORE_TYPES_LARGE} " + "wind energy systems.\n" f"- Do **not** include text that does not pertain at all to wind " "energy systems.\n" "\n3. ## Formatting & Structure ##:\n" @@ -460,7 +450,7 @@ class AccessoryWindOrdinanceTextExtractor(BaseTextExtractor): "- If **no relevant text** is found, return the response: " "'No relevant text.'" ) - """Prompt to extract ordinance text for accessory WECS""" + """Prompt to extract ordinance text for small WECS""" async def extract_wind_energy_system_section(self, text_chunks): """Extract ordinance text from input text chunks for WES @@ -483,8 +473,8 @@ async def extract_wind_energy_system_section(self, text_chunks): is_valid_chunk=_valid_chunk, ) - async def extract_accessory_wind_energy_system_section(self, text_chunks): - """Extract accessory WES ordinance text from input text chunks + async def extract_small_wind_energy_system_section(self, text_chunks): + """Extract small WES ordinance text from input text chunks Parameters ---------- @@ -500,9 +490,7 @@ async def extract_accessory_wind_energy_system_section(self, text_chunks): """ return await self._process( text_chunks=text_chunks, - instructions=( - self.ACCESSORY_WIND_ENERGY_SYSTEM_SECTION_FILTER_PROMPT - ), + instructions=self.SMALL_WIND_ENERGY_SYSTEM_SECTION_FILTER_PROMPT, is_valid_chunk=_valid_chunk, ) @@ -524,18 +512,18 @@ def parsers(self): ) yield ( "cleaned_ordinance_text", - self.extract_accessory_wind_energy_system_section, + self.extract_small_wind_energy_system_section, ) -class AccessoryWindPermittedUseDistrictsTextExtractor(BaseTextExtractor): +class SmallWindPermittedUseDistrictsTextExtractor(BaseTextExtractor): """Extract succinct ordinance text from input Purpose: Extract relevant ordinance text from document. Responsibilities: 1. Extract portions from chunked document text relevant to - particular ordinance type (e.g. wind zoning for accessory + particular ordinance type (e.g. wind zoning for small wind systems). Key Relationships: Uses a StructuredLLMCaller for LLM queries. @@ -656,7 +644,7 @@ async def extract_permitted_uses(self, text_chunks): ) async def extract_wes_permitted_uses(self, text_chunks): - """Extract permitted uses text for accessory WES from input text + """Extract permitted uses text for small WES from input text Parameters ---------- diff --git a/compass/extraction/accessory_wind/parse.py b/compass/extraction/small_wind/parse.py similarity index 89% rename from compass/extraction/accessory_wind/parse.py rename to compass/extraction/small_wind/parse.py index e9eba7530..7e1273f08 100644 --- a/compass/extraction/accessory_wind/parse.py +++ b/compass/extraction/small_wind/parse.py @@ -1,4 +1,4 @@ -"""Accessory wind ordinance structured parsing class""" +"""Small wind ordinance structured parsing class""" import asyncio import logging @@ -21,7 +21,7 @@ setup_graph_extra_restriction, setup_graph_permitted_use_districts, ) -from compass.extraction.accessory_wind.graphs import ( +from compass.extraction.small_wind.graphs import ( setup_graph_wes_types, setup_multiplier, setup_conditional_min, @@ -35,15 +35,17 @@ logger = logging.getLogger(__name__) DEFAULT_SYSTEM_MESSAGE = ( "You are a legal scholar informing a private resident about local " - "zoning ordinances for accessory wind energy systems. " + "zoning ordinances for small, medium, or non-commercial wind energy " + "systems. " ) SYSTEM_SIZE_REMINDER = ( "systems that would typically be defined as {tech} based on the text " "itself — for example, systems intended for onsite electricity " "generation or sale, systems that are a secondary or accessory use on " - "a parsel, or systems below defined thresholds such as height or rated " + "a parcel, or systems below defined thresholds such as height or rated " "capacity (often <1MW). Do not consider any text that applies **only** " - "to larger, utility, or commercial systems. " + "to private, micro, or building mounted systems, or larger, utility, or " + "commercial systems. " ) SETBACKS_SYSTEM_MESSAGE = ( f"{DEFAULT_SYSTEM_MESSAGE} " @@ -133,8 +135,8 @@ } -class AccessoryWindSetbackFeatures(SetbackFeatures): - """Mutually-exclusive feature descriptions for accessory wind""" +class SmallWindSetbackFeatures(SetbackFeatures): + """Mutually-exclusive feature descriptions for small wind systems""" DEFAULT_FEATURE_DESCRIPTIONS = { "structures": [ @@ -166,6 +168,11 @@ class AccessoryWindSetbackFeatures(SetbackFeatures): "transmission lines", ], "water": ["lakes", "reservoirs", "streams", "rivers", "wetlands"], + "public conservation lands": [ + "public conservation lands", + "natural resource protection areas", + "preservation areas", + ], } """Aliases for mutually-exclusive setback features""" FEATURES_AS_IGNORE = { @@ -176,6 +183,7 @@ class AccessoryWindSetbackFeatures(SetbackFeatures): "railroads": "railroads", "transmission": "transmission lines", "water": "wetlands", + "public conservation lands": "public conservation lands", } """Features as they should appear in ignore phrases""" FEATURE_CLARIFICATIONS = { @@ -185,12 +193,17 @@ class AccessoryWindSetbackFeatures(SetbackFeatures): "parcel boundaries unless the text **explicitly** makes that " "connection. " ), + "water": ( + "Public conservation lands (or similar) **are not equivalent** to " + "wetlands (or similar) unless the text **explicitly** makes that " + "connection. " + ), "roads": "Roads may also be labeled as rights-of-way. ", } """Clarifications to add to feature prompts""" -class StructuredAccessoryWindParser(BaseLLMCaller): +class StructuredSmallWindParser(BaseLLMCaller): """Base class for parsing structured data""" def _init_chat_llm_caller(self, system_message): @@ -203,7 +216,7 @@ def _init_chat_llm_caller(self, system_message): ) async def _check_wind_turbine_type(self, text): - """Get the accessory turbine size mentioned in the text""" + """Get the small turbine size mentioned in the text""" logger.info("Checking turbine types...") tree = setup_async_decision_tree( setup_graph_wes_types, @@ -212,23 +225,23 @@ async def _check_wind_turbine_type(self, text): ) decision_tree_wes_types_out = await run_async_tree(tree) - accessory_system = ( + small_wind_system = ( decision_tree_wes_types_out.get("wes_type") - or "**accessory** wind energy systems" + or "**small, medium, or non-commercial** wind energy systems" ) - if not decision_tree_wes_types_out.get("is_accessory", True): + if not decision_tree_wes_types_out.get("is_small", True): logger.info( - "Did not find accessory systems in text. Closest " + "Did not find small wind systems in text. Closest " "system found: %r", - accessory_system, + small_wind_system, ) return None - logger.info("Accessory WES type found in text: %r", accessory_system) - return accessory_system + logger.info("Small WES type found in text: %r", small_wind_system) + return small_wind_system -class StructuredAccessoryWindOrdinanceParser(StructuredAccessoryWindParser): +class StructuredSmallWindOrdinanceParser(StructuredSmallWindParser): """LLM ordinance document structured data scraping utility Purpose: @@ -258,7 +271,7 @@ async def parse(self, text): ------- pandas.DataFrame or None DataFrame containing parsed-out ordinance values. Can also - be ``None`` if an accessory wind energy system is not found + be ``None`` if a small wind energy system is not found in the text. """ wes_type = await self._check_wind_turbine_type(text) @@ -267,7 +280,7 @@ async def parse(self, text): outer_task_name = asyncio.current_task().get_name() num_to_process = ( - len(AccessoryWindSetbackFeatures.DEFAULT_FEATURE_DESCRIPTIONS) + len(SmallWindSetbackFeatures.DEFAULT_FEATURE_DESCRIPTIONS) + len(EXTRA_NUMERICAL_RESTRICTIONS) + len(EXTRA_QUALITATIVE_RESTRICTIONS) ) @@ -297,7 +310,7 @@ async def _parse_all_restrictions_with_pb( ), name=outer_task_name, ) - for feature_kwargs in AccessoryWindSetbackFeatures() + for feature_kwargs in SmallWindSetbackFeatures() ] extras_parsers = [ asyncio.create_task( @@ -366,7 +379,7 @@ async def _parse_extra_restriction( chat_llm_caller=self._init_chat_llm_caller(system_message), unit_clarification=unit_clarification, feature_clarifications=feature_clarifications, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format(tech=wes_type), ) info = await run_async_tree(tree) info.update({"feature": feature_id, "quantitative": is_numerical}) @@ -395,7 +408,9 @@ async def _parse_setback_feature( await self._extract_setback_values( text, base_messages=base_messages, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=wes_type + ), **feature_kwargs, ) ) @@ -405,7 +420,7 @@ async def _parse_setback_feature( output = await self._extract_setback_values_for_p_or_np( text, base_messages, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format(tech=wes_type), **feature_kwargs, ) sub_pb.update(task_id, advance=1, just_parsed=feature_id) @@ -419,7 +434,9 @@ async def _base_messages(self, text, **feature_kwargs): usage_sub_label=LLMUsageCategory.ORDINANCE_VALUE_EXTRACTION, text=text, chat_llm_caller=self._init_chat_llm_caller(system_message), - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=feature_kwargs["tech"] + ), **feature_kwargs, ) out = await run_async_tree(tree, response_as_json=False) @@ -479,7 +496,8 @@ async def _parse_p_or_np_text( "feature_clarifications", "" ), system_size_reminder=feature_kwargs.get( - "system_size_reminder", SYSTEM_SIZE_REMINDER + "system_size_reminder", + SYSTEM_SIZE_REMINDER.format(tech=feature_kwargs["tech"]), ), ) ) @@ -531,8 +549,8 @@ async def _run_setback_graph( return await run_async_tree(tree) -class StructuredAccessoryWindPermittedUseDistrictsParser( - StructuredAccessoryWindParser +class StructuredSmallWindPermittedUseDistrictsParser( + StructuredSmallWindParser ): """LLM permitted use districts scraping utility @@ -548,17 +566,12 @@ class StructuredAccessoryWindPermittedUseDistrictsParser( individual values. """ - _ACCESSORY_WES_CLARIFICATION = ( - "Accessory wind energy systems (AWES) may also be referred to as " - "wind turbines, wind energy conversion systems (WECS), wind energy " - "turbines (WET), small wind energy turbines (SWET), private wind " - "energy turbines (PWET), on-site wind energy systems, distributed " - "wind energy systems (DWES), medium wind energy systems (MWES), " + _SMALL_WES_CLARIFICATION = ( + "Small wind energy systems (AWES) may also be referred to as " + "non-commercial wind energy systems, on-site wind energy systems, " + "distributed wind energy systems, medium wind energy systems (MWES), " "agricultural wind energy systems (AWES), residential wind energy " - "systems (RWES), personal wind energy systems (PWES), private wind " - "turbines (PWT), micro turbines, small wind turbines (SWT), " - "accessory wind energy conversion systems (AWECS), alternate energy " - "systems (AES), or similar" + "systems, small wind turbines (SWT), or similar" ) _USE_TYPES = [ { @@ -600,8 +613,8 @@ async def parse(self, text): ------- pandas.DataFrame or None DataFrame containing parsed-out allowed-use district names. - Can also be ``None`` if an accessory wind energy system is - not found in the text. + Can also be ``None`` if a small wind energy system is not + found in the text. """ wes_type = await self._check_wind_turbine_type(text) if not wes_type: @@ -653,7 +666,7 @@ async def _parse_permitted_use_districts( feature_id=feature_id, tech=wes_type, clarifications=clarifications.format( - wes_clarification=self._ACCESSORY_WES_CLARIFICATION + wes_clarification=self._SMALL_WES_CLARIFICATION ), text=text, use_type=use_type, diff --git a/compass/extraction/solar/parse.py b/compass/extraction/solar/parse.py index 2be75e731..9972deb42 100644 --- a/compass/extraction/solar/parse.py +++ b/compass/extraction/solar/parse.py @@ -326,7 +326,9 @@ async def _parse_extra_restriction( chat_llm_caller=self._init_chat_llm_caller(system_message), unit_clarification=unit_clarification, feature_clarifications=feature_clarifications, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=largest_sef_type + ), ) info = await run_async_tree(tree) info.update({"feature": feature_id, "quantitative": is_numerical}) @@ -355,7 +357,10 @@ async def _parse_setback_feature( await self._extract_setback_values( text=text, base_messages=base_messages, - system_size_reminder=SYSTEM_SIZE_REMINDER**feature_kwargs, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=largest_sef_type + ), + **feature_kwargs, ) ) sub_pb.update(task_id, advance=1, just_parsed=feature_id) @@ -364,7 +369,9 @@ async def _parse_setback_feature( output = await self._extract_setback_values_for_p_or_np( text, base_messages, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=largest_sef_type + ), **feature_kwargs, ) sub_pb.update(task_id, advance=1, just_parsed=feature_id) @@ -378,7 +385,9 @@ async def _base_messages(self, text, **feature_kwargs): usage_sub_label=LLMUsageCategory.ORDINANCE_VALUE_EXTRACTION, text=text, chat_llm_caller=self._init_chat_llm_caller(system_message), - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=feature_kwargs["tech"] + ), **feature_kwargs, ) out = await run_async_tree(tree, response_as_json=False) @@ -438,7 +447,8 @@ async def _parse_p_or_np_text( "feature_clarifications", "" ), system_size_reminder=feature_kwargs.get( - "system_size_reminder", SYSTEM_SIZE_REMINDER + "system_size_reminder", + SYSTEM_SIZE_REMINDER.format(tech=feature_kwargs["tech"]), ), ) ) diff --git a/compass/extraction/wind/parse.py b/compass/extraction/wind/parse.py index 62e590b93..21bd84f80 100644 --- a/compass/extraction/wind/parse.py +++ b/compass/extraction/wind/parse.py @@ -316,7 +316,9 @@ async def _parse_extra_restriction( chat_llm_caller=self._init_chat_llm_caller(system_message), unit_clarification=unit_clarification, feature_clarifications=feature_clarifications, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=largest_wes_type + ), ) info = await run_async_tree(tree) info.update({"feature": feature_id, "quantitative": is_numerical}) @@ -345,7 +347,9 @@ async def _parse_setback_feature( await self._extract_setback_values( text, base_messages=base_messages, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=largest_wes_type + ), **feature_kwargs, ) ) @@ -355,7 +359,9 @@ async def _parse_setback_feature( output = await self._extract_setback_values_for_p_or_np( text, base_messages, - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=largest_wes_type + ), **feature_kwargs, ) sub_pb.update(task_id, advance=1, just_parsed=feature_id) @@ -369,7 +375,9 @@ async def _base_messages(self, text, **feature_kwargs): usage_sub_label=LLMUsageCategory.ORDINANCE_VALUE_EXTRACTION, text=text, chat_llm_caller=self._init_chat_llm_caller(system_message), - system_size_reminder=SYSTEM_SIZE_REMINDER, + system_size_reminder=SYSTEM_SIZE_REMINDER.format( + tech=feature_kwargs["tech"] + ), **feature_kwargs, ) out = await run_async_tree(tree, response_as_json=False) @@ -429,7 +437,8 @@ async def _parse_p_or_np_text( "feature_clarifications", "" ), system_size_reminder=feature_kwargs.get( - "system_size_reminder", SYSTEM_SIZE_REMINDER + "system_size_reminder", + SYSTEM_SIZE_REMINDER.format(tech=feature_kwargs["tech"]), ), ) ) diff --git a/compass/scripts/process.py b/compass/scripts/process.py index f738dfe34..c6dd6eba1 100644 --- a/compass/scripts/process.py +++ b/compass/scripts/process.py @@ -48,16 +48,16 @@ WIND_QUESTION_TEMPLATES, BEST_WIND_ORDINANCE_WEBSITE_URL_KEYWORDS, ) -from compass.extraction.accessory_wind import ( - AccessoryWindHeuristic, - AccessoryWindOrdinanceTextCollector, - AccessoryWindOrdinanceTextExtractor, - AccessoryWindPermittedUseDistrictsTextCollector, - AccessoryWindPermittedUseDistrictsTextExtractor, - StructuredAccessoryWindOrdinanceParser, - StructuredAccessoryWindPermittedUseDistrictsParser, - ACCESSORY_WIND_QUESTION_TEMPLATES, - BEST_ACCESSORY_WIND_ORDINANCE_WEBSITE_URL_KEYWORDS, +from compass.extraction.small_wind import ( + SmallWindHeuristic, + SmallWindOrdinanceTextCollector, + SmallWindOrdinanceTextExtractor, + SmallWindPermittedUseDistrictsTextCollector, + SmallWindPermittedUseDistrictsTextExtractor, + StructuredSmallWindOrdinanceParser, + StructuredSmallWindPermittedUseDistrictsParser, + SMALL_WIND_QUESTION_TEMPLATES, + BEST_SMALL_WIND_ORDINANCE_WEBSITE_URL_KEYWORDS, ) from compass.validation.location import JurisdictionWebsiteValidator from compass.llm import LLMCaller, OpenAIConfig @@ -134,6 +134,10 @@ SolarPermittedUseDistrictsTextExtractor: ( "Extracting solar permitted use text" ), + SmallWindOrdinanceTextExtractor: ("Extracting small wind ordinance text"), + SmallWindPermittedUseDistrictsTextExtractor: ( + "Extracting small wind permitted use text" + ), } _JUR_COLS = [ "Jurisdiction Type", @@ -201,7 +205,7 @@ async def process_jurisdictions_with_openai( # noqa: PLR0917, PLR0913 CSV file, all downloaded ordinance documents (PDFs and HTML), usage metadata, and default subdirectories for logs and intermediate outputs (unless otherwise specified). - tech : {"wind", "solar", "accessory wind"} + tech : {"wind", "solar", "small wind"} Label indicating which technology type is being processed. jurisdiction_fp : path-like Path to a CSV file specifying the jurisdictions to process. @@ -1337,18 +1341,18 @@ def _compile_tech_specs(tech): StructuredSolarPermittedUseDistrictsParser, BEST_SOLAR_ORDINANCE_WEBSITE_URL_KEYWORDS, ) - if tech.casefold() == "accessory wind": + if tech.casefold() == "small wind": return TechSpec( - "accessory wind", - ACCESSORY_WIND_QUESTION_TEMPLATES, - AccessoryWindHeuristic(), - AccessoryWindOrdinanceTextCollector, - AccessoryWindOrdinanceTextExtractor, - AccessoryWindPermittedUseDistrictsTextCollector, - AccessoryWindPermittedUseDistrictsTextExtractor, - StructuredAccessoryWindOrdinanceParser, - StructuredAccessoryWindPermittedUseDistrictsParser, - BEST_ACCESSORY_WIND_ORDINANCE_WEBSITE_URL_KEYWORDS, + "small wind", + SMALL_WIND_QUESTION_TEMPLATES, + SmallWindHeuristic(), + SmallWindOrdinanceTextCollector, + SmallWindOrdinanceTextExtractor, + SmallWindPermittedUseDistrictsTextCollector, + SmallWindPermittedUseDistrictsTextExtractor, + StructuredSmallWindOrdinanceParser, + StructuredSmallWindPermittedUseDistrictsParser, + BEST_SMALL_WIND_ORDINANCE_WEBSITE_URL_KEYWORDS, ) msg = f"Unknown tech input: {tech}" diff --git a/compass/utilities/finalize.py b/compass/utilities/finalize.py index 3729d21d0..58f6eaac3 100644 --- a/compass/utilities/finalize.py +++ b/compass/utilities/finalize.py @@ -59,7 +59,7 @@ def save_run_meta( dirs : :class:`~compass.utilities.base.Directories` Directories instance containing information about the output directories used for the run. - tech : {"wind", "solar", "accessory wind"} + tech : {"wind", "solar", "small wind"} Technology that was the target of the run. start_date, end_date : datetime.datetime Instances representing the start and end dates, respectively. diff --git a/compass/utilities/io.py b/compass/utilities/io.py index daa367b7f..c099e7956 100644 --- a/compass/utilities/io.py +++ b/compass/utilities/io.py @@ -18,14 +18,15 @@ async def load_local_docs(fps, **kwargs): Iterable of paths representing documents to load. kwargs Keyword-argument pairs to initialize - :class:`elm.web.file_loader.AsyncFileLoader`. + :class:`elm.web.file_loader.AsyncLocalFileLoader`. Returns ------- list List of non-empty document instances containing information from - the URL's. If a URL could not be fetched (i.e. document instance - is empty), it will not be included in the output list. + the local documents. If a file could not be loaded (i.e. + document instance is empty), it will not be included in the + output list. """ logger.trace("Loading docs for the following paths:\n%r", fps) logger.trace( diff --git a/docs/source/conf.py b/docs/source/conf.py index 09b52e872..2f962fe22 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -61,7 +61,7 @@ intersphinx_mapping = { "elm": ("https://nrel.github.io/elm", None), - "lc": ("https://python.langchain.com/api_reference/", None), + "lc": ("https://reference.langchain.com/python/", None), "matplotlib": ("https://matplotlib.org/stable", None), "networkx": ("https://networkx.org/documentation/stable", None), "numpy": ("https://numpy.org/doc/stable/", None),