From 4063456c2ac45e053ca668f1f263b760f47cafb1 Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Fri, 29 Jul 2022 18:17:46 -0700 Subject: [PATCH 01/10] add missing init files and set urban index to county name --- prereise/utility/__init__.py | 0 prereise/utility/generate_rural_shapefiles.py | 1 + prereise/utility/tests/__init__.py | 0 3 files changed, 1 insertion(+) create mode 100644 prereise/utility/__init__.py create mode 100644 prereise/utility/tests/__init__.py diff --git a/prereise/utility/__init__.py b/prereise/utility/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/prereise/utility/generate_rural_shapefiles.py b/prereise/utility/generate_rural_shapefiles.py index f490e5b2f..3b98fafd7 100644 --- a/prereise/utility/generate_rural_shapefiles.py +++ b/prereise/utility/generate_rural_shapefiles.py @@ -23,6 +23,7 @@ def append_rural_areas_to_urban(states, urban_areas): states = states.rename(columns={"STUSPS": "state"}) states = states.loc[states["state"].isin(lower_48_states_abv)] + urban_areas.index = urban_areas["NAME10"] urban_areas["state"] = urban_areas["NAME10"].str[-2:] # this drops 66 out of 3601 rows urban_areas = urban_areas.loc[urban_areas["state"].isin(lower_48_states_abv)] diff --git a/prereise/utility/tests/__init__.py b/prereise/utility/tests/__init__.py new file mode 100644 index 000000000..e69de29bb From 372ae06d4dc2e3d9f5a8f9b929de6255bb382d17 Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Thu, 20 Oct 2022 12:00:43 -0700 Subject: [PATCH 02/10] Add area name to urban-rural index --- prereise/utility/generate_rural_shapefiles.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/prereise/utility/generate_rural_shapefiles.py b/prereise/utility/generate_rural_shapefiles.py index 3b98fafd7..4790d285f 100644 --- a/prereise/utility/generate_rural_shapefiles.py +++ b/prereise/utility/generate_rural_shapefiles.py @@ -1,6 +1,7 @@ import os import geopandas as gpd +import pandas as pd from prereise.gather.const import abv2state from prereise.utility.shapefile import download_shapefiles @@ -23,16 +24,19 @@ def append_rural_areas_to_urban(states, urban_areas): states = states.rename(columns={"STUSPS": "state"}) states = states.loc[states["state"].isin(lower_48_states_abv)] - urban_areas.index = urban_areas["NAME10"] urban_areas["state"] = urban_areas["NAME10"].str[-2:] # this drops 66 out of 3601 rows urban_areas = urban_areas.loc[urban_areas["state"].isin(lower_48_states_abv)] + + urban_areas = urban_areas.rename(columns={"NAME10": "name"}) + urban_areas = urban_areas.set_index("name", verify_integrity=True) urban_areas["urban"] = True rural_areas = states.overlay(urban_areas, how="difference") + rural_areas.index = rural_areas["state"] + "_rural" rural_areas["urban"] = False - return urban_areas.append(rural_areas, ignore_index=True)[ + return pd.concat([urban_areas, rural_areas], verify_integrity=True)[ ["state", "geometry", "urban"] ] From 8307711dd6c024980ceaf03924054f5a3ef2d2ec Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Sun, 23 Oct 2022 16:40:33 -0700 Subject: [PATCH 03/10] feat: add script to translate urban and rural shapefiles to USA tamu loadzones --- prereise/gather/const.py | 79 ++++++++++ .../translate_urban_rural_to_usa_tamu.py | 147 ++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 prereise/utility/translate_urban_rural_to_usa_tamu.py diff --git a/prereise/gather/const.py b/prereise/gather/const.py index 1ce78e33e..ad75380d9 100644 --- a/prereise/gather/const.py +++ b/prereise/gather/const.py @@ -51,3 +51,82 @@ "WV": "West Virginia", "WY": "Wyoming", } + +zone2state_abv = { + "Alabama": "AL", + "Arizona": "AZ", + "Arkansas": "AR", + "Bay Area": "CA", + "Central California": "CA", + "Chicago North Illinois": "IL", + "Coast": "TX", + "Colorado": "CO", + "Connecticut": "CT", + "Delaware": "DE", + "East": "TX", + "East Texas": "TX", + "El Paso": "TX", + "Far West": "TX", + "Florida North": "FL", + "Florida Panhandle": "FL", + "Florida South": "FL", + "Georgia North": "GA", + "Georgia South": "GA", + "Idaho": "ID", + "Illinois Downstate": "IL", + "Indiana": "IN", + "Iowa": "IA", + "Kansas": "KS", + "Kentucky": "KY", + "Louisiana": "LA", + "Maine": "ME", + "Maryland": "MD", + "Massachusetts": "MA", + "Michigan Northern": "MI", + "Michigan Southern": "MI", + "Minnesota Northern": "MN", + "Minnesota Southern": "MN", + "Mississippi": "MS", + "Missouri East": "MO", + "Missouri West": "MO", + "Montana Eastern": "MT", + "Montana Western": "MT", + "Nebraska": "NE", + "Nevada": "NV", + "New Hampshire": "NH", + "New Jersey": "NJ", + "New Mexico Eastern": "NM", + "New Mexico Western": "NM", + "New York City": "NY", + "North": "TX", + "North Carolina": "NC", + "North Central": "TX", + "North Dakota": "ND", + "Northern California": "CA", + "Ohio Lake Erie": "OH", + "Ohio River": "OH", + "Oklahoma": "OK", + "Oregon": "OR", + "Pennsylvania Eastern": "PA", + "Pennsylvania Western": "PA", + "Rhode Island": "RI", + "South": "TX", + "South Carolina": "SC", + "South Central": "TX", + "South Dakota": "SD", + "Southeast California": "CA", + "Southwest California": "CA", + "Tennessee": "TN", + "Texas Panhandle": "TX", + "Upstate New York": "NY", + "Utah": "UT", + "Vermont": "VT", + "Virginia Mountains": "VA", + "Virginia Tidewater": "VA", + "Washington": "WA", + "West": "TX", + "West Virginia": "WV", + "Western North Carolina": "NC", + "Wisconsin": "WI", + "Wyoming": "WY", +} diff --git a/prereise/utility/translate_urban_rural_to_usa_tamu.py b/prereise/utility/translate_urban_rural_to_usa_tamu.py new file mode 100644 index 000000000..b82402e10 --- /dev/null +++ b/prereise/utility/translate_urban_rural_to_usa_tamu.py @@ -0,0 +1,147 @@ +import os +import numpy as np +import pandas as pd +import geopandas as gpd + +from powersimdata.input.grid import Grid +from powersimdata.scenario.scenario import Scenario +from prereise.utility.translate_zones import ( + filter_interesting_zones, + translate_zone_set, +) +from prereise.utility.generate_rural_shapefiles import ( + generate_urban_and_rural_shapefiles, +) +from prereise.gather.const import zone2state_abv +from prereise.utility.shapefile import download_shapefiles + + +def plot_loadzones(tamu_loadzones, states): + ax = tamu_loadzones.plot(figsize=(50, 50), cmap="tab20", alpha=0.5) + sub.plot( + ax=ax, + column="zone", + linewidth=1, + cmap="tab20", + # alpha= 0.5, + ) + tamu_loadzones.plot(ax=ax, linewidth=1, edgecolor="red", color="none", alpha=0.5) + states.plot( + ax=ax, + linewidth=1, + edgecolor="black", + color="none", + ) + + +def format_states_gdf(states): + # TODO clean + states = states.loc[~states["STUSPS"].isin(["AK", "DC", "HI", "PR"])] + states = states.rename(columns={"STUSPS": "state"}) + states.index = states["state"] + return states + + +def create_substation_gdf(grid): + sub2zone = grid.bus2sub.copy().join(grid.bus["zone_id"])[["sub_id", "zone_id"]] + sub2zone = sub2zone.groupby(["sub_id", "zone_id"]).sum() + sub2zone = sub2zone.reset_index(level="zone_id") + + sub = grid.sub.join(sub2zone) + sub["zone"] = sub["zone_id"].replace(grid.id2zone) + + sub = gpd.GeoDataFrame(sub, geometry=gpd.points_from_xy(sub.lon, sub.lat)) + sub = sub[["zone", "geometry"]] + + return sub + + +def fix_lz_border(states, state, geometry): + state_geom = states.loc[state, "geometry"] + + states_with_multiple_loadzones = [ + "MO", + "MN", + "MI", + "IL", + "OH", + "PA", + "NY", + "FL", + "GA", + "NC", + "VA", # Eastern + "CA", + "MT", + "NM", # Western + "TX", # ERCOT + ] + if state in states_with_multiple_loadzones: + # Make sure loadzones don't go outside state borders + return geometry.intersection(state_geom) + else: + # Replace single loadzone states with state border shape + return state_geom + + +def create_usa_tamu_convex_hull_shapefile(sub_gdf, states): + # convex hull: the smallest convex Polygon containing all the points in each geometry + tamu_loadzones = sub_gdf.dissolve("zone").convex_hull + tamu_loadzones = gpd.GeoDataFrame(tamu_loadzones) + tamu_loadzones = tamu_loadzones.rename(columns={0: "geometry"}) + + tamu_loadzones["state"] = tamu_loadzones.index + tamu_loadzones["state"] = tamu_loadzones["state"].replace(zone2state_abv) + + # epsg:4269 matches urban_rural and is most commonly used by U.S. federal agencies. + # https://www.nceas.ucsb.edu/sites/default/files/2020-04/OverviewCoordinateReferenceSystems.pdf + tamu_loadzones = tamu_loadzones.set_crs("epsg:4269") + + tamu_loadzones["geometry"] = tamu_loadzones.apply( + lambda x: fix_lz_border(states, x.state, x.geometry), axis=1 + ) + + return tamu_loadzones + + +def translate_urban_rural_to_usa_tamu(): + """Downloads shapefiles of state outlines and urban areas from BES azure blob storage. + Writes these shapefiles to a local folder, then creates new shapefiles that + include both urban and rural areas. + + :return: (*str*) -- path to the new shapefiles + """ + base_url = "https://besciences.blob.core.windows.net/shapefiles/USA/" + + # download states shapefiles + states_folder = os.path.join(os.path.dirname(__file__), "state-outlines") + states_file = download_shapefiles( + f"{base_url}state-outlines/", + "cb_2018_us_state_20m", + states_folder, + ) + states = gpd.read_file(states_file) + states = format_states_gdf(states) + + # download urban-and-rural shapefiles + urban_and_rural_folder = os.path.join( + os.path.dirname(__file__), "urban-and-rural-areas" + ) + urban_and_rural_file = download_shapefiles( + f"{base_url}urban-and-rural-areas/", + "urban-and-rural-areas", + urban_and_rural_folder, + ) + urban_and_rural = gpd.read_file(urban_and_rural_file) + + # Create USA tamu shapefiles + grid = Grid("USA") + sub = create_substation_gdf(grid) + tamu_loadzones = create_usa_tamu_convex_hull_shapefile(sub, states) + + return translate_zone_set( + urban_and_rural, + tamu_loadzones, + name_prev="urban_and_rural", + name_new="usa_tamu", + ) From 38fe4adfb0c303255fb5a6c9e0d26918e168b56c Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Sun, 23 Oct 2022 17:00:18 -0700 Subject: [PATCH 04/10] cleanup, start docs --- prereise/gather/const.py | 4 ++++ prereise/utility/generate_rural_shapefiles.py | 6 +----- .../translate_urban_rural_to_usa_tamu.py | 21 +++++++++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/prereise/gather/const.py b/prereise/gather/const.py index ad75380d9..33423c85a 100644 --- a/prereise/gather/const.py +++ b/prereise/gather/const.py @@ -52,6 +52,10 @@ "WY": "Wyoming", } +lower_48_states_abv = list(abv2state.keys()) +lower_48_states_abv.remove("AK") +lower_48_states_abv.remove("HI") + zone2state_abv = { "Alabama": "AL", "Arizona": "AZ", diff --git a/prereise/utility/generate_rural_shapefiles.py b/prereise/utility/generate_rural_shapefiles.py index 4790d285f..a0c6cccd0 100644 --- a/prereise/utility/generate_rural_shapefiles.py +++ b/prereise/utility/generate_rural_shapefiles.py @@ -3,7 +3,7 @@ import geopandas as gpd import pandas as pd -from prereise.gather.const import abv2state +from prereise.gather.const import lower_48_states_abv from prereise.utility.shapefile import download_shapefiles @@ -17,10 +17,6 @@ def append_rural_areas_to_urban(states, urban_areas): :return: (*geopandas.geodataframe.GeoDataFrame*) -- new gdf of urban and rural areas includes a column indicating whether the area is urban or not """ - lower_48_states_abv = list(abv2state.keys()) - lower_48_states_abv.remove("AK") - lower_48_states_abv.remove("HI") - states = states.rename(columns={"STUSPS": "state"}) states = states.loc[states["state"].isin(lower_48_states_abv)] diff --git a/prereise/utility/translate_urban_rural_to_usa_tamu.py b/prereise/utility/translate_urban_rural_to_usa_tamu.py index b82402e10..a6771aa4a 100644 --- a/prereise/utility/translate_urban_rural_to_usa_tamu.py +++ b/prereise/utility/translate_urban_rural_to_usa_tamu.py @@ -12,18 +12,28 @@ from prereise.utility.generate_rural_shapefiles import ( generate_urban_and_rural_shapefiles, ) -from prereise.gather.const import zone2state_abv +from prereise.gather.const import zone2state_abv, lower_48_states_abv from prereise.utility.shapefile import download_shapefiles -def plot_loadzones(tamu_loadzones, states): +def plot_loadzones(tamu_loadzones, substations, states): + """Plot usa tamu loadzone borders as choropleth. Has 20 colors; buckets zones in + alphabetical order + + :param geopandas.geodataframe.GeoDataFrame tamu_loadzones: GeoDataFrame with + index = zone, columns = ['geometry'] + :param geopandas.geodataframe.GeoDataFrame substations: GeoDataFrame with + index = substation id, columns = ['geometry', 'zone'] + :param geopandas.geodataframe.GeoDataFrame states: GeoDataFrame with + index = state name, columns = ['geometry'] + :return: (*matplotlib.axes._subplots.AxesSubplot) -- the plot object + """ ax = tamu_loadzones.plot(figsize=(50, 50), cmap="tab20", alpha=0.5) - sub.plot( + substations.plot( ax=ax, column="zone", linewidth=1, cmap="tab20", - # alpha= 0.5, ) tamu_loadzones.plot(ax=ax, linewidth=1, edgecolor="red", color="none", alpha=0.5) states.plot( @@ -35,9 +45,8 @@ def plot_loadzones(tamu_loadzones, states): def format_states_gdf(states): - # TODO clean - states = states.loc[~states["STUSPS"].isin(["AK", "DC", "HI", "PR"])] states = states.rename(columns={"STUSPS": "state"}) + states = states.loc[states["state"].isin(lower_48_states_abv)] states.index = states["state"] return states From abb10db59e031dfe14c26fc3a03ab225400a1b04 Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Mon, 24 Oct 2022 19:15:35 -0700 Subject: [PATCH 05/10] docs: add docstrings --- .../translate_urban_rural_to_usa_tamu.py | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/prereise/utility/translate_urban_rural_to_usa_tamu.py b/prereise/utility/translate_urban_rural_to_usa_tamu.py index a6771aa4a..415a5a035 100644 --- a/prereise/utility/translate_urban_rural_to_usa_tamu.py +++ b/prereise/utility/translate_urban_rural_to_usa_tamu.py @@ -17,16 +17,16 @@ def plot_loadzones(tamu_loadzones, substations, states): - """Plot usa tamu loadzone borders as choropleth. Has 20 colors; buckets zones in - alphabetical order + """Plot usa tamu loadzone borders and substations as choropleth. + Has 20 colors; buckets zones in alphabetical order. :param geopandas.geodataframe.GeoDataFrame tamu_loadzones: GeoDataFrame with index = zone, columns = ['geometry'] :param geopandas.geodataframe.GeoDataFrame substations: GeoDataFrame with index = substation id, columns = ['geometry', 'zone'] - :param geopandas.geodataframe.GeoDataFrame states: GeoDataFrame with - index = state name, columns = ['geometry'] - :return: (*matplotlib.axes._subplots.AxesSubplot) -- the plot object + :param geopandas.geodataframe.GeoDataFrame states: GeoDataFrame of lower 48 + US state borders with index = state, columns = ['geometry'] + :return: (*matplotlib.axes._subplots.AxesSubplot*) -- the plot object """ ax = tamu_loadzones.plot(figsize=(50, 50), cmap="tab20", alpha=0.5) substations.plot( @@ -42,9 +42,17 @@ def plot_loadzones(tamu_loadzones, substations, states): edgecolor="black", color="none", ) + return ax def format_states_gdf(states): + """Takes geodataframe of US state borders, filters to the lower 48 states, + and renames columns. + + :param geopandas.geodataframe.GeoDataFrame states: GeoDataFrame of US state + borders with index = state, columns = ['geometry', 'STUSPS'] + :return: (*geopandas.geodataframe.GeoDataFrame*) -- modified states gdf + """ states = states.rename(columns={"STUSPS": "state"}) states = states.loc[states["state"].isin(lower_48_states_abv)] states.index = states["state"] @@ -52,6 +60,12 @@ def format_states_gdf(states): def create_substation_gdf(grid): + """Takes a grid object and uses lat lon data to create a Geodataframe of + substations. + + :param powersimdata.input.grid.Grid grid: Grid instance. + :return: (*geopandas.geodataframe.GeoDataFrame*) -- gdf of substations + """ sub2zone = grid.bus2sub.copy().join(grid.bus["zone_id"])[["sub_id", "zone_id"]] sub2zone = sub2zone.groupby(["sub_id", "zone_id"]).sum() sub2zone = sub2zone.reset_index(level="zone_id") @@ -66,6 +80,18 @@ def create_substation_gdf(grid): def fix_lz_border(states, state, geometry): + """Takes a shape representing a loadzone and trims it to match state + borders. If there is only one loadzone in that state, we use the + original state border. If there are multiple loadzones in one state, we + trim the loadzone borders so they do not go outside the state border. + + :param geopandas.geodataframe.GeoDataFrame states: GeoDataFrame of lower 48 + US state borders with index = state, columns = ['geometry', 'state'] + :param str state: Abbreviated name of the state. + :param TODO geometry: The shape of the loadzone to be trimmed + :return: (*TODO*) -- the modified loadzone shape + """ + state_geom = states.loc[state, "geometry"] states_with_multiple_loadzones = [ @@ -94,7 +120,18 @@ def fix_lz_border(states, state, geometry): def create_usa_tamu_convex_hull_shapefile(sub_gdf, states): - # convex hull: the smallest convex Polygon containing all the points in each geometry + """Creates a Geodataframe that approximates the shapes of the USA tamu + loadzones. Loadzone shapes are made by taking the convex hull of all the + individual substations within that zone. A convex hull is the smallest + convex polygon containing all the points in each geometry. + + :param geopandas.geodataframe.GeoDataFrame sub_gdf: GeoDataFrame of + substations with index = sub id, columns = ['geometry', 'zone'] + :param geopandas.geodataframe.GeoDataFrame states: GeoDataFrame of lower 48 + US state borders with index = state, columns = ['geometry', 'state'] + :return: (*geopandas.geodataframe.GeoDataFrame*) -- gdf of USA tamu zones + """ + tamu_loadzones = sub_gdf.dissolve("zone").convex_hull tamu_loadzones = gpd.GeoDataFrame(tamu_loadzones) tamu_loadzones = tamu_loadzones.rename(columns={0: "geometry"}) From cc066c2ed80270500718517ec2c5d9794aee9850 Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Mon, 14 Nov 2022 21:08:13 -0800 Subject: [PATCH 06/10] test: add tests for urban-rural translation --- .../tests/test_urban_rural_to_usa_tamu.py | 111 ++++++++++++++++++ .../translate_urban_rural_to_usa_tamu.py | 43 +++---- 2 files changed, 125 insertions(+), 29 deletions(-) create mode 100644 prereise/utility/tests/test_urban_rural_to_usa_tamu.py diff --git a/prereise/utility/tests/test_urban_rural_to_usa_tamu.py b/prereise/utility/tests/test_urban_rural_to_usa_tamu.py new file mode 100644 index 000000000..525c2df13 --- /dev/null +++ b/prereise/utility/tests/test_urban_rural_to_usa_tamu.py @@ -0,0 +1,111 @@ +import geopandas as gpd +import pandas as pd +from shapely.geometry import Point, Polygon +from prereise.utility.translate_urban_rural_to_usa_tamu import ( + format_states_gdf, + create_substation_gdf, + fix_lz_border, + create_usa_tamu_convex_hull_shapefile, + translate_urban_rural_to_usa_tamu, + states_with_multiple_loadzones, +) +from powersimdata.input.grid import Grid + + +WA_lz_polygon = Polygon([(1, 1), (0, 5), (5, 6), (4, 1)]) +CA_lz_polygon = Polygon([(0, 10), (0, 15), (5, 15), (5, 10)]) +WA_border_polygon = Polygon([(0, 0), (0, 5), (5, 5), (5, 0)]) +CA_border_polygon = Polygon([(0, 10), (0, 20), (3, 20), (3, 10)]) +HI_border_polygon = Polygon([(100, 100), (100, 105), (105, 105)]) + +# Mock US State borders +states = gpd.GeoDataFrame( + { + "geometry": [WA_border_polygon, CA_border_polygon, HI_border_polygon], + "STUSPS": ["WA", "CA", "HI"], + } +) +formatted_states = gpd.GeoDataFrame( + { + "geometry": {"WA": WA_border_polygon, "CA": CA_border_polygon}, + "state": {"WA": "WA", "CA": "CA"}, + } +) +sub_gdf = gpd.GeoDataFrame( + { + "zone": {122: "Bay Area", 6134: "Washington"}, + "geometry": {122: Point(2, 3), 6134: Point(14, 4)}, + } +) + + +def test_format_states_gdf(): + results = format_states_gdf(states) + assert formatted_states.to_dict() == results.to_dict() + + +class MockGrid: + def __init__(self): + """Constructor. + :param dict grid_attrs: dictionary of {*field_name*, *data*} pairs + where *field_name* is the name of the field and *data* a dictionary + or DataFrame. + """ + self.bus2sub = pd.DataFrame({"sub_id": {0: 122, 1: 122, 2: 6134}}) + self.bus = pd.DataFrame({"zone_id": {0: 204, 1: 204, 2: 201}}) + self.id2zone = {204: "Bay Area", 201: "Washington"} + self.sub = pd.DataFrame({"lon": {122: 2, 6134: 14}, "lat": {122: 3, 6134: 4}}) + + @property + def __class__(self): + """If anyone asks, I'm a Grid object!""" + return Grid + + +def test_create_substation_gdf(): + mock_grid = MockGrid() + results = create_substation_gdf(mock_grid) + assert sub_gdf.to_dict() == results.to_dict() + + +def test_fix_lz_border_single_lz(): + results = fix_lz_border(formatted_states, "WA", WA_lz_polygon) + expected = WA_border_polygon + assert expected == results + + +def test_fix_lz_border_multi_lz(): + results = fix_lz_border(formatted_states, "CA", CA_lz_polygon) + expected = Polygon([(0, 15), (3, 15), (3, 10), (0, 10)]) + assert expected == results + + +def test_create_usa_tamu_convex_hull_shapefile(): + # gdf where CA substations match CA_lz_polygon + sub_gdf_2 = gpd.GeoDataFrame( + { + "zone": { + 122: "Bay Area", + 123: "Bay Area", + 124: "Bay Area", + 125: "Bay Area", + 6134: "Washington", + }, + "geometry": { + 122: Point(0, 10), + 123: Point(0, 15), + 124: Point(5, 15), + 125: Point(5, 10), + 6134: Point(14, 4), + }, + } + ) + results = create_usa_tamu_convex_hull_shapefile(sub_gdf_2, formatted_states) + expected = { + "geometry": { + "Bay Area": Polygon([(0, 15), (3, 15), (3, 10), (0, 10)]), + "Washington": WA_border_polygon, + }, + "state": {"Bay Area": "CA", "Washington": "WA"}, + } + assert expected == results.to_dict() diff --git a/prereise/utility/translate_urban_rural_to_usa_tamu.py b/prereise/utility/translate_urban_rural_to_usa_tamu.py index 415a5a035..2a2ed31ad 100644 --- a/prereise/utility/translate_urban_rural_to_usa_tamu.py +++ b/prereise/utility/translate_urban_rural_to_usa_tamu.py @@ -1,20 +1,22 @@ import os -import numpy as np -import pandas as pd import geopandas as gpd from powersimdata.input.grid import Grid -from powersimdata.scenario.scenario import Scenario from prereise.utility.translate_zones import ( - filter_interesting_zones, translate_zone_set, ) -from prereise.utility.generate_rural_shapefiles import ( - generate_urban_and_rural_shapefiles, -) from prereise.gather.const import zone2state_abv, lower_48_states_abv from prereise.utility.shapefile import download_shapefiles +seen = set() +states_with_multiple_loadzones = set( + [ + state + for state in list(zone2state_abv.values()) + if state in seen or seen.add(state) + ] +) + def plot_loadzones(tamu_loadzones, substations, states): """Plot usa tamu loadzone borders and substations as choropleth. @@ -94,23 +96,6 @@ def fix_lz_border(states, state, geometry): state_geom = states.loc[state, "geometry"] - states_with_multiple_loadzones = [ - "MO", - "MN", - "MI", - "IL", - "OH", - "PA", - "NY", - "FL", - "GA", - "NC", - "VA", # Eastern - "CA", - "MT", - "NM", # Western - "TX", # ERCOT - ] if state in states_with_multiple_loadzones: # Make sure loadzones don't go outside state borders return geometry.intersection(state_geom) @@ -144,18 +129,18 @@ def create_usa_tamu_convex_hull_shapefile(sub_gdf, states): tamu_loadzones = tamu_loadzones.set_crs("epsg:4269") tamu_loadzones["geometry"] = tamu_loadzones.apply( - lambda x: fix_lz_border(states, x.state, x.geometry), axis=1 + lambda x: fix_lz_border(states, x.state, x.geometry), + axis=1, ) return tamu_loadzones def translate_urban_rural_to_usa_tamu(): - """Downloads shapefiles of state outlines and urban areas from BES azure blob storage. - Writes these shapefiles to a local folder, then creates new shapefiles that - include both urban and rural areas. + """Downloads US state borders as well as census data for urban and rural + areas in the US. Creates a matrix that can be used to transform - :return: (*str*) -- path to the new shapefiles + :return: (*pandas.DataFrame*) -- matrix of urban and rural zones to USA tamu zones """ base_url = "https://besciences.blob.core.windows.net/shapefiles/USA/" From e8ff704fc66034351e9c48bd533140e294ea81cd Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Mon, 14 Nov 2022 21:09:25 -0800 Subject: [PATCH 07/10] isort --- prereise/utility/tests/test_urban_rural_to_usa_tamu.py | 10 +++++----- prereise/utility/translate_urban_rural_to_usa_tamu.py | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/prereise/utility/tests/test_urban_rural_to_usa_tamu.py b/prereise/utility/tests/test_urban_rural_to_usa_tamu.py index 525c2df13..7d7c68c75 100644 --- a/prereise/utility/tests/test_urban_rural_to_usa_tamu.py +++ b/prereise/utility/tests/test_urban_rural_to_usa_tamu.py @@ -1,16 +1,16 @@ import geopandas as gpd import pandas as pd +from powersimdata.input.grid import Grid from shapely.geometry import Point, Polygon + from prereise.utility.translate_urban_rural_to_usa_tamu import ( - format_states_gdf, create_substation_gdf, - fix_lz_border, create_usa_tamu_convex_hull_shapefile, - translate_urban_rural_to_usa_tamu, + fix_lz_border, + format_states_gdf, states_with_multiple_loadzones, + translate_urban_rural_to_usa_tamu, ) -from powersimdata.input.grid import Grid - WA_lz_polygon = Polygon([(1, 1), (0, 5), (5, 6), (4, 1)]) CA_lz_polygon = Polygon([(0, 10), (0, 15), (5, 15), (5, 10)]) diff --git a/prereise/utility/translate_urban_rural_to_usa_tamu.py b/prereise/utility/translate_urban_rural_to_usa_tamu.py index 2a2ed31ad..52a7423c7 100644 --- a/prereise/utility/translate_urban_rural_to_usa_tamu.py +++ b/prereise/utility/translate_urban_rural_to_usa_tamu.py @@ -1,12 +1,11 @@ import os -import geopandas as gpd +import geopandas as gpd from powersimdata.input.grid import Grid -from prereise.utility.translate_zones import ( - translate_zone_set, -) -from prereise.gather.const import zone2state_abv, lower_48_states_abv + +from prereise.gather.const import lower_48_states_abv, zone2state_abv from prereise.utility.shapefile import download_shapefiles +from prereise.utility.translate_zones import translate_zone_set seen = set() states_with_multiple_loadzones = set( From 85e80a4854b23847e5ded88f0e773a5ae32d8eb9 Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Mon, 14 Nov 2022 21:16:41 -0800 Subject: [PATCH 08/10] flake8 --- prereise/utility/tests/test_urban_rural_to_usa_tamu.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/prereise/utility/tests/test_urban_rural_to_usa_tamu.py b/prereise/utility/tests/test_urban_rural_to_usa_tamu.py index 7d7c68c75..735215b97 100644 --- a/prereise/utility/tests/test_urban_rural_to_usa_tamu.py +++ b/prereise/utility/tests/test_urban_rural_to_usa_tamu.py @@ -8,8 +8,6 @@ create_usa_tamu_convex_hull_shapefile, fix_lz_border, format_states_gdf, - states_with_multiple_loadzones, - translate_urban_rural_to_usa_tamu, ) WA_lz_polygon = Polygon([(1, 1), (0, 5), (5, 6), (4, 1)]) @@ -46,11 +44,7 @@ def test_format_states_gdf(): class MockGrid: def __init__(self): - """Constructor. - :param dict grid_attrs: dictionary of {*field_name*, *data*} pairs - where *field_name* is the name of the field and *data* a dictionary - or DataFrame. - """ + """Constructor""" self.bus2sub = pd.DataFrame({"sub_id": {0: 122, 1: 122, 2: 6134}}) self.bus = pd.DataFrame({"zone_id": {0: 204, 1: 204, 2: 201}}) self.id2zone = {204: "Bay Area", 201: "Washington"} From d25cd53d43cc02ae851808a78d32474318737db9 Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Mon, 14 Nov 2022 21:39:46 -0800 Subject: [PATCH 09/10] fix old urban-rural test --- .../tests/test_generate_rural_shapefiles.py | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/prereise/utility/tests/test_generate_rural_shapefiles.py b/prereise/utility/tests/test_generate_rural_shapefiles.py index 0058c1d27..e6bdfdfb4 100644 --- a/prereise/utility/tests/test_generate_rural_shapefiles.py +++ b/prereise/utility/tests/test_generate_rural_shapefiles.py @@ -28,44 +28,24 @@ def test_append_rural_areas_to_urban(): r_and_u = append_rural_areas_to_urban(mock_states, mock_urban) - # Convert to json then back to dict -- this unpacks the Polygon objects - assert json.loads(r_and_u.to_json()) == { - "type": "FeatureCollection", - "features": [ - { - "id": "0", - "type": "Feature", - "properties": {"state": "CO", "urban": True}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [[0.1, 0.1], [0.1, 0.5], [0.5, 0.5], [0.5, 0.1], [0.1, 0.1]] - ], - }, - }, - { - "id": "1", - "type": "Feature", - "properties": {"state": "CO", "urban": True}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [[4.0, 4.0], [4.0, 4.5], [4.5, 4.5], [4.5, 4.0], [4.0, 4.0]] - ], - }, - }, - { - "id": "2", - "type": "Feature", - "properties": {"state": "CO", "urban": False}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [[5.0, 5.0], [5.0, 0.0], [0.0, 0.0], [0.0, 5.0], [5.0, 5.0]], - [[4.5, 4.0], [4.5, 4.5], [4.0, 4.5], [4.0, 4.0], [4.5, 4.0]], - [[0.5, 0.5], [0.1, 0.5], [0.1, 0.1], [0.5, 0.1], [0.5, 0.5]], - ], - }, - }, - ], + expected = { + "geometry": { + "Boulder_CO": Polygon( + [[4.0, 4.0], [4.0, 4.5], [4.5, 4.5], [4.5, 4.0], [4.0, 4.0]] + ), + "CO_rural": Polygon( + [[5.0, 5.0], [5.0, 0.0], [0.0, 0.0], [0.0, 5.0], [5.0, 5.0]], + [ + [[4.5, 4.0], [4.5, 4.5], [4.0, 4.5], [4.0, 4.0], [4.5, 4.0]], + [[0.5, 0.5], [0.1, 0.5], [0.1, 0.1], [0.5, 0.1], [0.5, 0.5]], + ], + ), + "Denver_CO": Polygon( + [[0.1, 0.1], [0.1, 0.5], [0.5, 0.5], [0.5, 0.1], [0.1, 0.1]] + ), + }, + "state": {"Boulder_CO": "CO", "CO_rural": "CO", "Denver_CO": "CO"}, + "urban": {"Boulder_CO": True, "CO_rural": False, "Denver_CO": True}, } + + assert r_and_u.to_dict() == expected From 7723e22bc9cbff2345356b50d26dcc499a7f5192 Mon Sep 17 00:00:00 2001 From: Merrielle Ondreicka Date: Mon, 14 Nov 2022 21:40:29 -0800 Subject: [PATCH 10/10] remove unused import --- prereise/utility/tests/test_generate_rural_shapefiles.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/prereise/utility/tests/test_generate_rural_shapefiles.py b/prereise/utility/tests/test_generate_rural_shapefiles.py index e6bdfdfb4..9c42a316f 100644 --- a/prereise/utility/tests/test_generate_rural_shapefiles.py +++ b/prereise/utility/tests/test_generate_rural_shapefiles.py @@ -1,5 +1,3 @@ -import json - import geopandas as gpd from shapely.geometry import Polygon