Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add script to translate urban and rural shapefiles to USA tamu loadzones #319

Open
wants to merge 10 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions prereise/gather/const.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,86 @@
"WV": "West Virginia",
"WY": "Wyoming",
}

lower_48_states_abv = list(abv2state.keys())
lower_48_states_abv.remove("AK")
lower_48_states_abv.remove("HI")

zone2state_abv = {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we could reuse the same lookup from powersimdata, e.g.

from powersimdata.network.model import ModelImmutables
mi = ModelImmutables("usa_tamu")
zone2state_abv = mi.zones["loadzone2abv"]

"Alabama": "AL",
"Arizona": "AZ",
"Arkansas": "AR",
"Bay Area": "CA",
"Central California": "CA",
"Chicago North Illinois": "IL",
"Coast": "TX",
"Colorado": "CO",
"Connecticut": "CT",
"Delaware": "DE",
"East": "TX",
"East Texas": "TX",
"El Paso": "TX",
"Far West": "TX",
"Florida North": "FL",
"Florida Panhandle": "FL",
"Florida South": "FL",
"Georgia North": "GA",
"Georgia South": "GA",
"Idaho": "ID",
"Illinois Downstate": "IL",
"Indiana": "IN",
"Iowa": "IA",
"Kansas": "KS",
"Kentucky": "KY",
"Louisiana": "LA",
"Maine": "ME",
"Maryland": "MD",
"Massachusetts": "MA",
"Michigan Northern": "MI",
"Michigan Southern": "MI",
"Minnesota Northern": "MN",
"Minnesota Southern": "MN",
"Mississippi": "MS",
"Missouri East": "MO",
"Missouri West": "MO",
"Montana Eastern": "MT",
"Montana Western": "MT",
"Nebraska": "NE",
"Nevada": "NV",
"New Hampshire": "NH",
"New Jersey": "NJ",
"New Mexico Eastern": "NM",
"New Mexico Western": "NM",
"New York City": "NY",
"North": "TX",
"North Carolina": "NC",
"North Central": "TX",
"North Dakota": "ND",
"Northern California": "CA",
"Ohio Lake Erie": "OH",
"Ohio River": "OH",
"Oklahoma": "OK",
"Oregon": "OR",
"Pennsylvania Eastern": "PA",
"Pennsylvania Western": "PA",
"Rhode Island": "RI",
"South": "TX",
"South Carolina": "SC",
"South Central": "TX",
"South Dakota": "SD",
"Southeast California": "CA",
"Southwest California": "CA",
"Tennessee": "TN",
"Texas Panhandle": "TX",
"Upstate New York": "NY",
"Utah": "UT",
"Vermont": "VT",
"Virginia Mountains": "VA",
"Virginia Tidewater": "VA",
"Washington": "WA",
"West": "TX",
"West Virginia": "WV",
"Western North Carolina": "NC",
"Wisconsin": "WI",
"Wyoming": "WY",
}
Empty file added prereise/utility/__init__.py
Empty file.
13 changes: 7 additions & 6 deletions prereise/utility/generate_rural_shapefiles.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import os

import geopandas as gpd
import pandas as pd

from prereise.gather.const import abv2state
from prereise.gather.const import lower_48_states_abv
from prereise.utility.shapefile import download_shapefiles


Expand All @@ -16,22 +17,22 @@ def append_rural_areas_to_urban(states, urban_areas):
:return: (*geopandas.geodataframe.GeoDataFrame*) -- new gdf of urban and rural areas
includes a column indicating whether the area is urban or not
"""
lower_48_states_abv = list(abv2state.keys())
lower_48_states_abv.remove("AK")
lower_48_states_abv.remove("HI")

states = states.rename(columns={"STUSPS": "state"})
states = states.loc[states["state"].isin(lower_48_states_abv)]

urban_areas["state"] = urban_areas["NAME10"].str[-2:]
# this drops 66 out of 3601 rows
urban_areas = urban_areas.loc[urban_areas["state"].isin(lower_48_states_abv)]

urban_areas = urban_areas.rename(columns={"NAME10": "name"})
urban_areas = urban_areas.set_index("name", verify_integrity=True)
urban_areas["urban"] = True

rural_areas = states.overlay(urban_areas, how="difference")
rural_areas.index = rural_areas["state"] + "_rural"
rural_areas["urban"] = False

return urban_areas.append(rural_areas, ignore_index=True)[
return pd.concat([urban_areas, rural_areas], verify_integrity=True)[
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hadn't seen the verify_integrity parameter before, that's neat.

["state", "geometry", "urban"]
]

Expand Down
Empty file.
62 changes: 20 additions & 42 deletions prereise/utility/tests/test_generate_rural_shapefiles.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import json

import geopandas as gpd
from shapely.geometry import Polygon

Expand Down Expand Up @@ -28,44 +26,24 @@ def test_append_rural_areas_to_urban():

r_and_u = append_rural_areas_to_urban(mock_states, mock_urban)

# Convert to json then back to dict -- this unpacks the Polygon objects
assert json.loads(r_and_u.to_json()) == {
"type": "FeatureCollection",
"features": [
{
"id": "0",
"type": "Feature",
"properties": {"state": "CO", "urban": True},
"geometry": {
"type": "Polygon",
"coordinates": [
[[0.1, 0.1], [0.1, 0.5], [0.5, 0.5], [0.5, 0.1], [0.1, 0.1]]
],
},
},
{
"id": "1",
"type": "Feature",
"properties": {"state": "CO", "urban": True},
"geometry": {
"type": "Polygon",
"coordinates": [
[[4.0, 4.0], [4.0, 4.5], [4.5, 4.5], [4.5, 4.0], [4.0, 4.0]]
],
},
},
{
"id": "2",
"type": "Feature",
"properties": {"state": "CO", "urban": False},
"geometry": {
"type": "Polygon",
"coordinates": [
[[5.0, 5.0], [5.0, 0.0], [0.0, 0.0], [0.0, 5.0], [5.0, 5.0]],
[[4.5, 4.0], [4.5, 4.5], [4.0, 4.5], [4.0, 4.0], [4.5, 4.0]],
[[0.5, 0.5], [0.1, 0.5], [0.1, 0.1], [0.5, 0.1], [0.5, 0.5]],
],
},
},
],
expected = {
"geometry": {
"Boulder_CO": Polygon(
[[4.0, 4.0], [4.0, 4.5], [4.5, 4.5], [4.5, 4.0], [4.0, 4.0]]
),
"CO_rural": Polygon(
[[5.0, 5.0], [5.0, 0.0], [0.0, 0.0], [0.0, 5.0], [5.0, 5.0]],
[
[[4.5, 4.0], [4.5, 4.5], [4.0, 4.5], [4.0, 4.0], [4.5, 4.0]],
[[0.5, 0.5], [0.1, 0.5], [0.1, 0.1], [0.5, 0.1], [0.5, 0.5]],
],
),
"Denver_CO": Polygon(
[[0.1, 0.1], [0.1, 0.5], [0.5, 0.5], [0.5, 0.1], [0.1, 0.1]]
),
},
"state": {"Boulder_CO": "CO", "CO_rural": "CO", "Denver_CO": "CO"},
"urban": {"Boulder_CO": True, "CO_rural": False, "Denver_CO": True},
}

assert r_and_u.to_dict() == expected
105 changes: 105 additions & 0 deletions prereise/utility/tests/test_urban_rural_to_usa_tamu.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import geopandas as gpd
import pandas as pd
from powersimdata.input.grid import Grid
from shapely.geometry import Point, Polygon

from prereise.utility.translate_urban_rural_to_usa_tamu import (
create_substation_gdf,
create_usa_tamu_convex_hull_shapefile,
fix_lz_border,
format_states_gdf,
)

WA_lz_polygon = Polygon([(1, 1), (0, 5), (5, 6), (4, 1)])
CA_lz_polygon = Polygon([(0, 10), (0, 15), (5, 15), (5, 10)])
WA_border_polygon = Polygon([(0, 0), (0, 5), (5, 5), (5, 0)])
CA_border_polygon = Polygon([(0, 10), (0, 20), (3, 20), (3, 10)])
HI_border_polygon = Polygon([(100, 100), (100, 105), (105, 105)])

# Mock US State borders
states = gpd.GeoDataFrame(
{
"geometry": [WA_border_polygon, CA_border_polygon, HI_border_polygon],
"STUSPS": ["WA", "CA", "HI"],
}
)
formatted_states = gpd.GeoDataFrame(
{
"geometry": {"WA": WA_border_polygon, "CA": CA_border_polygon},
"state": {"WA": "WA", "CA": "CA"},
}
)
sub_gdf = gpd.GeoDataFrame(
{
"zone": {122: "Bay Area", 6134: "Washington"},
"geometry": {122: Point(2, 3), 6134: Point(14, 4)},
}
)


def test_format_states_gdf():
results = format_states_gdf(states)
assert formatted_states.to_dict() == results.to_dict()


class MockGrid:
def __init__(self):
"""Constructor"""
self.bus2sub = pd.DataFrame({"sub_id": {0: 122, 1: 122, 2: 6134}})
self.bus = pd.DataFrame({"zone_id": {0: 204, 1: 204, 2: 201}})
self.id2zone = {204: "Bay Area", 201: "Washington"}
self.sub = pd.DataFrame({"lon": {122: 2, 6134: 14}, "lat": {122: 3, 6134: 4}})

@property
def __class__(self):
"""If anyone asks, I'm a Grid object!"""
return Grid


def test_create_substation_gdf():
mock_grid = MockGrid()
results = create_substation_gdf(mock_grid)
assert sub_gdf.to_dict() == results.to_dict()


def test_fix_lz_border_single_lz():
results = fix_lz_border(formatted_states, "WA", WA_lz_polygon)
expected = WA_border_polygon
assert expected == results


def test_fix_lz_border_multi_lz():
results = fix_lz_border(formatted_states, "CA", CA_lz_polygon)
expected = Polygon([(0, 15), (3, 15), (3, 10), (0, 10)])
assert expected == results


def test_create_usa_tamu_convex_hull_shapefile():
# gdf where CA substations match CA_lz_polygon
sub_gdf_2 = gpd.GeoDataFrame(
{
"zone": {
122: "Bay Area",
123: "Bay Area",
124: "Bay Area",
125: "Bay Area",
6134: "Washington",
},
"geometry": {
122: Point(0, 10),
123: Point(0, 15),
124: Point(5, 15),
125: Point(5, 10),
6134: Point(14, 4),
},
}
)
results = create_usa_tamu_convex_hull_shapefile(sub_gdf_2, formatted_states)
expected = {
"geometry": {
"Bay Area": Polygon([(0, 15), (3, 15), (3, 10), (0, 10)]),
"Washington": WA_border_polygon,
},
"state": {"Bay Area": "CA", "Washington": "WA"},
}
assert expected == results.to_dict()
Loading