Skip to content

Commit ba1eb31

Browse files
committed
Multi-modal network creation for global trade
1 parent 878c68b commit ba1eb31

File tree

6 files changed

+483
-16
lines changed

6 files changed

+483
-16
lines changed

config/config.yaml

+34-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
##########################
2-
### TRANSPORT WORKFLOW ###
3-
##########################
1+
##################################
2+
### TRANSPORT DAMAGES WORKFLOW ###
3+
##################################
44

55
# OSM datasets in PBF format, principally from: https://download.geofabrik.de/ #
66
infrastructure_datasets:
@@ -66,6 +66,37 @@ keep_tags:
6666
# Number of slices to cut dataset into -- must be a square number
6767
slice_count: 64
6868

69+
#####################################
70+
### TRANSPORT DISRUPTION WORKFLOW ###
71+
#####################################
72+
73+
# country for which trade OD has been prepared, and we are to route land trade flows
74+
study_country_iso_a3: "THA"
75+
76+
# transport cost information
77+
# road
78+
road_cost_USD_t_km: 0.05
79+
road_cost_USD_t_h: 0.48
80+
road_default_speed_limit_km_h: 80
81+
# rail
82+
rail_cost_USD_t_km: 0.05
83+
rail_cost_USD_t_h: 0.38
84+
rail_average_freight_speed_km_h: 40
85+
86+
# cost of changing transport mode in USD per tonne
87+
# from mistral/ccg-critical-minerals/processed_data/transport_costs/intermodal.xlsx, 20240611
88+
intermodal_cost_USD_t:
89+
road_rail: 5
90+
maritime_road: 4
91+
maritime_rail: 5
92+
93+
# drop trade flows with less volume than this (accelerate flow allocation)
94+
# N.B. 50t threshold preserves 91% of total volume and 88% of total value
95+
# for combined agriculture & manufacturing flows between Thailand road nodes -> partner countries
96+
minimum_flow_volume_t: 50
97+
98+
# if disrupting a network, remove edges experiencing hazard values in excess of this
99+
edge_failure_threshold: 0.5
69100

70101
#########################
71102
### FLOODING WORKFLOW ###

environment.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
name: open-gira
44
channels:
55
- conda-forge # majority of dependencies
6-
- bioconda # snakemake
76
- defaults
87
dependencies:
98
- python=3.10
@@ -25,6 +24,7 @@ dependencies:
2524
- gdal>=3.3 # command-line tools for spatial data
2625
- geopandas==0.14.4 # geospatial dataframes
2726
- geopy # geocoding client
27+
- igraph # graph algorithms and data structures
2828
- ipykernel # notebook support
2929
- jupyter # notebook support
3030
- jq # JSON processing tool
@@ -50,7 +50,7 @@ dependencies:
5050
- rioxarray # xarray datasets from raster files
5151
- scipy # scientific computing library
5252
- spatialpandas # plotting large datasets
53-
- snakemake==7.18.2 # workflow management
53+
- bioconda::snakemake==7.18.2 # workflow management
5454
# https://github.com/snakemake/snakemake/issues/1891
5555
- tabulate==0.8.10 # snakemake dependency with bug in 9.0.0, pin previous
5656
- tqdm==4.62.3 # progress bars

workflow/Snakefile

+15-11
Original file line numberDiff line numberDiff line change
@@ -76,27 +76,29 @@ if not config["best_estimate_windspeed_failure_threshold"] in config["transmissi
7676

7777
# Constrain wildcards to NOT use _ or /
7878
wildcard_constraints:
79+
ADMIN_SLUG="admin-level-[0-4]",
80+
AGG_FUNC_SLUG="agg-sum",
81+
CHUNK_SLUG="chunk-[\d]+",
82+
COST_OR_FRACTION="cost|fraction",
83+
DATASET="[^_/]+",
84+
DIRECT_DAMAGE_TYPES="fraction_per_RP|cost_per_RP|EAD|EAD_and_cost_per_RP",
85+
EVENTS_OR_FIXED="events|fixed",
86+
FILTER_SLUG="filter-[^_/]+",
87+
FILENAME="[^/]+",
88+
HAZARD_SLUG="hazard-[^_/]+",
89+
IRIS_SCENARIO="PRESENT|SSP1-2050|SSP2-2050|SSP5-2050",
7990
# the output dir must end in 'results'
8091
# e.g. '/data/open-gira/results', './results', '/my-results', etc.
8192
# this prevents us matching past it into other folders for an incorrect OUTPUT_DIR,
8293
# but is more flexible than reading a value from, for example, config.yaml
8394
OUTPUT_DIR="^.*results",
84-
DATASET="[^_/]+",
95+
PROJECT_SLUG="project-[^_/]+",
8596
SLICE_SLUG="slice-[0-9]+",
86-
FILTER_SLUG="filter-[^_/]+",
87-
HAZARD_SLUG="hazard-[^_/]+",
88-
ADMIN_SLUG="admin-level-[0-4]",
89-
AGG_FUNC_SLUG="agg-sum",
90-
FILENAME="[^/]+",
9197
STORM_BASIN="EP|NA|NI|SI|SP|WP",
92-
STORM_RP="[0-9]+",
93-
IRIS_SCENARIO="PRESENT|SSP1-2050|SSP2-2050|SSP5-2050",
9498
STORM_MODEL="constant|CMCC-CM2-VHR4|CNRM-CM6-1-HR|EC-Earth3P-HR|HadGEM3-GC31-HM",
9599
STORM_MODEL_FUTURE="CMCC-CM2-VHR4|CNRM-CM6-1-HR|EC-Earth3P-HR|HadGEM3-GC31-HM",
100+
STORM_RP="[0-9]+",
96101
STORM_SET="(?:IBTrACS|STORM|IRIS)[^\/]*",
97-
EVENTS_OR_FIXED="events|fixed",
98-
COST_OR_FRACTION="cost|fraction",
99-
DIRECT_DAMAGE_TYPES="fraction_per_RP|cost_per_RP|EAD|EAD_and_cost_per_RP",
100102
SAMPLE="\d+",
101103
# may be upper or lower, one 'f' or two
102104
TIFF_FILE="[^\/\.\s]+\.[tT][iI][fF][fF]?",
@@ -130,6 +132,8 @@ include: "transport/create_network.smk"
130132
include: "transport/osm_to_geoparquet.smk"
131133
include: "transport/create_overall_bbox.smk"
132134
include: "transport/join_data.smk"
135+
include: "transport/maritime/maritime.smk"
136+
include: "transport/multi-modal/multi_modal.smk"
133137

134138
include: "flood/aqueduct.smk"
135139
include: "flood/trim_hazard_data.smk"
+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
rule create_maritime_network:
2+
input:
3+
nodes = "{OUTPUT_DIR}/input/networks/maritime/nodes.gpq",
4+
edges_no_geom = "{OUTPUT_DIR}/input/networks/maritime/edges_by_cargo/maritime_base_network_general_cargo.pq",
5+
edges_visualisation = "{OUTPUT_DIR}/input/networks/maritime/edges.gpq",
6+
output:
7+
nodes = "{OUTPUT_DIR}/maritime_network/nodes.gpq",
8+
edges = "{OUTPUT_DIR}/maritime_network/edges.gpq",
9+
run:
10+
import igraph as ig
11+
import geopandas as gpd
12+
from shapely.geometry import Point
13+
from shapely.ops import linemerge
14+
from tqdm import tqdm
15+
16+
from open_gira.network_creation import preprocess_maritime_network
17+
18+
# possible cargo types = ("container", "dry_bulk", "general_cargo", "roro", "tanker")
19+
# for now, just use 'general_cargo'
20+
maritime_nodes, maritime_edges_no_geom = preprocess_maritime_network(
21+
input.nodes,
22+
input.edges_no_geom
23+
)
24+
25+
if config["study_country_iso_a3"] == "THA":
26+
# put Bangkok port in the right place...
27+
maritime_nodes.loc[maritime_nodes.name == "Bangkok_Thailand", "geometry"] = Point((100.5753, 13.7037))
28+
29+
# %%
30+
# Jasper's maritime edges in 'edges_by_cargo' do not contain geometry
31+
# this is because the AIS data that they were derived from only contain origin and destination port, not route
32+
# this is a pain for visualisation, so we will create a geometry for each from `maritime_vis_edges`
33+
34+
maritime_vis_edges = gpd.read_parquet(input.edges_visualisation)
35+
vis_graph = ig.Graph.DataFrame(maritime_vis_edges, directed=True, use_vids=False)
36+
37+
maritime_edges = maritime_edges_no_geom.copy()
38+
change_of_port_mask = maritime_edges_no_geom.from_port != maritime_edges_no_geom.to_port
39+
port_pairs_to_generate_geom_for = maritime_edges_no_geom[change_of_port_mask]
40+
for index, row in tqdm(port_pairs_to_generate_geom_for.iterrows(), total=len(port_pairs_to_generate_geom_for)):
41+
edge_list = vis_graph.get_shortest_path(row.from_port, row.to_port, weights="distance", output="epath")
42+
route_edges = maritime_vis_edges.iloc[edge_list]
43+
route_linestring = linemerge(list(route_edges.geometry))
44+
maritime_edges.loc[index, "geometry"] = route_linestring
45+
46+
maritime_edges = gpd.GeoDataFrame(maritime_edges).set_crs(epsg=4326)
47+
48+
maritime_nodes.to_parquet(output.nodes)
49+
maritime_edges.to_parquet(output.edges)
50+
51+
52+
rule plot_maritime_network:
53+
input:
54+
nodes = "{OUTPUT_DIR}/maritime_network/nodes.gpq",
55+
edges = "{OUTPUT_DIR}/maritime_network/edges.gpq",
56+
output:
57+
edges_plot = "{OUTPUT_DIR}/maritime_network/edges.png",
58+
run:
59+
import geopandas as gpd
60+
import matplotlib
61+
import matplotlib.pyplot as plt
62+
import numpy as np
63+
64+
from open_gira.plot.utils import chop_at_antimeridian
65+
66+
matplotlib.use("Agg")
67+
plt.style.use("bmh")
68+
69+
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
70+
world.geometry = world.geometry.boundary
71+
72+
maritime_nodes = gpd.read_parquet(input.nodes)
73+
maritime_edges = gpd.read_parquet(input.edges)
74+
75+
# whole network
76+
f, ax = plt.subplots(figsize=(16, 7))
77+
chop_at_antimeridian(maritime_edges, drop_null_geometry=True).plot(
78+
ax=ax,
79+
linewidth=0.5,
80+
alpha=0.8
81+
)
82+
world.plot(ax=ax, lw=0.5, alpha=0.2)
83+
ax.set_xticks(np.linspace(-180, 180, 13))
84+
ax.set_yticks([-60, -30, 0, 30, 60])
85+
ax.set_ylim(-65, 85)
86+
ax.set_xlim(-180, 180)
87+
ax.grid(alpha=0.3)
88+
ax.set_xlabel("Longitude [deg]")
89+
ax.set_ylabel("Latitude [deg]")
90+
f.savefig(output.edges_plot)
91+
92+
93+
rule plot_port_connections:
94+
input:
95+
nodes = "{OUTPUT_DIR}/maritime_network/nodes.gpq",
96+
edges = "{OUTPUT_DIR}/maritime_network/edges.gpq",
97+
output:
98+
port_trade_plots = directory("{OUTPUT_DIR}/maritime_network/port_trade_plots"),
99+
run:
100+
import os
101+
102+
import geopandas as gpd
103+
import matplotlib
104+
import matplotlib.pyplot as plt
105+
from tqdm import tqdm
106+
107+
from open_gira.plot.utils import chop_at_antimeridian
108+
109+
matplotlib.use("Agg")
110+
plt.style.use("bmh")
111+
112+
maritime_nodes = gpd.read_parquet(input.nodes)
113+
maritime_edges = gpd.read_parquet(input.edges)
114+
115+
# disambiguate the global view and plot the routes from each port, one port at a time
116+
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
117+
world.geometry = world.geometry.boundary
118+
119+
ports = maritime_edges.from_port.unique()
120+
os.makedirs(output.port_trade_plots)
121+
for port_id in tqdm(ports):
122+
filepath = os.path.join(output.port_trade_plots, f"{port_id}.png")
123+
f, ax = plt.subplots(figsize=(10,10))
124+
port = maritime_nodes[maritime_nodes.id == f"{port_id}_land"]
125+
routes = maritime_edges[maritime_edges.from_port == port_id]
126+
if all(routes.geometry.isna()):
127+
continue
128+
try:
129+
chop_at_antimeridian(routes, drop_null_geometry=True).plot(
130+
column="to_port",
131+
categorical=True,
132+
ax=ax
133+
)
134+
except ValueError:
135+
print(f"Failed to plot {port_id}, skipping...")
136+
continue
137+
maritime_nodes[maritime_nodes.id == f"{port_id}_land"].plot(
138+
ax=ax,
139+
markersize=500,
140+
marker="*",
141+
facecolor="none",
142+
color="r"
143+
)
144+
xmin, xmax = ax.get_xlim()
145+
ymin, ymax = ax.get_ylim()
146+
world.plot(ax=ax, linewidth=0.5, alpha=0.4)
147+
ax.set_xlim(xmin, xmax)
148+
ax.set_ylim(ymin, ymax)
149+
port_name, = port.name
150+
ax.set_title(f"{port_id} ({port_name.replace('_', ', ')}) estimated routes")
151+
ax.get_xaxis().set_visible(False)
152+
ax.get_yaxis().set_visible(False)
153+
154+
f.savefig(filepath)
155+
plt.close(f)

0 commit comments

Comments
 (0)