Skip to content

Commit

Permalink
to latest
Browse files Browse the repository at this point in the history
  • Loading branch information
Robert John committed Nov 5, 2024
1 parent ff460ea commit d8739a8
Showing 1 changed file with 183 additions and 141 deletions.
324 changes: 183 additions & 141 deletions src/egon/data/datasets/industrial_gas_demand.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,29 @@
"""

from pathlib import Path
import logging
import os
import shutil

from geoalchemy2.types import Geometry
from shapely import wkt
import numpy as np
import pandas as pd
import requests

from egon.data import db
from egon.data import config, db
from egon.data.config import settings
from egon.data.datasets import Dataset
from egon.data.datasets.etrago_helpers import (
finalize_bus_insertion,
initialise_bus_insertion,
)
from egon.data.datasets.etrago_setup import link_geom_from_buses
from egon.data.datasets.pypsaeursec import read_network
from egon.data.datasets.pypsaeur import read_network
from egon.data.datasets.scenario_parameters import get_sector_parameters

logger = logging.getLogger(__name__)


class IndustrialGasDemand(Dataset):
"""
Expand All @@ -44,7 +48,7 @@ class IndustrialGasDemand(Dataset):
#:
name: str = "IndustrialGasDemand"
#:
version: str = "0.0.4"
version: str = "0.0.5"

def __init__(self, dependencies):
super().__init__(
Expand Down Expand Up @@ -402,32 +406,40 @@ def insert_industrial_gas_demand_egon2035():
None
"""
scn_name = "eGon2035"
delete_old_entries(scn_name)

industrial_gas_demand = pd.concat(
[
read_and_process_demand(
scn_name=scn_name,
carrier="CH4_for_industry",
grid_carrier="CH4",
),
read_and_process_demand(
scn_name=scn_name,
carrier="H2_for_industry",
grid_carrier="H2",
),
]
)
if "eGon2035" in config.settings()["egon-data"]["--scenarios"]:
scn_name = "eGon2035"
delete_old_entries(scn_name)

industrial_gas_demand = pd.concat(
[
read_and_process_demand(
scn_name=scn_name,
carrier="CH4_for_industry",
grid_carrier="CH4",
),
read_and_process_demand(
scn_name=scn_name,
carrier="H2_for_industry",
grid_carrier="H2",
),
]
)

industrial_gas_demand = (
industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
.apply(lambda x: [sum(y) for y in zip(*x)])
.reset_index(drop=False)
)
industrial_gas_demand = (
industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
.apply(lambda x: [sum(y) for y in zip(*x)])
.reset_index(drop=False)
)

industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
insert_industrial_gas_demand_time_series(industrial_gas_demand)
industrial_gas_demand = insert_new_entries(
industrial_gas_demand, scn_name
)
insert_industrial_gas_demand_time_series(industrial_gas_demand)
else:
print(
"""eGon2035 is not part of the scenario list. This task is not
executed"""
)


def insert_industrial_gas_demand_egon100RE():
Expand Down Expand Up @@ -460,104 +472,118 @@ def insert_industrial_gas_demand_egon100RE():
None
"""
scn_name = "eGon100RE"
delete_old_entries(scn_name)
if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]:
scn_name = "eGon100RE"
delete_old_entries(scn_name)

# read demands
industrial_gas_demand_CH4 = read_and_process_demand(
scn_name=scn_name, carrier="CH4_for_industry", grid_carrier="CH4"
)
industrial_gas_demand_H2 = read_and_process_demand(
scn_name=scn_name, carrier="H2_for_industry", grid_carrier="H2"
)
# read demands
industrial_gas_demand_CH4 = read_and_process_demand(
scn_name=scn_name, carrier="CH4_for_industry", grid_carrier="CH4"
)
industrial_gas_demand_H2 = read_and_process_demand(
scn_name=scn_name, carrier="H2_for_industry", grid_carrier="H2"
)

# adjust H2 and CH4 total demands (values from PES)
# CH4 demand = 0 in 100RE, therefore scale H2 ts
# fallback values see https://github.com/openego/eGon-data/issues/626
n = read_network()
# adjust H2 and CH4 total demands (values from PES)
# CH4 demand = 0 in 100RE, therefore scale H2 ts
# fallback values see https://github.com/openego/eGon-data/issues/626
n = read_network()

try:
H2_total_PES = (
n.loads[n.loads["carrier"] == "H2 for industry"].loc[
"DE0 0 H2 for industry", "p_set"
]
* 8760
)
except KeyError:
H2_total_PES = 42090000
print(
"Could not find data from PES-run, assigning fallback number."
)

try:
H2_total_PES = (
n.loads[n.loads["carrier"] == "H2 for industry"].loc[
"DE0 0 H2 for industry", "p_set"
]
* 8760
)
except KeyError:
H2_total_PES = 42090000
print("Could not find data from PES-run, assigning fallback number.")
try:
CH4_total_PES = (
n.loads[n.loads["carrier"] == "gas for industry"].loc[
"DE0 0 gas for industry", "p_set"
]
* 8760
)
except KeyError:
CH4_total_PES = 105490000
print(
"Could not find data from PES-run, assigning fallback number."
)

try:
CH4_total_PES = (
n.loads[n.loads["carrier"] == "gas for industry"].loc[
"DE0 0 gas for industry", "p_set"
]
* 8760
boundary = settings()["egon-data"]["--dataset-boundary"]
if boundary != "Everything":
# modify values for test mode
# the values are obtained by evaluating the share of H2 demand in
# test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2
# demand in full Germany model (NUTS0: DE). The task has been outsourced
# to save processing cost
H2_total_PES *= 0.01855683050330346
CH4_total_PES *= 0.01855683050330346

H2_total = (
industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum()
)
except KeyError:
CH4_total_PES = 105490000
print("Could not find data from PES-run, assigning fallback number.")

boundary = settings()["egon-data"]["--dataset-boundary"]
if boundary != "Everything":
# modify values for test mode
# the values are obtained by evaluating the share of H2 demand in
# test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2
# demand in full Germany model (NUTS0: DE). The task has been outsourced
# to save processing cost
H2_total_PES *= 0.01855683050330346
CH4_total_PES *= 0.01855683050330346

H2_total = industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum()

industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[
"p_set"
].apply(lambda x: [val / H2_total * CH4_total_PES for val in x])
industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[
"p_set"
].apply(lambda x: [val / H2_total * H2_total_PES for val in x])

# consistency check
total_CH4_distributed = sum(
[sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()]
)
total_H2_distributed = sum(
[sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()]
)
industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[
"p_set"
].apply(lambda x: [val / H2_total * CH4_total_PES for val in x])
industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[
"p_set"
].apply(lambda x: [val / H2_total * H2_total_PES for val in x])

print(
f"Total amount of industrial H2 demand distributed is "
f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand "
f"distributed is {total_CH4_distributed} MWh."
)
msg = (
f"Total amount of industrial H2 demand from P-E-S is equal to "
f"{H2_total_PES}, which should be identical to the distributed amount "
f"of {total_H2_distributed}, but it is not."
)
assert round(H2_total_PES) == round(total_H2_distributed), msg
# consistency check
total_CH4_distributed = sum(
[sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()]
)
total_H2_distributed = sum(
[sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()]
)

msg = (
f"Total amount of industrial CH4 demand from P-E-S is equal to "
f"{CH4_total_PES}, which should be identical to the distributed amount "
f"of {total_CH4_distributed}, but it is not."
)
assert round(CH4_total_PES) == round(total_CH4_distributed), msg
print(
f"Total amount of industrial H2 demand distributed is "
f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand "
f"distributed is {total_CH4_distributed} MWh."
)
msg = (
f"Total amount of industrial H2 demand from P-E-S is equal to "
f"{H2_total_PES}, which should be identical to the distributed amount "
f"of {total_H2_distributed}, but it is not."
)
assert round(H2_total_PES) == round(total_H2_distributed), msg

industrial_gas_demand = pd.concat(
[
industrial_gas_demand_CH4,
industrial_gas_demand_H2,
]
)
industrial_gas_demand = (
industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
.apply(lambda x: [sum(y) for y in zip(*x)])
.reset_index(drop=False)
)
msg = (
f"Total amount of industrial CH4 demand from P-E-S is equal to "
f"{CH4_total_PES}, which should be identical to the distributed amount "
f"of {total_CH4_distributed}, but it is not."
)
assert round(CH4_total_PES) == round(total_CH4_distributed), msg

industrial_gas_demand = pd.concat(
[
industrial_gas_demand_CH4,
industrial_gas_demand_H2,
]
)
industrial_gas_demand = (
industrial_gas_demand.groupby(["bus", "carrier"])["p_set"]
.apply(lambda x: [sum(y) for y in zip(*x)])
.reset_index(drop=False)
)

industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name)
insert_industrial_gas_demand_time_series(industrial_gas_demand)
industrial_gas_demand = insert_new_entries(
industrial_gas_demand, scn_name
)
insert_industrial_gas_demand_time_series(industrial_gas_demand)
else:
print(
"""eGon100RE is not part of the scenario list. This task is not
executed"""
)


def insert_industrial_gas_demand_time_series(egon_etrago_load_gas):
Expand Down Expand Up @@ -609,34 +635,50 @@ def download_industrial_gas_demand():
information on these data, refer to the `Extremos project documentation <https://opendata.ffe.de/project/extremos/>`_.
"""
correspondance_url = (
"http://opendata.ffe.de:3000/region?id_region_type=eq.38"
)

# Read and save data
result_corr = requests.get(correspondance_url)
target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
os.makedirs(os.path.dirname(target_file), exist_ok=True)
pd.read_json(result_corr.content).to_json(target_file)
try:
correspondance_url = (
"http://opendata.ffe.de:3000/region?id_region_type=eq.38"
)

carriers = {"H2_for_industry": "2,162", "CH4_for_industry": "2,11"}
url = "http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq."
# Read and save data
result_corr = requests.get(correspondance_url)
target_file = Path(".") / "datasets/gas_data/demand/region_corr.json"
os.makedirs(os.path.dirname(target_file), exist_ok=True)
pd.read_json(result_corr.content).to_json(target_file)

for scn_name in ["eGon2035", "eGon100RE"]:
year = str(
get_sector_parameters("global", scn_name)["population_year"]
carriers = {"H2_for_industry": "2,162", "CH4_for_industry": "2,11"}
url = (
"http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq."
)

for carrier, internal_id in carriers.items():
# Download the data
datafilter = "&&internal_id=eq.{" + internal_id + "}"
request = url + year + datafilter

# Read and save data
result = requests.get(request)
target_file = (
Path(".")
/ "datasets/gas_data/demand"
/ (carrier + "_" + scn_name + ".json")
for scn_name in ["eGon2035", "eGon100RE"]:
year = str(
get_sector_parameters("global", scn_name)["population_year"]
)
pd.read_json(result.content).to_json(target_file)

for carrier, internal_id in carriers.items():
# Download the data
datafilter = "&&internal_id=eq.{" + internal_id + "}"
request = url + year + datafilter

# Read and save data
result = requests.get(request)
target_file = (
Path(".")
/ "datasets/gas_data/demand"
/ (carrier + "_" + scn_name + ".json")
)
pd.read_json(result.content).to_json(target_file)
except:
logger.warning(
"""
Due to temporal problems in the FFE platform, data for the scenarios
eGon2035 and eGon100RE are imported lately from csv files. Data for
other scenarios is unfortunately unavailable.
"""
)
shutil.copytree(
"data_bundle_powerd_data/industrial_gas_demand",
"datasets/gas_data/demand",
dirs_exist_ok=True,
)

0 comments on commit d8739a8

Please sign in to comment.