diff --git a/src/egon/data/datasets/industrial_gas_demand.py b/src/egon/data/datasets/industrial_gas_demand.py index 885bbba58..f372a4176 100755 --- a/src/egon/data/datasets/industrial_gas_demand.py +++ b/src/egon/data/datasets/industrial_gas_demand.py @@ -9,7 +9,9 @@ """ from pathlib import Path +import logging import os +import shutil from geoalchemy2.types import Geometry from shapely import wkt @@ -17,7 +19,7 @@ import pandas as pd import requests -from egon.data import db +from egon.data import config, db from egon.data.config import settings from egon.data.datasets import Dataset from egon.data.datasets.etrago_helpers import ( @@ -25,9 +27,11 @@ initialise_bus_insertion, ) from egon.data.datasets.etrago_setup import link_geom_from_buses -from egon.data.datasets.pypsaeursec import read_network +from egon.data.datasets.pypsaeur import read_network from egon.data.datasets.scenario_parameters import get_sector_parameters +logger = logging.getLogger(__name__) + class IndustrialGasDemand(Dataset): """ @@ -44,7 +48,7 @@ class IndustrialGasDemand(Dataset): #: name: str = "IndustrialGasDemand" #: - version: str = "0.0.4" + version: str = "0.0.5" def __init__(self, dependencies): super().__init__( @@ -402,32 +406,40 @@ def insert_industrial_gas_demand_egon2035(): None """ - scn_name = "eGon2035" - delete_old_entries(scn_name) - - industrial_gas_demand = pd.concat( - [ - read_and_process_demand( - scn_name=scn_name, - carrier="CH4_for_industry", - grid_carrier="CH4", - ), - read_and_process_demand( - scn_name=scn_name, - carrier="H2_for_industry", - grid_carrier="H2", - ), - ] - ) + if "eGon2035" in config.settings()["egon-data"]["--scenarios"]: + scn_name = "eGon2035" + delete_old_entries(scn_name) + + industrial_gas_demand = pd.concat( + [ + read_and_process_demand( + scn_name=scn_name, + carrier="CH4_for_industry", + grid_carrier="CH4", + ), + read_and_process_demand( + scn_name=scn_name, + carrier="H2_for_industry", + grid_carrier="H2", + ), + ] + ) - industrial_gas_demand = ( - industrial_gas_demand.groupby(["bus", "carrier"])["p_set"] - .apply(lambda x: [sum(y) for y in zip(*x)]) - .reset_index(drop=False) - ) + industrial_gas_demand = ( + industrial_gas_demand.groupby(["bus", "carrier"])["p_set"] + .apply(lambda x: [sum(y) for y in zip(*x)]) + .reset_index(drop=False) + ) - industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name) - insert_industrial_gas_demand_time_series(industrial_gas_demand) + industrial_gas_demand = insert_new_entries( + industrial_gas_demand, scn_name + ) + insert_industrial_gas_demand_time_series(industrial_gas_demand) + else: + print( + """eGon2035 is not part of the scenario list. This task is not + executed""" + ) def insert_industrial_gas_demand_egon100RE(): @@ -460,104 +472,118 @@ def insert_industrial_gas_demand_egon100RE(): None """ - scn_name = "eGon100RE" - delete_old_entries(scn_name) + if "eGon100RE" in config.settings()["egon-data"]["--scenarios"]: + scn_name = "eGon100RE" + delete_old_entries(scn_name) - # read demands - industrial_gas_demand_CH4 = read_and_process_demand( - scn_name=scn_name, carrier="CH4_for_industry", grid_carrier="CH4" - ) - industrial_gas_demand_H2 = read_and_process_demand( - scn_name=scn_name, carrier="H2_for_industry", grid_carrier="H2" - ) + # read demands + industrial_gas_demand_CH4 = read_and_process_demand( + scn_name=scn_name, carrier="CH4_for_industry", grid_carrier="CH4" + ) + industrial_gas_demand_H2 = read_and_process_demand( + scn_name=scn_name, carrier="H2_for_industry", grid_carrier="H2" + ) - # adjust H2 and CH4 total demands (values from PES) - # CH4 demand = 0 in 100RE, therefore scale H2 ts - # fallback values see https://github.com/openego/eGon-data/issues/626 - n = read_network() + # adjust H2 and CH4 total demands (values from PES) + # CH4 demand = 0 in 100RE, therefore scale H2 ts + # fallback values see https://github.com/openego/eGon-data/issues/626 + n = read_network() + + try: + H2_total_PES = ( + n.loads[n.loads["carrier"] == "H2 for industry"].loc[ + "DE0 0 H2 for industry", "p_set" + ] + * 8760 + ) + except KeyError: + H2_total_PES = 42090000 + print( + "Could not find data from PES-run, assigning fallback number." + ) - try: - H2_total_PES = ( - n.loads[n.loads["carrier"] == "H2 for industry"].loc[ - "DE0 0 H2 for industry", "p_set" - ] - * 8760 - ) - except KeyError: - H2_total_PES = 42090000 - print("Could not find data from PES-run, assigning fallback number.") + try: + CH4_total_PES = ( + n.loads[n.loads["carrier"] == "gas for industry"].loc[ + "DE0 0 gas for industry", "p_set" + ] + * 8760 + ) + except KeyError: + CH4_total_PES = 105490000 + print( + "Could not find data from PES-run, assigning fallback number." + ) - try: - CH4_total_PES = ( - n.loads[n.loads["carrier"] == "gas for industry"].loc[ - "DE0 0 gas for industry", "p_set" - ] - * 8760 + boundary = settings()["egon-data"]["--dataset-boundary"] + if boundary != "Everything": + # modify values for test mode + # the values are obtained by evaluating the share of H2 demand in + # test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2 + # demand in full Germany model (NUTS0: DE). The task has been outsourced + # to save processing cost + H2_total_PES *= 0.01855683050330346 + CH4_total_PES *= 0.01855683050330346 + + H2_total = ( + industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum() ) - except KeyError: - CH4_total_PES = 105490000 - print("Could not find data from PES-run, assigning fallback number.") - boundary = settings()["egon-data"]["--dataset-boundary"] - if boundary != "Everything": - # modify values for test mode - # the values are obtained by evaluating the share of H2 demand in - # test region (NUTS1: DEF, Schleswig-Holstein) with respect to the H2 - # demand in full Germany model (NUTS0: DE). The task has been outsourced - # to save processing cost - H2_total_PES *= 0.01855683050330346 - CH4_total_PES *= 0.01855683050330346 - - H2_total = industrial_gas_demand_H2["p_set"].apply(sum).astype(float).sum() - - industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[ - "p_set" - ].apply(lambda x: [val / H2_total * CH4_total_PES for val in x]) - industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[ - "p_set" - ].apply(lambda x: [val / H2_total * H2_total_PES for val in x]) - - # consistency check - total_CH4_distributed = sum( - [sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()] - ) - total_H2_distributed = sum( - [sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()] - ) + industrial_gas_demand_CH4["p_set"] = industrial_gas_demand_H2[ + "p_set" + ].apply(lambda x: [val / H2_total * CH4_total_PES for val in x]) + industrial_gas_demand_H2["p_set"] = industrial_gas_demand_H2[ + "p_set" + ].apply(lambda x: [val / H2_total * H2_total_PES for val in x]) - print( - f"Total amount of industrial H2 demand distributed is " - f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand " - f"distributed is {total_CH4_distributed} MWh." - ) - msg = ( - f"Total amount of industrial H2 demand from P-E-S is equal to " - f"{H2_total_PES}, which should be identical to the distributed amount " - f"of {total_H2_distributed}, but it is not." - ) - assert round(H2_total_PES) == round(total_H2_distributed), msg + # consistency check + total_CH4_distributed = sum( + [sum(x) for x in industrial_gas_demand_CH4["p_set"].to_list()] + ) + total_H2_distributed = sum( + [sum(x) for x in industrial_gas_demand_H2["p_set"].to_list()] + ) - msg = ( - f"Total amount of industrial CH4 demand from P-E-S is equal to " - f"{CH4_total_PES}, which should be identical to the distributed amount " - f"of {total_CH4_distributed}, but it is not." - ) - assert round(CH4_total_PES) == round(total_CH4_distributed), msg + print( + f"Total amount of industrial H2 demand distributed is " + f"{total_H2_distributed} MWh. Total amount of industrial CH4 demand " + f"distributed is {total_CH4_distributed} MWh." + ) + msg = ( + f"Total amount of industrial H2 demand from P-E-S is equal to " + f"{H2_total_PES}, which should be identical to the distributed amount " + f"of {total_H2_distributed}, but it is not." + ) + assert round(H2_total_PES) == round(total_H2_distributed), msg - industrial_gas_demand = pd.concat( - [ - industrial_gas_demand_CH4, - industrial_gas_demand_H2, - ] - ) - industrial_gas_demand = ( - industrial_gas_demand.groupby(["bus", "carrier"])["p_set"] - .apply(lambda x: [sum(y) for y in zip(*x)]) - .reset_index(drop=False) - ) + msg = ( + f"Total amount of industrial CH4 demand from P-E-S is equal to " + f"{CH4_total_PES}, which should be identical to the distributed amount " + f"of {total_CH4_distributed}, but it is not." + ) + assert round(CH4_total_PES) == round(total_CH4_distributed), msg + + industrial_gas_demand = pd.concat( + [ + industrial_gas_demand_CH4, + industrial_gas_demand_H2, + ] + ) + industrial_gas_demand = ( + industrial_gas_demand.groupby(["bus", "carrier"])["p_set"] + .apply(lambda x: [sum(y) for y in zip(*x)]) + .reset_index(drop=False) + ) - industrial_gas_demand = insert_new_entries(industrial_gas_demand, scn_name) - insert_industrial_gas_demand_time_series(industrial_gas_demand) + industrial_gas_demand = insert_new_entries( + industrial_gas_demand, scn_name + ) + insert_industrial_gas_demand_time_series(industrial_gas_demand) + else: + print( + """eGon100RE is not part of the scenario list. This task is not + executed""" + ) def insert_industrial_gas_demand_time_series(egon_etrago_load_gas): @@ -609,34 +635,50 @@ def download_industrial_gas_demand(): information on these data, refer to the `Extremos project documentation `_. """ - correspondance_url = ( - "http://opendata.ffe.de:3000/region?id_region_type=eq.38" - ) - - # Read and save data - result_corr = requests.get(correspondance_url) - target_file = Path(".") / "datasets/gas_data/demand/region_corr.json" - os.makedirs(os.path.dirname(target_file), exist_ok=True) - pd.read_json(result_corr.content).to_json(target_file) + try: + correspondance_url = ( + "http://opendata.ffe.de:3000/region?id_region_type=eq.38" + ) - carriers = {"H2_for_industry": "2,162", "CH4_for_industry": "2,11"} - url = "http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq." + # Read and save data + result_corr = requests.get(correspondance_url) + target_file = Path(".") / "datasets/gas_data/demand/region_corr.json" + os.makedirs(os.path.dirname(target_file), exist_ok=True) + pd.read_json(result_corr.content).to_json(target_file) - for scn_name in ["eGon2035", "eGon100RE"]: - year = str( - get_sector_parameters("global", scn_name)["population_year"] + carriers = {"H2_for_industry": "2,162", "CH4_for_industry": "2,11"} + url = ( + "http://opendata.ffe.de:3000/opendata?id_opendata=eq.66&&year=eq." ) - for carrier, internal_id in carriers.items(): - # Download the data - datafilter = "&&internal_id=eq.{" + internal_id + "}" - request = url + year + datafilter - - # Read and save data - result = requests.get(request) - target_file = ( - Path(".") - / "datasets/gas_data/demand" - / (carrier + "_" + scn_name + ".json") + for scn_name in ["eGon2035", "eGon100RE"]: + year = str( + get_sector_parameters("global", scn_name)["population_year"] ) - pd.read_json(result.content).to_json(target_file) + + for carrier, internal_id in carriers.items(): + # Download the data + datafilter = "&&internal_id=eq.{" + internal_id + "}" + request = url + year + datafilter + + # Read and save data + result = requests.get(request) + target_file = ( + Path(".") + / "datasets/gas_data/demand" + / (carrier + "_" + scn_name + ".json") + ) + pd.read_json(result.content).to_json(target_file) + except: + logger.warning( + """ + Due to temporal problems in the FFE platform, data for the scenarios + eGon2035 and eGon100RE are imported lately from csv files. Data for + other scenarios is unfortunately unavailable. + """ + ) + shutil.copytree( + "data_bundle_powerd_data/industrial_gas_demand", + "datasets/gas_data/demand", + dirs_exist_ok=True, + )