Merge pull request #326 from openego/features/#325-import-status2019

Features/#325 import status2019

src/egon/data/airflow/dags/pipeline.py

@@ -98,6 +98,7 @@
 from egon.data.datasets.zensus import ZensusMiscellaneous, ZensusPopulation
 from egon.data.datasets.zensus_mv_grid_districts import ZensusMvGridDistricts
 from egon.data.datasets.zensus_vg250 import ZensusVg250
+from egon.data.datasets.scenario_path import CreateIntermediateScenarios
 
 # Set number of threads used by numpy and pandas
 set_numexpr_threads()
@@ -672,6 +673,18 @@
         ]
     )
 
+    # Create intermediate scenarios based on status2019 and eGon100RE
+    create_intemediate_scenarios = CreateIntermediateScenarios(
+        dependencies=[
+            storage_etrago,
+            hts_etrago_table,
+            fill_etrago_generators,
+            household_electricity_demand_annual,
+            cts_demand_buildings,
+            emobility_mit,
+        ]
+    )
+
     # ########## Keep this dataset at the end
     # Sanity Checks
     sanity_checks = SanityChecks(

src/egon/data/datasets.yml

@@ -1271,3 +1271,7 @@ home_batteries:
     home_batteries:
       schema: 'supply'
       table: 'egon_home_batteries'
+
+scenario_path:
+  sources:
+    url_status2019: 'https://zenodo.org/records/13865306/files/PoWerD_status2019_v2.backup'

src/egon/data/datasets/scenario_path/__init__.py

@@ -0,0 +1,18 @@
+from egon.data.datasets import Dataset
+from egon.data.datasets.scenario_path.import_status2019 import (
+    download_status2019,
+    import_scn_status2019,
+)
+
+
+class CreateIntermediateScenarios(Dataset):
+    def __init__(self, dependencies):
+        super().__init__(
+            name="scenario_path",
+            version="0.0.1",
+            dependencies=dependencies,
+            tasks=(
+                download_status2019,
+                import_scn_status2019,
+            ),
+        )

src/egon/data/datasets/scenario_path/import_status2019.py

@@ -0,0 +1,102 @@
+"""
+Read eTraGo tables for the status2019 and import it to db
+"""
+
+from pathlib import Path
+from urllib.request import urlretrieve
+import os
+import subprocess
+
+import pandas as pd
+
+from egon.data import config, db
+import egon.data.config
+
+sources = egon.data.config.datasets()["scenario_path"]["sources"]
+
+
+def download_status2019():
+    """
+    Download the status2019 etrago tables from Zenodo
+
+    Returns
+    -------
+    None.
+
+    """
+    # Get parameters from config and set download URL
+    url = sources["url_status2019"]
+    status2019_path = Path(".") / "PoWerD_status2019.backup"
+
+    # Retrieve files
+    urlretrieve(url, status2019_path)
+
+    return
+
+
+def import_scn_status2019():
+    """
+    Read and import the scenario status2019 and import it into db
+
+    Parameters
+    ----------
+    *No parameters required
+
+    """
+    # Connect to the data base
+    con = db.engine()
+
+    # Clean existing data for status2019
+    tables = pd.read_sql(
+        """
+        SELECT tablename FROM pg_catalog.pg_tables
+        WHERE schemaname = 'grid'
+        """,
+        con,
+    )
+
+    tables = tables[
+        ~tables["tablename"].isin(
+            [
+                "egon_etrago_carrier",
+                "egon_etrago_temp_resolution",
+            ]
+        )
+    ]
+
+    for table in tables["tablename"]:
+        db.execute_sql(
+            f"""
+        DELETE FROM grid.{table} WHERE scn_name = 'status2019';
+        """
+        )
+
+    config_data = config.settings()["egon-data"]
+    database = config_data["--database-name"]
+    host = config_data["--database-host"]
+    port = config_data["--database-port"]
+    user = config_data["--database-user"]
+
+    my_env = os.environ.copy()
+    my_env["PGPASSWORD"] = config_data["--database-password"]
+
+    for table in tables["tablename"]:
+        subprocess.Popen(
+            [
+                "pg_restore",
+                "-d",
+                database,
+                "--host",
+                host,
+                "--port",
+                port,
+                "-U",
+                user,
+                "-a",
+                "--single-transaction",
+                f"--table={table}",
+                "PoWerD_status2019.backup",
+            ],
+            env=my_env,
+        )
+    return