Merge pull request #117 from openego/fixes/#116-entsoe-py-can-not-be-…

…used

Fixes/#116 entsoe py can not be used

README.rst

@@ -137,6 +137,15 @@ packages are required too. Right now these are:
   You also have to agree on the `terms of use
   <https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products>`_
 
+* To download generation and demand data from entsoe you need to register in
+  the `entsoe platform <https://transparency.entsoe.eu/>`_. Once the user is
+  created, a personal token can be generated to access the available data. This
+  token must be saved in a text file called *.entsoe-token* in the home directory.
+  In Ubuntu you can go to the home directory by typing :code:`cd ~` in a
+  terminal. To create the file execute :code:`nano ".entsoe-token"` and then
+  paste your personal token (36 characters). Finally press :code:`CRL + x` to 
+  save and exit.
+
 * Make sure you have enough free disk space (~350 GB) in your working
   directory.
 

setup.py

@@ -87,7 +87,7 @@ def read(*names, **kwargs):
         "atlite==0.2.5",
         "cdsapi",
         "click<8.1",
-        #"entsoe-py >=0.3.1",
+        "entsoe-py >=0.5.5",
         "GeoAlchemy2",
         "geopandas>=0.10.0",
         "geopy",

src/egon/data/datasets/electrical_neighbours.py

@@ -3,7 +3,7 @@
 
 import zipfile
 
-# import entsoe
+import entsoe
 import requests
 import logging
 
@@ -19,6 +19,7 @@
 from egon.data.datasets.fix_ehv_subnetworks import select_bus_id
 from egon.data.datasets.fill_etrago_gen import add_marginal_costs
 from egon.data.datasets.scenario_parameters import get_sector_parameters
+from os import path
 
 
 def get_cross_border_buses(scenario, sources):
@@ -1300,8 +1301,11 @@ def tyndp_demand():
 
 
 def entsoe_historic_generation_capacities(
-    entsoe_token=None, year_start="20190101", year_end="20200101"
+    year_start="20190101", year_end="20200101"
 ):
+    entsoe_token = open(
+        path.join(path.expanduser("~"), ".entsoe-token"), "r"
+    ).read(36)
     client = entsoe.EntsoePandasClient(api_key=entsoe_token)
 
     start = pd.Timestamp(year_start, tz="Europe/Brussels")
@@ -1343,7 +1347,7 @@ def entsoe_historic_generation_capacities(
             pass
 
     if not_retrieved:
-        logger.warning(
+        logging.warning(
             f"Data for country (-ies) {', '.join(not_retrieved)} could not be retrieved."
         )
     df = pd.concat(dfs)
@@ -1353,9 +1357,10 @@ def entsoe_historic_generation_capacities(
     return df
 
 
-def entsoe_historic_demand(
-    entsoe_token=None, year_start="20190101", year_end="20200101"
-):
+def entsoe_historic_demand(year_start="20190101", year_end="20200101"):
+    entsoe_token = open(
+        path.join(path.expanduser("~"), ".entsoe-token"), "r"
+    ).read(36)
     client = entsoe.EntsoePandasClient(api_key=entsoe_token)
 
     start = pd.Timestamp(year_start, tz="Europe/Brussels")
@@ -1404,7 +1409,7 @@ def entsoe_historic_demand(
             not_retrieved.append(country)
             pass
     if not_retrieved:
-        logger.warning(
+        logging.warning(
             f"Data for country (-ies) {', '.join(not_retrieved)} could not be retrieved."
         )
 
@@ -1470,7 +1475,7 @@ def entsoe_to_bus_etrago():
     return map_entsoe.map(for_bus)
 
 
-def insert_generators_sq(gen_sq=None, scn_name="status2019"):
+def insert_generators_sq(scn_name="status2019"):
     """
     Insert generators for foreign countries based on ENTSO-E data
 
@@ -1487,11 +1492,16 @@ def insert_generators_sq(gen_sq=None, scn_name="status2019"):
     None.
 
     """
-    ################# TEMPORAL ####################
-    gen_sq = pd.read_csv(
-        "data_bundle_powerd_data/entsoe/gen_entsoe.csv", index_col="Index"
-    )
-    ################# TEMPORAL ####################
+    try:
+        gen_sq = entsoe_historic_generation_capacities()
+    except:
+        logging.warning(
+            """Generation data from entsoe could not be retrieved.
+                        Backup data is used instead"""
+        )
+        gen_sq = pd.read_csv(
+            "data_bundle_powerd_data/entsoe/gen_entsoe.csv", index_col="Index"
+        )
 
     targets = config.datasets()["electrical_neighbours"]["targets"]
     # Delete existing data
@@ -1629,7 +1639,7 @@ def insert_generators_sq(gen_sq=None, scn_name="status2019"):
     return
 
 
-def insert_loads_sq(load_sq=None, scn_name="status2019"):
+def insert_loads_sq(scn_name="status2019"):
     """
     Copy load timeseries data from entso-e.
 
@@ -1640,12 +1650,16 @@ def insert_loads_sq(load_sq=None, scn_name="status2019"):
     """
     sources = config.datasets()["electrical_neighbours"]["sources"]
     targets = config.datasets()["electrical_neighbours"]["targets"]
-
-    ################# TEMPORAL ####################
-    load_sq = pd.read_csv(
-        "data_bundle_powerd_data/entsoe/load_entsoe.csv", index_col="Index"
-    )
-    ################# TEMPORAL ####################
+    try:
+        load_sq = entsoe_historic_demand()
+    except:
+        logging.warning(
+            """Demand data from entsoe could not be retrieved.
+                        Backup data is used instead"""
+        )
+        load_sq = pd.read_csv(
+            "data_bundle_powerd_data/entsoe/load_entsoe.csv", index_col="Index"
+        )
 
     # Delete existing data
     db.execute_sql(
@@ -1728,7 +1742,7 @@ class ElectricalNeighbours(Dataset):
     def __init__(self, dependencies):
         super().__init__(
             name="ElectricalNeighbours",
-            version="0.0.9",
+            version="0.0.10",
             dependencies=dependencies,
             tasks=tasks,
         )