Merge pull request #352 from openego/dev

Release v0.2.0

.gitignore

@@ -22,6 +22,8 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+doc/_html/
+.idea/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -56,6 +58,9 @@ doc/_build/
 # PyBuilder
 target/
 
+# Data
+.pkl
+
 
 
 

.readthedocs.yml

@@ -18,6 +18,6 @@ formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.6
+  version: 3.8
   install:
     - requirements: rtd_requirements.txt

.travis.yml

@@ -1,12 +1,12 @@
+dist: bionic
 language: python
 python:
-  - "3.4"
-  - "3.5"
-  - "3.6"
+  - "3.6.10"
   # - "3.7"
 # command to install dependencies
 install:
-  - pip install -r requirements.txt
+  - pip install -r requirements.txt -r dev_requirements.txt
+  - pip freeze
 # command to run tests
 script:
   - pytest -vv

README.md

@@ -7,15 +7,16 @@
 DING0
 =====
 DIstribution Network GeneratOr -- A tool to generate synthetic medium and low
-voltage power distribution grids based on open (or at least accessible) data.
+voltage power distribution grids based on open (or at least publicly available)
+data.
 
 Installation and usage
 ----------------------
 
 Ding0 is provided through the package management tool *pip*. Install it by
 
 ```
-pip3 install ding0
+pip install ding0
 ```
 
 Further details regarding the installation including dependencies are provided
@@ -24,12 +25,13 @@ in the [documentation](https://dingo.readthedocs.io)
 A [set of examples](https://dingo.readthedocs.io/en/dev/usage_details.html#examples)
 is provided to show how to use Ding0.
 
-An overview on versions of data already generated [can be found in the wiki](https://github.com/openego/ding0/wiki/Ding0-datasets).
+An overview on versions of data already generated
+[can be found in the wiki](https://github.com/openego/ding0/wiki/Ding0-datasets).
 
 LICENSE
 -------
 
-Copyright (C) 2017 openego project group
+Copyright (C) 2017-2021 openego project group
 
 This program is free software: you can redistribute it and/or modify it under
 the terms of the GNU Affero General Public License as published by the Free

dev_requirements.txt

@@ -0,0 +1,5 @@
+pytest
+pytest-dependency
+pytest-parallel
+sphinx
+sphinx_rtd_theme

ding0/core/__init__.py

@@ -22,10 +22,12 @@
 from ding0.core.network.stations import *
 from ding0.core.structure.regions import *
 from ding0.core.powerflow import *
-from ding0.tools import pypsa_io
+from ding0.tools.pypsa_io import initialize_component_dataframes, fill_mvgd_component_dataframes
 from ding0.tools.animation import AnimationDing0
 from ding0.tools.plots import plot_mv_topology
 from ding0.flexopt.reinforce_grid import *
+from ding0.tools.logger import get_default_home_dir
+from ding0.tools.tools import merge_two_dicts_of_dataframes
 
 import os
 import logging
@@ -38,6 +40,7 @@
 from sqlalchemy import func
 from geoalchemy2.shape import from_shape
 import subprocess
+import json
 import oedialect
 
 if not 'READTHEDOCS' in os.environ:
@@ -266,33 +269,28 @@ def run_ding0(self, session, mv_grid_districts_no=None, debug=False, export_figu
             Generators are connected to grids, used approach depends on voltage
             level.
 
-        * STEP 8: Set IDs for all branches in MV and LV grids
-
-            While IDs of imported objects can be derived from dataset's ID, branches
-            are created in steps 5+6 and need unique IDs (e.g. for PF calculation).
-
-        * STEP 9: Relocate switch disconnectors in MV grid
+        * STEP 8: Relocate switch disconnectors in MV grid
 
             Switch disconnectors are set during routing process (step 6) according
             to the load distribution within a ring. After further modifications of
             the grid within step 6+7 they have to be relocated (note: switch
             disconnectors are called circuit breakers in DING0 for historical reasons).
 
-        * STEP 10: Open all switch disconnectors in MV grid
+        * STEP 9: Open all switch disconnectors in MV grid
 
             Under normal conditions, rings are operated in open state (half-rings).
             Furthermore, this is required to allow powerflow for MV grid.
 
-        * STEP 11: Do power flow analysis of MV grid
+        * STEP 10: Do power flow analysis of MV grid
 
             The technically working MV grid created in step 6 was extended by satellite
             loads and generators. It is finally tested again using powerflow calculation.
 
-        * STEP 12: Reinforce MV grid
+        * STEP 11: Reinforce MV grid
 
             MV grid is eventually reinforced persuant to results from step 11.
 
-        STEP 13: Close all switch disconnectors in MV grid
+        * STEP 12: Close all switch disconnectors in MV grid
             The rings are finally closed to hold a complete graph (if the SDs are open,
             the edges adjacent to a SD will not be exported!)
         """
@@ -326,18 +324,15 @@ def run_ding0(self, session, mv_grid_districts_no=None, debug=False, export_figu
         if export_figures:
             plot_mv_topology(grid, subtitle='Generators connected', filename='2_generators_connected.png')
 
-        # STEP 8: Set IDs for all branches in MV and LV grids
-        self.set_branch_ids()
-
-        # STEP 9: Relocate switch disconnectors in MV grid
+        # STEP 8: Relocate switch disconnectors in MV grid
         self.set_circuit_breakers(debug=debug)
         if export_figures:
             plot_mv_topology(grid, subtitle='Circuit breakers relocated', filename='3_circuit_breakers_relocated.png')
 
-        # STEP 10: Open all switch disconnectors in MV grid
+        # STEP 9: Open all switch disconnectors in MV grid
         self.control_circuit_breakers(mode='open')
 
-        # STEP 11: Do power flow analysis of MV grid
+        # STEP 10: Do power flow analysis of MV grid
         self.run_powerflow(session, method='onthefly', export_pypsa=False, debug=debug)
         if export_figures:
             plot_mv_topology(grid, subtitle='PF result (load case)',
@@ -347,10 +342,10 @@ def run_ding0(self, session, mv_grid_districts_no=None, debug=False, export_figu
                              filename='5_PF_result_feedin.png',
                              line_color='loading', node_color='voltage', testcase='feedin')
 
-        # STEP 12: Reinforce MV grid
+        # STEP 11: Reinforce MV grid
         self.reinforce_grid()
 
-        # STEP 13: Close all switch disconnectors in MV grid
+        # STEP 12: Close all switch disconnectors in MV grid
         self.control_circuit_breakers(mode='close')
 
         if export_figures:
@@ -979,7 +974,7 @@ def import_res_generators():
                     generator = GeneratorFluctuatingDing0(
                         id_db=id_db,
                         mv_grid=mv_grid,
-                        capacity=row['electrical_capacity'],
+                        capacity=float(row['electrical_capacity']),
                         type=row['generation_type'],
                         subtype=row['generation_subtype'],
                         v_level=int(row['voltage_level']),
@@ -988,7 +983,7 @@ def import_res_generators():
                     generator = GeneratorDing0(
                         id_db=id_db,
                         mv_grid=mv_grid,
-                        capacity=row['electrical_capacity'],
+                        capacity=float(row['electrical_capacity']),
                         type=row['generation_type'],
                         subtype=row['generation_subtype'],
                         v_level=int(row['voltage_level']))
@@ -1349,7 +1344,7 @@ def validate_grid_districts(self):
         for grid_district in self.mv_grid_districts():
 
             # there's only one node (MV station) => grid is empty
-            if len(grid_district.mv_grid._graph.nodes()) == 1:
+            if len(grid_district.mv_grid.graph.nodes()) == 1:
                 invalid_mv_grid_districts.append(grid_district)
                 msg_invalidity.append('MV Grid District {} seems to be empty ' \
                                       'and ' \
@@ -1412,7 +1407,7 @@ def export_mv_grid(self, session, mv_grid_districts):
             lines = []
 
             # get nodes from grid's graph and append to corresponding array
-            for node in grid_district.mv_grid._graph.nodes():
+            for node in grid_district.mv_grid.graph.nodes():
                 if isinstance(node, LVLoadAreaCentreDing0):
                     lv_load_area_centres.append((node.geo_data.x, node.geo_data.y))
                 elif isinstance(node, MVCableDistributorDing0):
@@ -1527,7 +1522,7 @@ def export_mv_grid_new(self, session, mv_grid_districts):
         for grid_district in self.mv_grid_districts():
 
             # get nodes from grid's graph and create datasets
-            for node in grid_district.mv_grid._graph.nodes():
+            for node in grid_district.mv_grid.graph.nodes():
                 if hasattr(node, 'voltage_res'):
                     node_name = '_'.join(['MV',
                                           str(grid_district.mv_grid.id_db),
@@ -1610,8 +1605,9 @@ def export_mv_grid_new(self, session, mv_grid_districts):
 
         logger.info('=====> MV Grids exported (NEW)')
 
-    def to_dataframe(self):
+    def to_dataframe_old(self):
         """
+        Todo: remove? or replace by part of to_csv()
         Export grid data to dataframes for statistical analysis.
 
         The export to dataframe is similar to db tables exported by `export_mv_grid_new`.
@@ -1728,6 +1724,95 @@ def to_dataframe(self):
 
         return nodes_df, edges_df
 
+    def to_csv(self, dir = '', only_export_mv = False):
+        '''
+        Function to export network to csv. Converts network in dataframes which are adapted to pypsa format.
+        Respectively saves files for network, buses, lines, transformers, loads and generators.
+
+        Parameters
+        ----------
+        dir: :obj:`str`
+            Directory to which network is saved.
+        only_export_mv: bool
+            When True only mv topology is exported with aggregated lv grid districts
+        '''
+
+        buses_df, generators_df, lines_df, loads_df, transformer_df = initialize_component_dataframes()
+        if (dir == ''):
+            dir = get_default_home_dir()  # eventuell ändern
+        # open all switch connectors
+        self.control_circuit_breakers(mode='open')
+        # start filling component dataframes
+        for grid_district in self.mv_grid_districts():
+            gd_components, network_df, _ = fill_mvgd_component_dataframes(
+                grid_district, buses_df, generators_df,
+                lines_df, loads_df,transformer_df, only_export_mv)
+            # save network and components to csv
+            path = os.path.join(dir, str(grid_district.id_db))
+            if not os.path.exists(path):
+                os.makedirs(path)
+            network_df.to_csv(os.path.join(path, 'network.csv'))
+            gd_components['HVMV_Transformer'].to_csv(
+                os.path.join(path, 'transformers_hvmv.csv'))
+            gd_components['Transformer'].to_csv(
+                os.path.join(path, 'transformers.csv'))
+            gd_components['Bus'].to_csv(
+                os.path.join(path, 'buses.csv'))
+            gd_components['Line'].to_csv(
+                os.path.join(path, 'lines.csv'))
+            gd_components['Load'].to_csv(
+                os.path.join(path, 'loads.csv'))
+            gd_components['Generator'].to_csv(
+                os.path.join(path, 'generators.csv'))
+            gd_components['Switch'].to_csv(
+                os.path.join(path, 'switches.csv'))
+
+            # Merge metadata of multiple runs
+            if 'metadata' not in locals():
+                metadata = self.metadata
+
+            else:
+                if isinstance(grid_district, list):
+                    metadata['mv_grid_districts'].extend(grid_district)
+                else:
+                    metadata['mv_grid_districts'].append(grid_district)
+
+            # Save metadata to disk
+        with open(os.path.join(path, 'Ding0_{}.meta'.format(metadata['run_id'])),
+                  'w') as f:
+            json.dump(metadata, f)
+
+
+    def to_dataframe(self,  only_export_mv = False):
+        '''
+        Function to export network to csv. Converts network in dataframes which are adapted to pypsa format.
+        Respectively saves files for network, buses, lines, transformers, loads and generators.
+
+        Parameters
+        ----------
+        only_export_mv: bool
+            When True only mv topology is exported with aggregated lv grid districts
+        '''
+        buses_df, generators_df, lines_df, loads_df, transformer_df = initialize_component_dataframes()
+        components = {}
+        networks = pd.DataFrame()
+        # open all switch connectors
+        self.control_circuit_breakers(mode='open')
+        # start filling component dataframes
+        for grid_district in self.mv_grid_districts():
+            gd_components, network_df, _ = fill_mvgd_component_dataframes(grid_district, buses_df, generators_df,
+                                                                            lines_df, loads_df, transformer_df, only_export_mv)
+            if len(components) == 0:
+                components = gd_components
+                networks = network_df
+            else:
+                components = merge_two_dicts_of_dataframes(components, gd_components)
+                networks = networks.append(network_df)
+
+        components['Network'] = network_df
+        return components
+
+
     def mv_routing(self, debug=False, animation=False):
         """
         Performs routing on all MV grids.
@@ -1825,21 +1910,6 @@ def mv_parametrize_grid(self, debug=False):
 
         logger.info('=====> MV Grids parametrized')
 
-    def set_branch_ids(self):
-        """
-        Performs generation and setting of ids
-        of branches for all MV and underlying LV grids.
-
-        See Also
-        --------
-        ding0.core.network.grids.MVGridDing0.set_branch_ids
-        """
-
-        for grid_district in self.mv_grid_districts():
-            grid_district.mv_grid.set_branch_ids()
-
-        logger.info('=====> Branch IDs set')
-
     def set_circuit_breakers(self, debug=False):
         """
         Calculates the optimal position of the existing circuit breakers
@@ -1884,7 +1954,7 @@ def control_circuit_breakers(self, mode=None):
         elif mode == 'close':
             logger.info('=====> MV Circuit Breakers closed')
 
-    def run_powerflow(self, session, method='onthefly', export_pypsa=False, debug=False):
+    def run_powerflow(self, session = None, method='onthefly', only_calc_mv = True, export_pypsa=False, debug=False, export_result_dir=None):
         """
         Performs power flow calculation for all MV grids
 
@@ -1915,20 +1985,22 @@ def run_powerflow(self, session, method='onthefly', export_pypsa=False, debug=Fa
                     export_pypsa_dir = repr(grid_district.mv_grid)
                 else:
                     export_pypsa_dir = None
-                grid_district.mv_grid.run_powerflow(session, method='db',
+                grid_district.mv_grid.run_powerflow(method='db',
                                                     export_pypsa_dir=export_pypsa_dir,
-                                                    debug=debug)
+                                                    debug=debug,
+                                                    export_result_dir=export_result_dir)
 
         elif method == 'onthefly':
             for grid_district in self.mv_grid_districts():
                 if export_pypsa:
                     export_pypsa_dir = repr(grid_district.mv_grid)
                 else:
                     export_pypsa_dir = None
-                grid_district.mv_grid.run_powerflow(session,
-                                                    method='onthefly',
+                grid_district.mv_grid.run_powerflow(method='onthefly',
+                                                    only_calc_mv = only_calc_mv,
                                                     export_pypsa_dir=export_pypsa_dir,
-                                                    debug=debug)
+                                                    debug=debug,
+                                                    export_result_dir=export_result_dir)
 
     def reinforce_grid(self):
         """