Decouple core elements (#123)

* New utils progress * Deleted * Update checks for exposure decoupling * Update I/O * Further add exposure utils * Decouple integrated pieces of code * Added new submodule * Update submodule name * Add hidden import * Update build gdal version, pin numpy version * Quartodoc from conda * Update init file * Further separation of methods * Updated tests with refactoring * Update testdata * Removal op exposure table object, relying on tablelazy and workflow methods * Update example index string * type change * Some progress * Moved fields type map * Update GeomSource functionality * Add libgdal-netcdf as opt-dep * Update env build * Split geom drivers in read and write * Driver not found error fix * Fix map type * Update quarto to v1.5.x, add working dark image * Fix architecture error * Renamed 'math' submodule to 'methods' * Rename old geom file * Small progress * Chunk differently * Rename add_feature methods * First time running no csv workflow * Improved locking and chunking * Implemented risk, different exposure types * f*ck if I know * Removed unused object * Updated docs * Added reproject for hazard and exposure in gridmodel * Small docstring fix
Deltares · Oct 3, 2024 · 6f9b025 · 6f9b025
1 parent 2ad1350
commit 6f9b025
Show file tree

Hide file tree

Showing 57 changed files with 1,737 additions and 1,605 deletions.
diff --git a/.build/build.spec b/.build/build.spec
@@ -31,7 +31,7 @@ a = Analysis(
     pathex=[Path(build_dir), Path(project_root, "src")],
     binaries=[],
     datas=[],
-    hiddenimports=["fiat_build_time"],
+    hiddenimports=["fiat_build_time", "fiat.methods"],
     hookspath=[build_dir.as_posix()],
     hooksconfig={},
     runtime_hooks=[Path(build_dir, 'runtime_hooks.py')],

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -77,7 +77,7 @@ jobs:
           sudo apt install -y curl
           sudo apt install -y gdebi-core
           mkdir tmp
-          curl -L https://github.com/quarto-dev/quarto-cli/releases/download/v1.3.450/quarto-1.3.450-linux-amd64.deb --output tmp/quarto.deb
+          curl -L https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.57/quarto-1.5.57-linux-amd64.deb --output tmp/quarto.deb
           chmod +x tmp/quarto.deb
           sudo gdebi -n tmp/quarto.deb
 

diff --git a/.testdata/create_test_data.py b/.testdata/create_test_data.py
@@ -31,8 +31,8 @@ def create_dbase_stucture():
 def create_exposure_dbase():
     """_summary_."""
     with open(Path(p, "exposure", "spatial.csv"), "w") as f:
-        f.write("Object ID,Extraction Method,Ground Floor Height,Ground Elevation,")
-        f.write("Damage Function: Structure,Max Potential Damage: Structure\n")
+        f.write("object_id,extract_method,ground_flht,ground_elevtn,")
+        f.write("fn_damage_structure,max_damage_structure\n")
         for n in range(5):
             if (n + 1) % 2 != 0:
                 dmc = "struct_1"
@@ -63,13 +63,13 @@ def create_exposure_geoms():
     )
 
     field = ogr.FieldDefn(
-        "Object ID",
+        "object_id",
         ogr.OFTInteger,
     )
     layer.CreateField(field)
 
     field = ogr.FieldDefn(
-        "ObjectName",
+        "object_name",
         ogr.OFTString,
     )
     field.SetWidth(50)
@@ -78,8 +78,8 @@ def create_exposure_geoms():
     for idx, geom in enumerate(geoms):
         geom = ogr.CreateGeometryFromWkt(geom)
         ft = ogr.Feature(layer.GetLayerDefn())
-        ft.SetField("Object ID", idx + 1)
-        ft.SetField("ObjectName", f"fp_{idx+1}")
+        ft.SetField("object_id", idx + 1)
+        ft.SetField("object_name", f"fp_{idx+1}")
         ft.SetGeometry(geom)
 
         layer.CreateFeature(ft)
@@ -110,22 +110,22 @@ def create_exposure_geoms_2():
     )
 
     field = ogr.FieldDefn(
-        "Object ID",
+        "object_id",
         ogr.OFTInteger,
     )
     layer.CreateField(field)
 
     field = ogr.FieldDefn(
-        "ObjectName",
+        "object_name",
         ogr.OFTString,
     )
     field.SetWidth(50)
     layer.CreateField(field)
 
     geom = ogr.CreateGeometryFromWkt(geoms[0])
     ft = ogr.Feature(layer.GetLayerDefn())
-    ft.SetField("Object ID", 5)
-    ft.SetField("ObjectName", f"fp_{5}")
+    ft.SetField("object_id", 5)
+    ft.SetField("object_name", f"fp_{5}")
     ft.SetGeometry(geom)
 
     layer.CreateFeature(ft)
@@ -158,22 +158,22 @@ def create_exposure_geoms_3():
     )
 
     field = ogr.FieldDefn(
-        "Object ID",
+        "object_id",
         ogr.OFTInteger,
     )
     layer.CreateField(field)
 
     field = ogr.FieldDefn(
-        "ObjectName",
+        "object_name",
         ogr.OFTString,
     )
     field.SetWidth(50)
     layer.CreateField(field)
 
     geom = ogr.CreateGeometryFromWkt(geoms[0])
     ft = ogr.Feature(layer.GetLayerDefn())
-    ft.SetField("Object ID", 5)
-    ft.SetField("ObjectName", f"fp_{5}")
+    ft.SetField("object_id", 5)
+    ft.SetField("object_name", f"fp_{5}")
     ft.SetGeometry(geom)
 
     layer.CreateFeature(ft)
@@ -182,8 +182,8 @@ def create_exposure_geoms_3():
 
     geom = ogr.CreateGeometryFromWkt(geoms[1])
     ft = ogr.Feature(layer.GetLayerDefn())
-    ft.SetField("Object ID", 6)
-    ft.SetField("ObjectName", f"fp_{6}")
+    ft.SetField("object_id", 6)
+    ft.SetField("object_name", f"fp_{6}")
     ft.SetGeometry(geom)
 
     layer.CreateFeature(ft)
@@ -226,7 +226,7 @@ def create_exposure_grid():
     for x, y in product(oneD, oneD):
         data[x, y] = 2000 + ((x + y) * 100)
     band.WriteArray(data)
-    band.SetMetadataItem("damage_function", "struct_1")
+    band.SetMetadataItem("fn_damage", "struct_1")
 
     band.FlushCache()
     src.FlushCache()
@@ -325,7 +325,6 @@ def create_settings_geom():
     doc = {
         "global": {
             "crs": "EPSG:4326",
-            "keep_temp_files": True,
         },
         "output": {
             "path": "output/geom_event",
@@ -403,7 +402,6 @@ def create_settings_grid():
     doc = {
         "global": {
             "crs": "EPSG:4326",
-            "keep_temp_files": True,
         },
         "output": {
             "path": "output/grid_event",

diff --git a/docs/PDF_Documentation.qmd b/docs/PDF_Documentation.qmd
@@ -218,7 +218,7 @@ damages per asset
 [**exposure.geom**]
 `csv`: This will create an exposure CSV file within the exposure folder ([folder strucute](index.qmd)) that contains the [required information](exposure.qmd) per asset.
 `crs`: The projection of the exposure vector file.
-`file1`: This will create an exposure vector file within the exposure folder with the assets'geometry and Object ID.
+`file1`: This will create an exposure vector file within the exposure folder with the assets'geometry and object_id.
 
 [**vulnerability**]
 `file`: This will create an vulnerability curves CSV file within the vulnerability folder ([folder strucute](index.qmd)) that contains the damage curves.
@@ -254,9 +254,9 @@ If the user prefers to create a more advanced model or to utilize grid data inst
 | var_ as_band                 | True/False Read netCDF subdatasets as raster bands                                                                                                                           | No                                  | False                                     |
 | **[exposure.geom]**          |                                                                                                                                                                              |                                     |                                           |
 | csv                          | File path to exposure.csv file, that contains information about e.g. asset type, max. potential damage, aggregation and so forth)                                            | Yes                                 | 'exposure/exposure.csv'                   |
-| file1                        | File path of exposure vector file with Object ID column to enable linking exposure.csv output to vector file.                                                                | Yes                                 | 'exposure/buildings.gpkg'                 |
+| file1                        | File path of exposure vector file with object_id column to enable linking exposure.csv output to vector file.                                                                | Yes                                 | 'exposure/buildings.gpkg'                 |
 | crs                          | Projection of exposure data                                                                                                                                                  | Yes, if crs is unknown in dataset   | 'EPSG:32617'                              |
-| index                        | Define the name of the index column of the data to link the exposure CSV file with the exposure vector file.                                                                 | No                                  | 'Object ID'                               |
+| index                        | Define the name of the index column of the data to link the exposure CSV file with the exposure vector file.                                                                 | No                                  | 'object_id'                               |
 | **[exposure.grid]**          |                                                                                                                                                                              |                                     |                                           |
 | file                         | File path to exposure.nc grid file, that contains the spatial information and information about the maximum potential damage per cell.                                       | Yes, if netCDF is provided as input | 'exposure/raster.nc'                      |
 | crs                          | Output projection                                                                                                                                                            | Yes, if crs is unknown in dataset   | 'EPSG:32617'                              |
@@ -324,7 +324,7 @@ For users who would want to create their own exposure data, or modify existing e
 
 | Field                                  | Description                                                                                | Required                                 | Example                    |
 |----------------------------------------|--------------------------------------------------------------------------------------------|------------------------------------------|----------------------------|
-| Object ID                              | Unique numerical indentifier of the object                                                 | Yes                                      | 1                          |
+| object_id                              | Unique numerical indentifier of the object                                                 | Yes                                      | 1                          |
 | Object Name                            | Unique name of the object                                                                  | No                                       | fp_1                       |
 | Primary Object Type                    | Object type                                                                                | No                                       | RES1_1SNB                  |
 | Secondary Object Type                  | More specification about object type                                                       | No                                       | Res 1,1 Story no basement  |
@@ -342,8 +342,8 @@ For users who would want to create their own exposure data, or modify existing e
 
 A more detailed description of the data fields in the *exposure.csv* can be found below;
 
- **Object ID/Object name**
- Object ID and Object name are administrative information, which the user is free to choose. Input must be unique for each object, if they are not unique, FIAT gives a warning and stops the model built-up.
+ **object_id/Object name**
+ object_id and Object name are administrative information, which the user is free to choose. Input must be unique for each object, if they are not unique, FIAT gives a warning and stops the model built-up.
 
 **Primary/Secondary object type**
 The primary object type describes the category of the asset (e.g. residential or commercial). The secondary object type allows for a more detailed profile of the object (e.g. single-story home, or grocery store). The developer of the exposure dataset is free to set their own categories of object types. (*Exception: FIAT requires **roads** to be assigned as **primary object type = ‘road**’, to summarize road damages separately from buildings and utilities*.)

diff --git a/docs/_quarto.yml b/docs/_quarto.yml
@@ -36,13 +36,20 @@ website:
   page-footer:
     left: |
       Made possible by:
-        <a href="https://www.deltares.nl/en" target="_blank" rel="noreferrer noopener">
-          <img
-            src="/_static/images/deltares-blue.svg"
-            alt="Deltares"
-            style="height: 40px;"
-          />
-        </a>
+       <a href="https://www.deltares.nl/en" target="_blank" rel="noreferrer noopener">
+        <img
+          src="/_static/images/deltares-blue.svg"
+          alt="Deltares"
+          class="footer-image-light"
+          style="height: 40px;"
+        />
+        <img
+          src="/_static/images/deltares-white.svg"
+          alt="Deltares"
+          class="footer-image-dark"
+          style="height: 40px;"
+        />
+      </a>
   navbar:
     logo: _static/fiat.svg
     search: true
@@ -86,9 +93,9 @@ website:
             - "setup_guide/general/docker.qmd"
         - section: "FIAT package"
           contents:
-            - setup_guide/kernel/installation.qmd
-            - setup_guide/kernel/application.qmd
-            - setup_guide/kernel/linux.qmd
+            - setup_guide/kernel/install.qmd
+            - setup_guide/kernel/dev.qmd
+            - setup_guide/kernel/build.qmd
     - title: "User guide"
       collapse-level: 1
       contents:
@@ -108,8 +115,9 @@ website:
             - text: Exposure data
               file: user_guide/data/exposure.qmd
               contents:
-                - user_guide/data/exposure/csv.qmd
+                - user_guide/data/exposure/data.qmd
                 - user_guide/data/exposure/geometries.qmd
+                - user_guide/data/exposure/csv.qmd
             - user_guide/data/vulnerability.qmd
             - user_guide/data/supported.qmd
         - title: Examples
@@ -176,6 +184,14 @@ quartodoc:
           children: separate
         - name: GridModel
           children: separate
+    - subtitle: Methods
+      desc: The hazard functions
+      package: fiat.methods
+      contents:
+        - ead.risk_density
+        - ead.calc_ead
+        - flood.calculate_hazard
+        - flood.calculate_damage
 
     # Logging
     - title: Logging

diff --git a/docs/_static/theme-dark.scss b/docs/_static/theme-dark.scss
@@ -16,6 +16,8 @@ $navbar-hl: $links ;
 
 // Code blocks
 $code-block-bg-alpha: -.8;
+$code-bg: $highlight;
+$code-color: $text;
 
 .navbar-nav .dropdown-menu {
     background-color: $highlight; // Dark mode dropdown background color
@@ -68,3 +70,18 @@ $code-block-bg-alpha: -.8;
 //     border-color: rgb(255, 72, 0);
 //     // border-width: 1px;
 // }
+
+// :root {
+//   --footer-image: url('/_static/images/deltares-white.svg');
+// }
+//
+// footer img {
+//   content: var(--footer-image);
+// }
+
+.footer-image-light {
+  display: none;
+}
+.footer-image-dark {
+  display: inline;
+}
diff --git a/docs/_static/theme-light.scss b/docs/_static/theme-light.scss
@@ -23,3 +23,10 @@ $navbar-hl: $links ;
     border-color: $links;
     border: 1px solid $links;
 }
+
+.footer-image-dark {
+  display: none;
+}
+.footer-image-light {
+  display: inline;
+}
diff --git a/docs/changelog.qmd b/docs/changelog.qmd
@@ -6,38 +6,59 @@ title: "What's new?"
 This contains the unreleased changes to Delft-FIAT.
 
 ### Added
+- Attribute `size` of `GeomSource` object for in situ size return; `count` becomes private
+- Attribute `size` of `GridSource` object for in situ size return; `count` becomes private
+- Attributes `dtypes` and `geom_type` of `GeomModel`
 - Build time to FIAT cli (when build with pyinstaller), viewed with `fiat --version`
+- Different types of exposure (e.g. 'damage', 'affected', 'outage' etc.)
 - Docker file for docker image creation
+- Extra arguments to `grid.reproject`
 - Function (`generate_jobs`) to generate jobs for parallelization
 - Function (`execute_pool`) to execute code in parallel using `multiprocessing`
+- Flood hazard/ damage functions (`methods` submodule)
 - General method of creating output files in `GeomModel` (`_setup_output_files`)
 - Method `_create_model_dirs` of `ConfigReader` object for creating result directories in one go
 - Method `add_handler` of the `Log` object; user setting of custom stream
+- Method `add_feature_with_map` of `GeomModel` to set features with extra info
+- Method `create` of `GeomModel` to create an ogr.DataSource (new dataset)
+- Method `create_equal_grids` of `GridModel` for making hazard and exposure grid spatially equal
 - Method `set` of `ConfigReader` object
-- Method `size` of `GeomSource` object for in situ size return; `count` becomes private
-- Method `size` of `GridSource` object for in situ size return; `count` becomes private
+- Not stopping when exposure and hazard grids (`GridModel`), but instead make them spatially equal
 - Numpy >= 2.0.0 support
 - Python 3.12 support
-- Settings toml file: global setting 'loglevel'
+- Settings toml file: global setting 'global.loglevel'; default 'INFO'
+- Settings toml file: exposure setting 'exposure.types'; default 'flood'
 - Setting return period as a variable in hazard map bands (risk)
 - Support for using pixi for binary creation (properly)
 
 ### Changed
 - Better version of `BufferHandler`
+- Exposure data headers are now lower-/ snakecase ('object_id' -> 'objectId'), see [docs](./user_guide/data/exposure/data.qmd)
 - Fixed binary creation in general, but also specifically for `GDAL >= v3.9.1`
 - Made read methods of `BaseModel`, `GeomModel` and `GridModel` public (removed underscore)
+- Made csv files (exposure data) optional
+- Moved hazard/ damage calculation function to `methods` submodule
+- Proper checking for duplicate columns in csv files
+- Settings toml file: exposure setting 'exposure.csv.file' (becomes optional)
 - Testing of workers (not properly caught due to using `multiprocessing`)
 - Testing only based on integers
 
 ### Deprecated
 - Base object `_BaseHandler`; incompatible with Python 3.12
+- Function `open_exp` from `fiat.io`, superseded by general use of `open_csv`
 - Method `add_c_handler` in favour of the more generalized `add_handler`
 - Methods `_create_output_dir`, `_create_tmp_dir` and `_create_risk_dir` of the `ConfigReader` object
+- Object `ExposureTable`, now done via `TableLazy`
+- Resolve stage of `GeomModel`; now properly handled in `GeomModel.run`
 - Setting return period via the name of the hazard band (risk)
+- Settings toml file: global setting 'global.keep_temp_files'
+- Settings toml file: output setting 'output.geom.chunk'; superseded by 'global.geom.chunk'
 - Support of `Python` versions under `3.9.0`
+- Temporary files (`GeomModel`)
 - `TextHandler` object; unused
 
 ### Documentation
+- Added methods to the api
 - Cleaner home page
 - Getting started remade into `Information`
 

diff --git a/docs/examples/single_event.ipynb b/docs/examples/single_event.ipynb
@@ -79,7 +79,7 @@
     "from pathlib import Path\n",
     "\n",
     "# check the output\n",
-    "out = open_csv(Path(\"../../.testdata/output/geom_event\", \"output.csv\"), index=\"Object ID\")\n",
+    "out = open_csv(Path(\"../../.testdata/output/geom_event\", \"output.csv\"), index=\"object_id\")\n",
     "print(out.columns)"
    ]
   },
@@ -98,8 +98,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert float(out[2, \"Total Damage\"]) == 740\n",
-    "assert float(out[3, \"Total Damage\"]) == 1038"
+    "assert float(out[2, \"total_damage\"]) == 740\n",
+    "assert float(out[3, \"total_damage\"]) == 1038"
    ]
   }
  ],
@@ -119,7 +119,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.12.4"
   }
  },
  "nbformat": 4,

diff --git a/docs/setup_guide/general/conda.qmd b/docs/setup_guide/general/conda.qmd
@@ -6,7 +6,8 @@ In order to develop on **FIAT** locally, the Python package manager **Miniforge3
 
 Download and install [Miniforge3](https://github.com/conda-forge/miniforge#mambaforge)
 
-Initialize conda by running the following in the Miniforge prompt:
+Make sure the conda binary (and mamba) is added to PATH. In windows this is simply done via the 'set environment variables' screen, on linux one can append the 'PATH' variable via the `.bashrc` configurations file (or another rc file corresponding with the shell in use).
+Initialize conda by running the following command in your shell.
 
 ```bash
 conda init