Fix/integrate roofer

docker/.env

@@ -6,6 +6,8 @@ DAGSTER_POSTGRES_DB="postgres_db"
 TOOLS_DIR=/opt/3dbag-pipeline/tools
 
 BAG3D_VENVS=/3dbag-pipeline/venvs
+BAG3D_FILESTORE=/data/volume
+BAG3D_RELEASE_VERSION=develop
 BAG3D_TEST_DATA=/data/volume
 BAG3D_FLOORS_ESTIMATION_MODEL=/data/volume/model/pipeline_model1_gbr_untuned.joblib
 BAG3D_RELEASE_VERSION=test_version
@@ -27,6 +29,7 @@ TYLER_METADATA_JSON=/opt/3dbag-pipeline/tools/share/tyler/resources/geof/metadat
 EXE_PATH_TYLER=/opt/3dbag-pipeline/tools/bin/tyler
 EXE_PATH_TYLER_DB=/opt/3dbag-pipeline/tools/bin/tyler-db
 EXE_PATH_ROOFER_CROP=/opt/3dbag-pipeline/tools/bin/crop
+EXE_PATH_ROOFER_ROOFER=/opt/3dbag-pipeline/tools/bin/roofer
 EXE_PATH_ROOFER_RECONSTRUCT=/opt/3dbag-pipeline/tools/bin/geof
 FLOWCHART_PATH_RECONSTRUCT=/opt/3dbag-pipeline/tools/share/geoflow-bundle/flowcharts/reconstruct_bag.json
 EXE_PATH_OGR2OGR=/opt/3dbag-pipeline/tools/bin/ogr2ogr

docker/compose.yaml

@@ -11,6 +11,8 @@ services:
       POSTGRES_DB: $DAGSTER_POSTGRES_DB
     networks:
       - bag3d-dev-network
+    volumes:
+      - bag3d-dev-dagster-postgresql:/var/lib/postgresql/data
 
   # This service stores the asset data that is generated by the pipeline
   data-postgresql:
@@ -76,15 +78,21 @@ services:
       data-postgresql:
         condition: service_healthy
         restart: true
+    # Normally we wouldn't need watch:action:rebuild, but because we are using
+    # dagster's DockerRunLauncher, it always starts a new container from the image, for
+    # each run. While, watch:action:sync (or sync+restart)copies the changed local code into the
+    # 3dbag-pipeline-core-develop container (not the image). But since DockerRunLauncher
+    # doesn't actaully use this container, but always starts a new one, we have to build
+    # a new image on each change.
     develop:
       watch:
-        - action: sync
+        - action: rebuild
           path: ../packages/common
           target: /opt/3dbag-pipeline/packages/common
         - action: rebuild
           path: ../packages/common/pyproject.toml
           target: /opt/3dbag-pipeline/packages/common/pyproject.toml
-        - action: sync
+        - action: rebuild
           path: ../packages/core
           target: /opt/3dbag-pipeline/packages/core
         - action: rebuild
@@ -115,13 +123,13 @@ services:
         restart: true
     develop:
       watch:
-        - action: sync
+        - action: rebuild
           path: ../packages/common
           target: /opt/3dbag-pipeline/packages/common
         - action: rebuild
           path: ../packages/common/pyproject.toml
           target: /opt/3dbag-pipeline/packages/common/pyproject.toml
-        - action: sync
+        - action: rebuild
           path: ../packages/floors_estimation
           target: /opt/3dbag-pipeline/packages/floors_estimation
         - action: rebuild
@@ -151,13 +159,13 @@ services:
         restart: true
     develop:
       watch:
-        - action: sync
+        - action: rebuild
           path: ../packages/common
           target: /opt/3dbag-pipeline/packages/common
         - action: rebuild
           path: ../packages/common/pyproject.toml
           target: /opt/3dbag-pipeline/packages/common/pyproject.toml
-        - action: sync
+        - action: rebuild
           path: ../packages/party_walls
           target: /opt/3dbag-pipeline/packages/party_walls
         - action: rebuild
@@ -228,6 +236,8 @@ volumes:
     external: true
   bag3d-dev-data-pipeline:
     external: true
+  bag3d-dev-dagster-postgresql:
+    external: true
   bag3d-dev-dagster-home:
     external: true
 

docker/pipeline/bag3d-core.dockerfile

@@ -1,11 +1,11 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.10.31 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.11.09 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"
 LABEL org.opencontainers.image.vendor="3DBAG"
 LABEL org.opencontainers.image.title="3dbag-pipeline-core"
 LABEL org.opencontainers.image.description="The core workflow package of the 3dbag-pipeline. Image for building the pipeline packages."
 LABEL org.opencontainers.image.version=$VERSION
 LABEL org.opencontainers.image.licenses="(MIT OR Apache-2.0)"
 
 WORKDIR $BAG3D_PIPELINE_LOCATION

docker/pipeline/bag3d-floors-estimation.dockerfile

@@ -1,11 +1,11 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.10.31 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.11.09 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"
 LABEL org.opencontainers.image.vendor="3DBAG"
 LABEL org.opencontainers.image.title="3dbag-pipeline-floors-estimation"
 LABEL org.opencontainers.image.description="The floors_estimation workflow package of the 3dbag-pipeline. Image for building the pipeline packages."
 LABEL org.opencontainers.image.version=$VERSION
 LABEL org.opencontainers.image.licenses="(MIT OR Apache-2.0)"
 
 WORKDIR $BAG3D_PIPELINE_LOCATION

docker/pipeline/bag3d-party-walls.dockerfile

@@ -1,11 +1,11 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.10.31 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.11.09 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"
 LABEL org.opencontainers.image.vendor="3DBAG"
 LABEL org.opencontainers.image.title="3dbag-pipeline-party-walls"
 LABEL org.opencontainers.image.description="The party_walls workflow package of the 3dbag-pipeline. Image for building the pipeline packages."
 LABEL org.opencontainers.image.version=$VERSION
 LABEL org.opencontainers.image.licenses="(MIT OR Apache-2.0)"
 
 RUN rm -rf $VIRTUAL_ENV

makefile

@@ -25,11 +25,13 @@ docker_volume_create:
 	docker cp docker/dagster/dagster.yaml bag3d-dev-temp-container:/opt/dagster/dagster_home/
 	docker cp docker/dagster/workspace.yaml bag3d-dev-temp-container:/opt/dagster/dagster_home/
 	docker rm -f bag3d-dev-temp-container
+	docker volume create bag3d-dev-dagster-postgresql
 
 docker_volume_rm:
 	docker volume rm -f bag3d-dev-data-pipeline
 	docker volume rm -f bag3d-dev-data-postgresql
 	docker volume rm -f bag3d-dev-dagster-home
+	docker volume rm -f bag3d-dev-dagster-postgresql
 
 docker_volume_recreate: docker_volume_rm docker_volume_create
 

packages/common/src/bag3d/common/resources/__init__.py

@@ -40,10 +40,10 @@
 
 
 file_store = FileStoreResource(
-    data_dir="/tmp", dir_id=os.getenv("BAG3D_RELEASE_VERSION")
+    data_dir=os.getenv("BAG3D_FILESTORE"), dir_id=os.getenv("BAG3D_RELEASE_VERSION")
 )
 file_store_fastssd = FileStoreResource(
-    data_dir="/tmp",
+    data_dir=os.getenv("BAG3D_FILESTORE"),
     dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
 )
 
@@ -62,7 +62,10 @@
     exe_tyler=os.getenv("EXE_PATH_TYLER"), exe_tyler_db=os.getenv("EXE_PATH_TYLER_DB")
 )
 
-roofer = RooferResource(exe_roofer_crop=os.getenv("EXE_PATH_ROOFER_CROP"))
+roofer = RooferResource(
+    exe_crop=os.getenv("EXE_PATH_ROOFER_CROP"),
+    exe_roofer=os.getenv("EXE_PATH_ROOFER_ROOFER"),
+)
 
 geoflow = GeoflowResource(
     exe_geoflow=os.getenv("EXE_PATH_ROOFER_RECONSTRUCT"),

packages/common/src/bag3d/common/resources/executables.py

@@ -435,23 +435,28 @@ def app(self) -> AppImage:
 class RooferResource(ConfigurableResource):
     """
     A RooferResource can be configured by providing the paths to
-    Roofer `crop` executable on the local system.
+    Roofer `crop` and `roofer` executables on the local system.
 
     Example:
 
-        roofer_resource = RooferResource(exe_roofer_crop=os.getenv("EXE_PATH_ROOFER_CROP"))
+        roofer_resource = RooferResource(exe_crop=os.getenv("EXE_PATH_ROOFER_CROP"),
+                                         exe_roofer=os.getenv("EXE_PATH_ROOFER_ROOFER"))
 
     After the resource has been instantiated, roofer (AppImage) can
     be acquired with the `app` property:
 
         roofer = roofer_resource.app
     """
 
-    exe_roofer_crop: str
+    exe_crop: str
+    exe_roofer: str
 
     @property
     def exes(self) -> Dict[str, str]:
-        return {"crop": self.exe_roofer_crop}
+        return {
+            "crop": self.exe_crop,
+            "roofer": self.exe_roofer,
+        }
 
     @property
     def with_docker(self) -> bool:

packages/common/src/bag3d/common/utils/geodata.py

@@ -216,6 +216,7 @@ def ogr2postgres(
             "-oo WRITE_GFS=NO",
             "-lco UNLOGGED=ON",
             "-lco SPATIAL_INDEX=NONE",
+            "-lco GEOMETRY_NAME=wkb_geometry",
             '-f PostgreSQL PG:"{dsn}"',
             "/vsizip/{local_path}/{dataset}_{feature_type}.gml {feature_type}",
         ]

packages/core/src/bag3d/core/assets/ahn/core.py

@@ -25,13 +25,13 @@ def ahn_filename(tile_id: str) -> str:
 
 
 def ahn_dir(root_dir: Path, ahn_version: int) -> Path:
-    """Create a directory path where to store the AHN LAZ files for the given AHN
+    """Return a directory path where to store the AHN LAZ files for the given AHN
     version."""
     return Path(root_dir) / "pointcloud" / f"AHN{ahn_version}"
 
 
 def ahn_laz_dir(root_dir: Path, ahn_version: int) -> Path:
-    """Create a directory path where to store the AHN LAZ files for the given AHN
+    """Return a directory path where to store the AHN LAZ files for the given AHN
     version."""
     return ahn_dir(root_dir, ahn_version) / "as_downloaded" / "LAZ"
 
@@ -69,7 +69,7 @@ def download_ahn_index(
         "requestedEpsg": "28992",
         "outputFormat": "application/json",
         "CountDefault": "2000",
-        "typeName": "layerId_14b12666-cfbb-4362-905a-8832afe5ffa8",
+        "typeName": "layerId_1e56b6d6-3802-4246-a7ed-8f49824b85db",
     }
     logger.info(f"Downloading the AHN tile boundaries from {service_url}")
 

packages/core/src/bag3d/core/assets/ahn/download.py

@@ -134,7 +134,7 @@ def sha256_ahn5(context):
 
 
 @asset
-def tile_index_pdok(context):
+def tile_index_ahn(context):
     """The AHN tile index, including the tile geometry and the file download links."""
     return download_ahn_index(with_geom=True)
 
@@ -143,7 +143,7 @@ def tile_index_pdok(context):
     required_resource_keys={"file_store"},
     partitions_def=PartitionDefinitionAHN(),
 )
-def laz_files_ahn3(context, md5_ahn3, tile_index_pdok):
+def laz_files_ahn3(context, md5_ahn3, tile_index_ahn):
     """AHN3 LAZ files as they are downloaded from PDOK.
 
     The download links are retrieved from the AHN tile index service (blaadindex).
@@ -153,7 +153,7 @@ def laz_files_ahn3(context, md5_ahn3, tile_index_pdok):
     tile_id = context.partition_key
     laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 3)
     laz_dir.mkdir(exist_ok=True, parents=True)
-    url_laz = tile_index_pdok[tile_id]["AHN3_LAZ"]
+    url_laz = tile_index_ahn[tile_id]["AHN3_LAZ"]
     fpath = laz_dir / url_laz.split("/")[-1]
     # Because https://ns_hwh.fundaments.nl is not configured properly.
     # Check with https://www.digicert.com/help/
@@ -198,7 +198,7 @@ def laz_files_ahn3(context, md5_ahn3, tile_index_pdok):
     required_resource_keys={"file_store"},
     partitions_def=PartitionDefinitionAHN(),
 )
-def laz_files_ahn4(context, md5_ahn4, tile_index_pdok):
+def laz_files_ahn4(context, md5_ahn4, tile_index_ahn):
     """AHN4 LAZ files as they are downloaded from PDOK.
 
     The download links are retrieved from the AHN tile index service (blaadindex).
@@ -209,7 +209,7 @@ def laz_files_ahn4(context, md5_ahn4, tile_index_pdok):
 
     laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 4)
     laz_dir.mkdir(exist_ok=True, parents=True)
-    url_laz = tile_index_pdok[tile_id]["AHN4_LAZ"]
+    url_laz = tile_index_ahn[tile_id]["AHN4_LAZ"]
     fpath = laz_dir / url_laz.split("/")[-1]
     # Because https://ns_hwh.fundaments.nl is not configured properly.
     # Check with https://www.digicert.com/help/
@@ -256,7 +256,7 @@ def laz_files_ahn4(context, md5_ahn4, tile_index_pdok):
     required_resource_keys={"file_store"},
     partitions_def=PartitionDefinitionAHN(),
 )
-def laz_files_ahn5(context, sha256_ahn5, tile_index_pdok):
+def laz_files_ahn5(context, sha256_ahn5, tile_index_ahn):
     """AHN5 LAZ files as they are downloaded from PDOK.
 
     The download links are retrieved from the AHN tile index service (blaadindex).
@@ -265,7 +265,7 @@ def laz_files_ahn5(context, sha256_ahn5, tile_index_pdok):
     tile_id = context.partition_key
     laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 5)
     laz_dir.mkdir(exist_ok=True, parents=True)
-    url_laz = tile_index_pdok[tile_id]["AHN5_LAZ"]
+    url_laz = tile_index_ahn[tile_id]["AHN5_LAZ"]
     fpath = laz_dir / url_laz.split("/")[-1]
     # Because https://ns_hwh.fundaments.nl is not configured properly.
     # Check with https://www.digicert.com/help/

packages/core/src/bag3d/core/assets/ahn/metadata.py

@@ -43,12 +43,12 @@ def metadata_table_ahn5(context):
     required_resource_keys={"pdal", "db_connection"},
     partitions_def=PartitionDefinitionAHN(),
 )
-def metadata_ahn3(context, laz_files_ahn3, metadata_table_ahn3, tile_index_pdok):
+def metadata_ahn3(context, laz_files_ahn3, metadata_table_ahn3, tile_index_ahn):
     """Metadata of the AHN3 LAZ file, retrieved from the PDOK tile index and
     computed with 'pdal info'.
     The metadata is loaded into the metadata database table."""
     return compute_load_metadata(
-        context, laz_files_ahn3, metadata_table_ahn3, tile_index_pdok
+        context, laz_files_ahn3, metadata_table_ahn3, tile_index_ahn
     )
 
 
@@ -66,12 +66,12 @@ def metadata_ahn3(context, laz_files_ahn3, metadata_table_ahn3, tile_index_pdok)
     required_resource_keys={"pdal", "db_connection"},
     partitions_def=PartitionDefinitionAHN(),
 )
-def metadata_ahn4(context, laz_files_ahn4, metadata_table_ahn4, tile_index_pdok):
+def metadata_ahn4(context, laz_files_ahn4, metadata_table_ahn4, tile_index_ahn):
     """Metadata of the AHN4 LAZ file, retrieved from the PDOK tile index and
     computed with 'pdal info'.
     The metadata is loaded into the metadata database table."""
     return compute_load_metadata(
-        context, laz_files_ahn4, metadata_table_ahn4, tile_index_pdok
+        context, laz_files_ahn4, metadata_table_ahn4, tile_index_ahn
     )
 
 
@@ -89,12 +89,12 @@ def metadata_ahn4(context, laz_files_ahn4, metadata_table_ahn4, tile_index_pdok)
     required_resource_keys={"pdal", "db_connection"},
     partitions_def=PartitionDefinitionAHN(),
 )
-def metadata_ahn5(context, laz_files_ahn5, metadata_table_ahn5, tile_index_pdok):
+def metadata_ahn5(context, laz_files_ahn5, metadata_table_ahn5, tile_index_ahn):
     """Metadata of the AHN5 LAZ file, retrieved from the PDOK tile index and
     computed with 'pdal info'.
     The metadata is loaded into the metadata database table."""
     return compute_load_metadata(
-        context, laz_files_ahn5, metadata_table_ahn5, tile_index_pdok
+        context, laz_files_ahn5, metadata_table_ahn5, tile_index_ahn
     )
 
 

packages/core/src/bag3d/core/assets/bag/download.py

@@ -12,7 +12,6 @@
 )
 from lxml import objectify
 
-from bag3d.common.utils.geodata import bbox_from_wkt
 from bag3d.common.utils.files import unzip
 from bag3d.common.utils.requests import download_file
 from bag3d.common.utils.database import (
@@ -298,19 +297,36 @@ def load_bag_layer(
         layer_zip = Path(f"{extract_dir}/9999{layer_id}{shortdate}.zip")
         layer_dir = Path(f"{extract_dir}/9999{layer_id}{shortdate}")
         unzip(layer_zip, layer_dir, remove=remove_zip)
+        # Create an empty layer for appending
+        cmd = [
+            "{exe}",
+            "-limit 0",
+            "-overwrite",
+            "-nln {new_table}",
+            "-lco UNLOGGED=ON",
+            "-lco SPATIAL_INDEX=NONE",
+        ]
+        cmd.append("-f PostgreSQL PG:'{dsn}'")
+        cmd.append(str(layer_dir))
+        cmd = " ".join(cmd)
+        return_code, output = context.resources.gdal.app.execute(
+            "ogr2ogr", cmd, kwargs=kwargs, local_path=extract_dir
+        )
+        if return_code != 0:
+            return False
+        # Parallel insert
         cmd = [
             'parallel "{exe}',
             "--config PG_USE_COPY=YES",
-            "-overwrite",
+            "-append",
             "-nln {new_table}",
             "-lco UNLOGGED=ON",
             "-lco SPATIAL_INDEX=NONE",
         ]
         geofilter = context.op_config.get("geofilter")
         if geofilter:
-            bbox = bbox_from_wkt(geofilter)
-            cmd.append("-spat {bbox}")
-            kwargs["bbox"] = " ".join(map(str, bbox))
+            cmd.append("-clipsrc {wkt}")
+            kwargs["wkt"] = geofilter
         cmd.append("-f PostgreSQL PG:'{dsn}'")
         cmd.append('{{}}"')
         cmd.append(f"::: {layer_dir}/*.xml")