Merge pull request #87 from 3DBAG/fix/integrate-roofer

Fix/integrate roofer

Author: balazsdukai

docker/.env

@@ -6,6 +6,8 @@ DAGSTER_POSTGRES_DB="postgres_db"
 TOOLS_DIR=/opt/3dbag-pipeline/tools
 
 BAG3D_VENVS=/3dbag-pipeline/venvs
+BAG3D_FILESTORE=/data/volume
+BAG3D_RELEASE_VERSION=develop
 BAG3D_TEST_DATA=/data/volume
 BAG3D_FLOORS_ESTIMATION_MODEL=/data/volume/model/pipeline_model1_gbr_untuned.joblib
 BAG3D_RELEASE_VERSION=test_version
@@ -27,6 +29,7 @@ TYLER_METADATA_JSON=/opt/3dbag-pipeline/tools/share/tyler/resources/geof/metadat
 EXE_PATH_TYLER=/opt/3dbag-pipeline/tools/bin/tyler
 EXE_PATH_TYLER_DB=/opt/3dbag-pipeline/tools/bin/tyler-db
 EXE_PATH_ROOFER_CROP=/opt/3dbag-pipeline/tools/bin/crop
+EXE_PATH_ROOFER_ROOFER=/opt/3dbag-pipeline/tools/bin/roofer
 EXE_PATH_ROOFER_RECONSTRUCT=/opt/3dbag-pipeline/tools/bin/geof
 FLOWCHART_PATH_RECONSTRUCT=/opt/3dbag-pipeline/tools/share/geoflow-bundle/flowcharts/reconstruct_bag.json
 EXE_PATH_OGR2OGR=/opt/3dbag-pipeline/tools/bin/ogr2ogr

docker/compose.yaml

@@ -11,6 +11,8 @@ services:
       POSTGRES_DB: $DAGSTER_POSTGRES_DB
     networks:
       - bag3d-dev-network
+    volumes:
+      - bag3d-dev-dagster-postgresql:/var/lib/postgresql/data
 
   # This service stores the asset data that is generated by the pipeline
   data-postgresql:
@@ -76,15 +78,21 @@ services:
       data-postgresql:
         condition: service_healthy
         restart: true
+    # Normally we wouldn't need watch:action:rebuild, but because we are using
+    # dagster's DockerRunLauncher, it always starts a new container from the image, for
+    # each run. While, watch:action:sync (or sync+restart)copies the changed local code into the
+    # 3dbag-pipeline-core-develop container (not the image). But since DockerRunLauncher
+    # doesn't actaully use this container, but always starts a new one, we have to build
+    # a new image on each change.
     develop:
       watch:
-        - action: sync
+        - action: rebuild
           path: ../packages/common
           target: /opt/3dbag-pipeline/packages/common
         - action: rebuild
           path: ../packages/common/pyproject.toml
           target: /opt/3dbag-pipeline/packages/common/pyproject.toml
-        - action: sync
+        - action: rebuild
           path: ../packages/core
           target: /opt/3dbag-pipeline/packages/core
         - action: rebuild
@@ -115,13 +123,13 @@ services:
         restart: true
     develop:
       watch:
-        - action: sync
+        - action: rebuild
           path: ../packages/common
           target: /opt/3dbag-pipeline/packages/common
         - action: rebuild
           path: ../packages/common/pyproject.toml
           target: /opt/3dbag-pipeline/packages/common/pyproject.toml
-        - action: sync
+        - action: rebuild
           path: ../packages/floors_estimation
           target: /opt/3dbag-pipeline/packages/floors_estimation
         - action: rebuild
@@ -151,13 +159,13 @@ services:
         restart: true
     develop:
       watch:
-        - action: sync
+        - action: rebuild
           path: ../packages/common
           target: /opt/3dbag-pipeline/packages/common
         - action: rebuild
           path: ../packages/common/pyproject.toml
           target: /opt/3dbag-pipeline/packages/common/pyproject.toml
-        - action: sync
+        - action: rebuild
           path: ../packages/party_walls
           target: /opt/3dbag-pipeline/packages/party_walls
         - action: rebuild
@@ -228,6 +236,8 @@ volumes:
     external: true
   bag3d-dev-data-pipeline:
     external: true
+  bag3d-dev-dagster-postgresql:
+    external: true
   bag3d-dev-dagster-home:
     external: true
 

docker/pipeline/bag3d-core.dockerfile

@@ -1,4 +1,4 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.10.31 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.11.09 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"

docker/pipeline/bag3d-floors-estimation.dockerfile

@@ -1,4 +1,4 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.10.31 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.11.09 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"

docker/pipeline/bag3d-party-walls.dockerfile

@@ -1,4 +1,4 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.10.31 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.11.09 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"

makefile

@@ -25,11 +25,13 @@ docker_volume_create:
 	docker cp docker/dagster/dagster.yaml bag3d-dev-temp-container:/opt/dagster/dagster_home/
 	docker cp docker/dagster/workspace.yaml bag3d-dev-temp-container:/opt/dagster/dagster_home/
 	docker rm -f bag3d-dev-temp-container
+	docker volume create bag3d-dev-dagster-postgresql
 
 docker_volume_rm:
 	docker volume rm -f bag3d-dev-data-pipeline
 	docker volume rm -f bag3d-dev-data-postgresql
 	docker volume rm -f bag3d-dev-dagster-home
+	docker volume rm -f bag3d-dev-dagster-postgresql
 
 docker_volume_recreate: docker_volume_rm docker_volume_create
 

packages/common/src/bag3d/common/resources/__init__.py

@@ -40,10 +40,10 @@
 
 
 file_store = FileStoreResource(
-    data_dir="/tmp", dir_id=os.getenv("BAG3D_RELEASE_VERSION")
+    data_dir=os.getenv("BAG3D_FILESTORE"), dir_id=os.getenv("BAG3D_RELEASE_VERSION")
 )
 file_store_fastssd = FileStoreResource(
-    data_dir="/tmp",
+    data_dir=os.getenv("BAG3D_FILESTORE"),
     dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
 )
 
@@ -62,7 +62,10 @@
     exe_tyler=os.getenv("EXE_PATH_TYLER"), exe_tyler_db=os.getenv("EXE_PATH_TYLER_DB")
 )
 
-roofer = RooferResource(exe_roofer_crop=os.getenv("EXE_PATH_ROOFER_CROP"))
+roofer = RooferResource(
+    exe_crop=os.getenv("EXE_PATH_ROOFER_CROP"),
+    exe_roofer=os.getenv("EXE_PATH_ROOFER_ROOFER"),
+)
 
 geoflow = GeoflowResource(
     exe_geoflow=os.getenv("EXE_PATH_ROOFER_RECONSTRUCT"),

packages/common/src/bag3d/common/resources/executables.py

@@ -435,23 +435,28 @@ def app(self) -> AppImage:
 class RooferResource(ConfigurableResource):
     """
     A RooferResource can be configured by providing the paths to
-    Roofer `crop` executable on the local system.
+    Roofer `crop` and `roofer` executables on the local system.
 
     Example:
 
-        roofer_resource = RooferResource(exe_roofer_crop=os.getenv("EXE_PATH_ROOFER_CROP"))
+        roofer_resource = RooferResource(exe_crop=os.getenv("EXE_PATH_ROOFER_CROP"),
+                                         exe_roofer=os.getenv("EXE_PATH_ROOFER_ROOFER"))
 
     After the resource has been instantiated, roofer (AppImage) can
     be acquired with the `app` property:
 
         roofer = roofer_resource.app
     """
 
-    exe_roofer_crop: str
+    exe_crop: str
+    exe_roofer: str
 
     @property
     def exes(self) -> Dict[str, str]:
-        return {"crop": self.exe_roofer_crop}
+        return {
+            "crop": self.exe_crop,
+            "roofer": self.exe_roofer,
+        }
 
     @property
     def with_docker(self) -> bool:

packages/common/src/bag3d/common/utils/geodata.py

@@ -216,6 +216,7 @@ def ogr2postgres(
             "-oo WRITE_GFS=NO",
             "-lco UNLOGGED=ON",
             "-lco SPATIAL_INDEX=NONE",
+            "-lco GEOMETRY_NAME=wkb_geometry",
             '-f PostgreSQL PG:"{dsn}"',
             "/vsizip/{local_path}/{dataset}_{feature_type}.gml {feature_type}",
         ]

packages/core/src/bag3d/core/assets/ahn/core.py

@@ -25,13 +25,13 @@ def ahn_filename(tile_id: str) -> str:
 
 
 def ahn_dir(root_dir: Path, ahn_version: int) -> Path:
-    """Create a directory path where to store the AHN LAZ files for the given AHN
+    """Return a directory path where to store the AHN LAZ files for the given AHN
     version."""
     return Path(root_dir) / "pointcloud" / f"AHN{ahn_version}"
 
 
 def ahn_laz_dir(root_dir: Path, ahn_version: int) -> Path:
-    """Create a directory path where to store the AHN LAZ files for the given AHN
+    """Return a directory path where to store the AHN LAZ files for the given AHN
     version."""
     return ahn_dir(root_dir, ahn_version) / "as_downloaded" / "LAZ"
 
@@ -69,7 +69,7 @@ def download_ahn_index(
         "requestedEpsg": "28992",
         "outputFormat": "application/json",
         "CountDefault": "2000",
-        "typeName": "layerId_14b12666-cfbb-4362-905a-8832afe5ffa8",
+        "typeName": "layerId_1e56b6d6-3802-4246-a7ed-8f49824b85db",
     }
     logger.info(f"Downloading the AHN tile boundaries from {service_url}")