merge

Author: GinaStavropoulou

.github/workflows/build-docker-tools.yaml

@@ -0,0 +1,39 @@
+name: Build and Publish The Tools Docker Image
+
+on:
+  push:
+    tags:
+      - '*'
+    branches:
+      - "develop"
+      - "pc-*"
+    paths:
+      - docker/tools/Dockerfile
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set docker image tag
+        run: echo "DOCKER_TAG=$(date +'%Y.%m.%d')" >> $GITHUB_ENV
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push tools image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./docker/tools/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            JOBS=2
+            VERSION=${{ env.DOCKER_TAG }}
+          push: true
+          tags: 3dgi/3dbag-pipeline-tools:${{ env.DOCKER_TAG }}
+

docker/.env

@@ -12,9 +12,9 @@ TEMP_CONTAINER="${COMPOSE_PROJECT_NAME}-temp-container"
 # Dagster instance configuration
 DAGSTER_HOME="/opt/dagster/dagster_home"
 DAGSTER_SERVER_PORT=3003
-DAGSTER_POSTGRES_USER="postgres_user"
-DAGSTER_POSTGRES_PASSWORD="postgres_password"
-DAGSTER_POSTGRES_DB="postgres_db"
+DAGSTER_POSTGRES_USER="dagster"
+DAGSTER_POSTGRES_PASSWORD="dagster_pwd"
+DAGSTER_POSTGRES_DB="dagster"
 
 # 3dbag-pipeline resource configuration
 DAGSTER_ENVIRONMENT="test"

docker/compose.yaml

@@ -4,7 +4,11 @@ services:
   # and event log storage.
   dagster-postgresql:
     image: postgres:16
-    container_name: bag3d-dagster-postgresql
+    container_name: bag3d-dagster-postgresql-${BAG3D_DOCKER_IMAGE_TAG}
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -d ${DAGSTER_POSTGRES_DB} -U ${DAGSTER_POSTGRES_USER}" ]
+      interval: 3s
+      retries: 10
     environment:
       POSTGRES_USER: $DAGSTER_POSTGRES_USER
       POSTGRES_PASSWORD: $DAGSTER_POSTGRES_PASSWORD
@@ -23,14 +27,11 @@ services:
         - pg_pswd=$BAG3D_PG_PASSWORD
         - pg_db=$BAG3D_PG_DATABASE
     command: ["-c", "shared_buffers=8GB", "-c", "max_connections=200", "-c", "work_mem=30MB", "-c", "logging_collector=on", "-c", "log_statement=all", "-c", "log_directory=/var/log/postgresql/"]
-    platform: linux/amd64
-    container_name: bag3d-data-postgresql
+    container_name: bag3d-data-postgresql-${BAG3D_DOCKER_IMAGE_TAG}
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d ${BAG3D_PG_DATABASE}"]
+      test: ["CMD-SHELL", "pg_isready -d ${BAG3D_PG_DATABASE} -U {BAG3D_PG_USER}"]
       interval: 3s
-      retries: 10
-      start_period: 30s
-      timeout: 10s
+      retries: 20
     shm_size: '1gb' 
     ports:
       - "${BAG3D_PG_PORT_ON_HOST}:5432"
@@ -62,9 +63,14 @@ services:
       dockerfile: docker/pipeline/bag3d-core.dockerfile
       args:
         - VERSION=develop
-    container_name: 3dbag-pipeline-core
     image: 3dbag-pipeline-core:${BAG3D_DOCKER_IMAGE_TAG}
     pull_policy: build
+    container_name: 3dbag-pipeline-core-${BAG3D_DOCKER_IMAGE_TAG}
+    # the port is built into the image
+    healthcheck:
+      test: ["CMD-SHELL", "dagster api grpc-health-check -p 4000"]
+      interval: 2s
+      retries: 10
     restart: always
     environment:
       DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-core:${BAG3D_DOCKER_IMAGE_TAG}"
@@ -106,9 +112,14 @@ services:
       dockerfile: docker/pipeline/bag3d-floors-estimation.dockerfile
       args:
         - VERSION=develop
-    container_name: 3dbag-pipeline-floors-estimation
     image: 3dbag-pipeline-floors-estimation:${BAG3D_DOCKER_IMAGE_TAG}
     pull_policy: build
+    container_name: 3dbag-pipeline-floors-estimation-${BAG3D_DOCKER_IMAGE_TAG}
+    # the port is built into the image
+    healthcheck:
+      test: [ "CMD-SHELL", "dagster api grpc-health-check -p 4001" ]
+      interval: 2s
+      retries: 10
     restart: always
     environment:
       DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-floors-estimation:${BAG3D_DOCKER_IMAGE_TAG}"
@@ -137,15 +148,21 @@ services:
         - action: rebuild
           path: ../packages/floors_estimation/pyproject.toml
           target: /opt/3dbag-pipeline/packages/floors_estimation/pyproject.toml
+
   bag3d-party-walls:
     build:
       context: ../
       dockerfile: docker/pipeline/bag3d-party-walls.dockerfile
       args:
         - VERSION=develop
-    container_name: 3dbag-pipeline-party-walls
     image: 3dbag-pipeline-party-walls:${BAG3D_DOCKER_IMAGE_TAG}
     pull_policy: build
+    container_name: 3dbag-pipeline-party-walls-${BAG3D_DOCKER_IMAGE_TAG}
+    # the port is built into the image
+    healthcheck:
+      test: [ "CMD-SHELL", "dagster api grpc-health-check -p 4002" ]
+      interval: 2s
+      retries: 10
     restart: always
     environment:
       DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-party-walls:${BAG3D_DOCKER_IMAGE_TAG}"
@@ -183,6 +200,7 @@ services:
       context: ./dagster
     image: 3dgi/3dbag-pipeline-dagster:${BAG3D_DOCKER_IMAGE_TAG}
     pull_policy: build
+    container_name: bag3d-dagster-webserver-${BAG3D_DOCKER_IMAGE_TAG}
     entrypoint:
       - dagster-webserver
       - -h
@@ -191,7 +209,6 @@ services:
       - "3000"
       - -w
       - workspace.yaml
-    container_name: bag3d-dagster-webserver
     expose:
       - "3000"
     ports:
@@ -204,10 +221,18 @@ services:
     networks:
       - bag3d-network
     depends_on:
-      - dagster-postgresql
-      - bag3d-core
-      - bag3d-floors-estimation
-      - bag3d-party-walls
+      dagster-postgresql:
+        condition: service_healthy
+        restart: true
+      bag3d-core:
+        condition: service_healthy
+        restart: true
+      bag3d-floors-estimation:
+        condition: service_healthy
+        restart: true
+      bag3d-party-walls:
+        condition: service_healthy
+        restart: true
 
   # This service runs the dagster-daemon process, which is responsible for taking runs
   # off of the queue and launching them, as well as creating runs from schedules or sensors.
@@ -216,10 +241,10 @@ services:
       context: ./dagster
     image: 3dgi/3dbag-pipeline-dagster:${BAG3D_DOCKER_IMAGE_TAG}
     pull_policy: build
+    container_name: bag3d-dagster-daemon-${BAG3D_DOCKER_IMAGE_TAG}
     entrypoint:
       - dagster-daemon
       - run
-    container_name: bag3d-dagster-daemon
     restart: on-failure
     env_file:
       - ./.env
@@ -229,10 +254,18 @@ services:
     networks:
       - bag3d-network
     depends_on:
-      - dagster-postgresql
-      - bag3d-core
-      - bag3d-floors-estimation
-      - bag3d-party-walls
+      dagster-postgresql:
+        condition: service_healthy
+        restart: true
+      bag3d-core:
+        condition: service_healthy
+        restart: true
+      bag3d-floors-estimation:
+        condition: service_healthy
+        restart: true
+      bag3d-party-walls:
+        condition: service_healthy
+        restart: true
 
 volumes:
   bag3d-data-postgresql:

docker/pipeline/bag3d-core.dockerfile

@@ -1,4 +1,4 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.12.06 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"
@@ -20,7 +20,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --python $VIRTUAL_ENV/bin/python
 
 COPY . $BAG3D_PIPELINE_LOCATION
-COPY ./docker/.env $BAG3D_PIPELINE_LOCATION/.env
 
 # Install the workflow package
 RUN --mount=type=cache,target=/root/.cache/uv \
@@ -32,4 +31,4 @@ EXPOSE 4000
 
 # CMD allows this to be overridden from run launchers or executors that want
 # to run other commands against your repository
-CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "bag3d.core.code_location", "--inject-env-vars-from-instance"]
+CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "bag3d.core.code_location"]

docker/pipeline/bag3d-floors-estimation.dockerfile

@@ -1,4 +1,4 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.12.06 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"
@@ -20,7 +20,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --python $VIRTUAL_ENV/bin/python
 
 COPY . $BAG3D_PIPELINE_LOCATION
-COPY ./docker/.env $BAG3D_PIPELINE_LOCATION/.env
 
 # Install the workflow package
 RUN --mount=type=cache,target=/root/.cache/uv \
@@ -32,4 +31,4 @@ EXPOSE 4001
 
 # CMD allows this to be overridden from run launchers or executors that want
 # to run other commands against your repository
-CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4001", "-m", "bag3d.floors_estimation.code_location", "--inject-env-vars-from-instance"]
+CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4001", "-m", "bag3d.floors_estimation.code_location"]

docker/pipeline/bag3d-party-walls.dockerfile

@@ -1,4 +1,4 @@
-FROM 3dgi/3dbag-pipeline-tools:2024.12.06 AS develop
+FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
 LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"
@@ -33,7 +33,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --python $VIRTUAL_ENV/bin/python
 
 COPY . $BAG3D_PIPELINE_LOCATION
-COPY ./docker/.env $BAG3D_PIPELINE_LOCATION/.env
 
 # Install the workflow package
 RUN --mount=type=cache,target=/root/.cache/uv \
@@ -45,4 +44,4 @@ EXPOSE 4002
 
 # CMD allows this to be overridden from run launchers or executors that want
 # to run other commands against your repository
-CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4002", "-m", "bag3d.party_walls.code_location", "--inject-env-vars-from-instance"]
+CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4002", "-m", "bag3d.party_walls.code_location"]

docker/tools/Dockerfile

@@ -100,13 +100,13 @@ FROM roofer AS geoflow-old
 ARG JOBS=2
 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
 
-COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof
-COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /export /export
-COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib
-COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so
-COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/LASlib /export/usr/local/lib/
-COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/gdalplugins /export/usr/local/lib/
-COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins
+COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof
+COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /export /export
+COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib
+COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so
+COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/LASlib /export/usr/local/lib/
+COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/gdalplugins /export/usr/local/lib/
+COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins
 ENV GF_PLUGIN_FOLDER=$BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins
 
 RUN echo /export/usr/local/lib >> /etc/ld.so.conf.d/geoflow.conf; \

packages/core/src/bag3d/core/assets/export/metadata.py

@@ -99,13 +99,17 @@ def feature_evaluation(context):
     attributes_to_include = (
         "b3_pw_selectie_reden",
         "b3_pw_bron",
-        "b3_puntdichtheid_ahn3",
-        "b3_puntdichtheid_ahn4",
-        "b3_mutatie_ahn3_ahn4",
-        "b3_nodata_fractie_ahn3",
-        "b3_nodata_fractie_ahn4",
-        "b3_nodata_radius_ahn3",
-        "b3_nodata_radius_ahn4",
+        "b3_puntdichtheid_AHN3",
+        "b3_puntdichtheid_AHN4",
+        "b3_puntdichtheid_AHN5",
+        "b3_mutatie_AHN3_AHN4",
+        "b3_mutatie_AHN4_AHN5",
+        "b3_nodata_fractie_AHN3",
+        "b3_nodata_fractie_AHN4",
+        "b3_nodata_fractie_AHN5",
+        "b3_nodata_radius_AHN3",
+        "b3_nodata_radius_AHN4",
+        "b3_nodata_radius_AHN5",
     )
     cityobject_info = {lod: 0 for lod in lods}
     cityobject_info["has_geometry"] = False

packages/core/src/bag3d/core/assets/export/tile.py

@@ -44,7 +44,7 @@ def reconstruction_output_tiles_func(context, format: str, **kwargs: dict):
     )
     # # Set the parallelism in tyler from the dagster instance configuration (the dagster.yaml in $DAGSTER_HOME)
     # num_threads = context.instance.run_coordinator.inst_data.config_dict["max_concurrent_runs"]
-    num_threads = 4
+    num_threads = 40
     cmd = [
         f"RAYON_NUM_THREADS={num_threads}",
         "RUST_LOG=info",