diff --git a/docker/dagster/dagster.yaml b/docker/dagster/dagster.yaml index c2b352f8..5c82f419 100644 --- a/docker/dagster/dagster.yaml +++ b/docker/dagster/dagster.yaml @@ -1,6 +1,10 @@ run_coordinator: module: dagster.core.run_coordinator class: QueuedRunCoordinator + tag_concurrency_limits: + - key: "dagster/concurrency_key" + value: "laz_download" + limit: 1 run_launcher: module: dagster.core.launcher diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index 9ec5a863..f1c4160a 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2025.08.25 AS develop +FROM 3dgi/3dbag-pipeline-tools:2025.08.27 AS develop ARG VERSION=develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 36c3c516..31883c77 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2025.08.25 AS develop +FROM 3dgi/3dbag-pipeline-tools:2025.08.27 AS develop ARG VERSION=develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index 55d5842d..44dd5ed2 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2025.08.25 AS develop +FROM 3dgi/3dbag-pipeline-tools:2025.08.27 AS develop ARG VERSION=develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index c7fbd588..2d04f6d7 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -100,9 +100,9 @@ FROM tyler AS roofer ARG JOBS=2 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline -COPY --from=3dgi/roofer:v1.0.0-beta.4 /opt/roofer/bin/. $BAG3D_PIPELINE_LOCATION/tools/bin/ -COPY --from=3dgi/roofer:v1.0.0-beta.4 /opt/roofer/share/proj $BAG3D_PIPELINE_LOCATION/tools/share/proj -COPY --from=3dgi/roofer:v1.0.0-beta.4 /opt/roofer/share/gdal $BAG3D_PIPELINE_LOCATION/tools/share/gdal +COPY --from=3dgi/roofer:v1.0.0-beta.5 /opt/roofer/bin/. $BAG3D_PIPELINE_LOCATION/tools/bin/ +COPY --from=3dgi/roofer:v1.0.0-beta.5 /opt/roofer/share/proj $BAG3D_PIPELINE_LOCATION/tools/share/proj +COPY --from=3dgi/roofer:v1.0.0-beta.5 /opt/roofer/share/gdal $BAG3D_PIPELINE_LOCATION/tools/share/gdal # We only need this until we replace the old geoflow with roofer FROM roofer AS geoflow-old diff --git a/packages/common/uv.lock b/packages/common/uv.lock index cf5b4f49..afa0f67f 100644 --- a/packages/common/uv.lock +++ b/packages/common/uv.lock @@ -88,7 +88,7 @@ requires-dist = [ [[package]] name = "bag3d-specs" version = "2025.7.24" -source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#87464cb127e45e4d53b650cb0bd4ccd23fd2ef44" } +source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#bb85d2f67a93e4718ce8022b75e57fe86f00967a" } dependencies = [ { name = "jsonschema" }, ] diff --git a/packages/core/src/bag3d/core/assets/ahn/download.py b/packages/core/src/bag3d/core/assets/ahn/download.py index 06390035..891a5bea 100644 --- a/packages/core/src/bag3d/core/assets/ahn/download.py +++ b/packages/core/src/bag3d/core/assets/ahn/download.py @@ -157,6 +157,7 @@ class LazFilesConfig(Config): @asset( required_resource_keys={"file_store"}, partitions_def=partition_definition_ahn, + op_tags={"dagster/concurrency_key": "laz_download"}, ) def laz_files_ahn3(context, config: LazFilesConfig, md5_ahn3, tile_index_ahn): """AHN3 LAZ files as they are downloaded from PDOK. @@ -210,6 +211,7 @@ def laz_files_ahn3(context, config: LazFilesConfig, md5_ahn3, tile_index_ahn): @asset( required_resource_keys={"file_store"}, partitions_def=partition_definition_ahn, + op_tags={"dagster/concurrency_key": "laz_download"}, ) def laz_files_ahn4(context, config: LazFilesConfig, md5_ahn4, tile_index_ahn): """AHN4 LAZ files as they are downloaded from PDOK. @@ -266,6 +268,7 @@ def laz_files_ahn4(context, config: LazFilesConfig, md5_ahn4, tile_index_ahn): @asset( required_resource_keys={"file_store"}, partitions_def=partition_definition_ahn, + op_tags={"dagster/concurrency_key": "laz_download"}, ) def laz_files_ahn5(context, config: LazFilesConfig, sha256_ahn5, tile_index_ahn): """AHN5 LAZ files as they are downloaded from PDOK. @@ -408,33 +411,32 @@ def download_ahn_laz( def download_laz( file_size, fpath, is_new, nr_retries, success, url, url_laz, verify_ssl ): + fpath_download = Path() for i in range(nr_retries): - try: - fpath = download_file( - url=url, - target_path=fpath, - chunk_size=1024 * 1024, - verify=verify_ssl, - ) - if fpath is None: - # Download failed - logger.warning(format_laz_log(fpath, "Downloading failed!")) + fpath_download = download_file( + url=url, + target_path=fpath, + chunk_size=1024 * 1024, + verify=verify_ssl, + ) + if fpath_download is None: + # Download failed + if i == nr_retries - 1: url_laz = None - fpath = Path() + fpath_download = Path() success = False is_new = False file_size = 0.0 + logger.error(f"Download failed after {i + 1} retries") else: - success = True - is_new = True - file_size = round(fpath.stat().st_size / 1e6, 2) - break - except ConnectionError as e: - if i == 4: - raise e - else: - logger.warning(f"Retrying ({i + 1}/5) due to {e}") - return file_size, fpath, is_new, success, url_laz + logger.warning(f"Retrying ({i + 1}/{nr_retries})") + else: + success = True + is_new = True + file_size = round(fpath_download.stat().st_size / 1e6, 2) + break + + return file_size, fpath_download, is_new, success, url_laz def match_sha( diff --git a/packages/core/src/bag3d/core/assets/export/metadata.py b/packages/core/src/bag3d/core/assets/export/metadata.py index 67028862..1e1c9b27 100644 --- a/packages/core/src/bag3d/core/assets/export/metadata.py +++ b/packages/core/src/bag3d/core/assets/export/metadata.py @@ -360,9 +360,11 @@ def metadata(context: AssetExecutionContext): "software": [ { "name": "geoflow-bundle", - "version": resource_defs["geoflow"].app.version("geof"), + "version": resource_defs["geoflow"].app.version( + "geof", version_cmd="--list-plugins --verbose" + ), "repository": "https://github.com/geoflow3d/geoflow-bundle", - "description": "3D building model reconstruction", + "description": "Format conversion to CityJSON, OBJ, GeoPackage, glTF", }, { "name": "roofer", diff --git a/packages/core/src/bag3d/core/jobs.py b/packages/core/src/bag3d/core/jobs.py index 2e0ff7cf..3ff5f250 100644 --- a/packages/core/src/bag3d/core/jobs.py +++ b/packages/core/src/bag3d/core/jobs.py @@ -21,9 +21,6 @@ name="ahn3", description="Make sure that the available AHN 3 LAZ files are present on disk, " "and their metadata is recorded.", - executor_def=multiprocess_executor.configured( - {"max_concurrent": int(getenv("BAG3D_CONCURRENCY_JOB_AHN3", 1))} - ), selection=AssetSelection.assets(["ahn", "laz_files_ahn3"]) | AssetSelection.assets(["ahn", "metadata_ahn3"]) | AssetSelection.assets(["ahn", "lasindex_ahn3"]), @@ -33,9 +30,6 @@ name="ahn4", description="Make sure that the available AHN 4 LAZ files are present on disk, " "and their metadata is recorded.", - executor_def=multiprocess_executor.configured( - {"max_concurrent": int(getenv("BAG3D_CONCURRENCY_JOB_AHN4", 1))} - ), selection=AssetSelection.assets(["ahn", "laz_files_ahn4"]) | AssetSelection.assets(["ahn", "metadata_ahn4"]) | AssetSelection.assets(["ahn", "lasindex_ahn4"]), @@ -45,9 +39,6 @@ name="ahn5", description="Make sure that the available AHN 5 LAZ files are present on disk, " "and their metadata is recorded.", - executor_def=multiprocess_executor.configured( - {"max_concurrent": int(getenv("BAG3D_CONCURRENCY_JOB_AHN5", 1))} - ), selection=AssetSelection.assets(["ahn", "laz_files_ahn5"]) | AssetSelection.assets(["ahn", "metadata_ahn5"]) | AssetSelection.assets(["ahn", "lasindex_ahn5"]), diff --git a/packages/core/uv.lock b/packages/core/uv.lock index b85645ba..56516a41 100644 --- a/packages/core/uv.lock +++ b/packages/core/uv.lock @@ -118,7 +118,7 @@ requires-dist = [ [[package]] name = "bag3d-specs" version = "2025.7.24" -source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#87464cb127e45e4d53b650cb0bd4ccd23fd2ef44" } +source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#bb85d2f67a93e4718ce8022b75e57fe86f00967a" } dependencies = [ { name = "jsonschema" }, ] diff --git a/packages/floors_estimation/uv.lock b/packages/floors_estimation/uv.lock index e3e53e5a..bedc9ec2 100644 --- a/packages/floors_estimation/uv.lock +++ b/packages/floors_estimation/uv.lock @@ -114,7 +114,7 @@ requires-dist = [ [[package]] name = "bag3d-specs" version = "2025.7.24" -source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#87464cb127e45e4d53b650cb0bd4ccd23fd2ef44" } +source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#bb85d2f67a93e4718ce8022b75e57fe86f00967a" } dependencies = [ { name = "jsonschema" }, ] diff --git a/packages/party_walls/uv.lock b/packages/party_walls/uv.lock index 0608531c..d1d78f49 100644 --- a/packages/party_walls/uv.lock +++ b/packages/party_walls/uv.lock @@ -116,7 +116,7 @@ requires-dist = [ [[package]] name = "bag3d-specs" version = "2025.7.24" -source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#87464cb127e45e4d53b650cb0bd4ccd23fd2ef44" } +source = { git = "https://github.com/3DBAG/3dbag-specs.git?branch=develop#bb85d2f67a93e4718ce8022b75e57fe86f00967a" } dependencies = [ { name = "jsonschema" }, ]