From 7395e56ea10b98a58446d47dbb624edb06df634a Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Sun, 8 Dec 2024 11:32:26 +0100 Subject: [PATCH 01/40] Add yoc-attribute --- .../core/src/bag3d/core/assets/reconstruction/reconstruction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index b5d077cb..79f6f2e2 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -156,6 +156,7 @@ def create_roofer_config( polygon-source = "{footprint_file}" id-attribute = "identificatie" force-lod11-attribute = "b3_kas_warenhuis" + yoc-attribute = "oorspronkelijkbouwjaar" lod11-fallback-area = 30000 split-cjseq = true From fd4447322714ba53a7cb7a166bd67d0c4b6c0fa1 Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Mon, 9 Dec 2024 12:40:50 +0100 Subject: [PATCH 02/40] use ahn5 tiles in as_downloaded --- .../src/bag3d/core/assets/reconstruction/reconstruction.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index 79f6f2e2..2e776e69 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -285,7 +285,9 @@ def create_roofer_config( footprint_file=f"PG:{context.resources.db_connection.connect.dsn} tables={tile_view}", ahn3_files=laz_files_ahn3, ahn4_files=laz_files_ahn4, - ahn5_files=laz_files_ahn5, + ahn5_files=ahn_dir( + context.resources.file_store.file_store.data_dir, ahn_version=5 + ).joinpath("as_downloaded/LAZ"), output_path=output_dir, ) path_toml = output_dir / "roofer.toml" From 7c729032ad11f5e5a01bc186fb93b0a8e45a8a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 19:40:23 +0100 Subject: [PATCH 03/40] fastssd --- docker/.env | 1 + packages/common/src/bag3d/common/resources/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/.env b/docker/.env index 5c92b9b9..3845e3b7 100644 --- a/docker/.env +++ b/docker/.env @@ -19,6 +19,7 @@ DAGSTER_POSTGRES_DB="postgres_db" # 3dbag-pipeline resource configuration DAGSTER_ENVIRONMENT="test" BAG3D_FILESTORE="/data/volume" +BAG3D_FILESTORE_FASTSSD="/data/volume" BAG3D_RELEASE_VERSION="develop" BAG3D_FLOORS_ESTIMATION_MODEL="/data/volume/model/pipeline_model1_gbr_untuned.joblib" BAG3D_TEST_DATA="/data/volume" diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index a9864780..02243531 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -41,7 +41,7 @@ file_store = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE")) -file_store_fastssd = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE")) +file_store_fastssd = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE_FASTSSD")) file_store_test = FileStoreResource( data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input") From bbbe3140041c5688edf4078a04379a2115cd4d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 20:01:42 +0100 Subject: [PATCH 04/40] env name --- packages/common/src/bag3d/common/resources/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 02243531..c1e8bf08 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -135,7 +135,9 @@ "test": RESOURCES_TEST, "default": RESOURCES_DEFAULT, } + env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() +logger.info(f"{env_name=}") if env_name not in resource_defs_by_env_name.keys(): logger.warning(f"Invalid environment: {env_name}, setting to default") env_name = "default" From 50bab2a71cd0ae8fc2254360d193f524eeb360ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 20:08:40 +0100 Subject: [PATCH 05/40] env name --- packages/common/src/bag3d/common/resources/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index c1e8bf08..2d70aeff 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -137,7 +137,7 @@ } env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() -logger.info(f"{env_name=}") +print(f"{env_name=}") if env_name not in resource_defs_by_env_name.keys(): logger.warning(f"Invalid environment: {env_name}, setting to default") env_name = "default" From 7fce9338ed74fbb7a93e8db8885e46b747ee7ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 20:21:16 +0100 Subject: [PATCH 06/40] env name --- packages/common/src/bag3d/common/resources/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 2d70aeff..5f1ec704 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -137,8 +137,8 @@ } env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() -print(f"{env_name=}") -if env_name not in resource_defs_by_env_name.keys(): +print(f"env_name={env_name}") +if env_name not in resource_defs_by_env_name: logger.warning(f"Invalid environment: {env_name}, setting to default") env_name = "default" From bc7350881ca5e854efb0581591504e286d23486d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 20:26:36 +0100 Subject: [PATCH 07/40] env name --- .../src/bag3d/common/resources/__init__.py | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 5f1ec704..dfda86ad 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -75,7 +75,7 @@ ) -RESOURCES_LOCAL = { +resource_defs = { "gdal": gdal, "file_store": file_store, "file_store_fastssd": file_store_fastssd, @@ -89,57 +89,57 @@ } -RESOURCES_TEST = { - "gdal": gdal, - "file_store": file_store_test, - "file_store_fastssd": file_store_fastssd_test, - "db_connection": db_connection, - "pdal": pdal, - "lastools": lastools, - "tyler": tyler, - "geoflow": geoflow, - "roofer": roofer, - "version": version, -} - -RESOURCES_PROD = { - "gdal": gdal, - "file_store": file_store_gilfoyle, - "file_store_fastssd": file_store_gilfoyle_fastssd, - "db_connection": db_connection, - "pdal": pdal, - "lastools": lastools, - "tyler": tyler, - "geoflow": geoflow, - "roofer": roofer, - "version": version, -} - -RESOURCES_DEFAULT = { - "gdal": GDALResource(), - "file_store": FileStoreResource(), - "file_store_fastssd": FileStoreResource(), - "db_connection": DatabaseResource(), - "pdal": PDALResource(), - "lastools": LASToolsResource(), - "tyler": TylerResource(), - "geoflow": GeoflowResource(), - "roofer": RooferResource(), - "version": VersionResource(), -} - - -resource_defs_by_env_name = { - "prod": RESOURCES_PROD, - "local": RESOURCES_LOCAL, - "test": RESOURCES_TEST, - "default": RESOURCES_DEFAULT, -} - -env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() -print(f"env_name={env_name}") -if env_name not in resource_defs_by_env_name: - logger.warning(f"Invalid environment: {env_name}, setting to default") - env_name = "default" - -resource_defs = resource_defs_by_env_name[env_name] +# RESOURCES_TEST = { +# "gdal": gdal, +# "file_store": file_store_test, +# "file_store_fastssd": file_store_fastssd_test, +# "db_connection": db_connection, +# "pdal": pdal, +# "lastools": lastools, +# "tyler": tyler, +# "geoflow": geoflow, +# "roofer": roofer, +# "version": version, +# } +# +# RESOURCES_PROD = { +# "gdal": gdal, +# "file_store": file_store_gilfoyle, +# "file_store_fastssd": file_store_gilfoyle_fastssd, +# "db_connection": db_connection, +# "pdal": pdal, +# "lastools": lastools, +# "tyler": tyler, +# "geoflow": geoflow, +# "roofer": roofer, +# "version": version, +# } +# +# RESOURCES_DEFAULT = { +# "gdal": GDALResource(), +# "file_store": FileStoreResource(), +# "file_store_fastssd": FileStoreResource(), +# "db_connection": DatabaseResource(), +# "pdal": PDALResource(), +# "lastools": LASToolsResource(), +# "tyler": TylerResource(), +# "geoflow": GeoflowResource(), +# "roofer": RooferResource(), +# "version": VersionResource(), +# } +# +# +# resource_defs_by_env_name = { +# "prod": RESOURCES_PROD, +# "local": RESOURCES_LOCAL, +# "test": RESOURCES_TEST, +# "default": RESOURCES_DEFAULT, +# } +# +# env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() +# print(f"env_name={env_name}") +# if env_name not in resource_defs_by_env_name: +# logger.warning(f"Invalid environment: {env_name}, setting to default") +# env_name = "default" +# +# resource_defs = resource_defs_by_env_name[env_name] From adb5f82f79d2c67748a0a6f6b9f41985d9e05b59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 20:35:28 +0100 Subject: [PATCH 08/40] Revert "env name" This reverts commit bc7350881ca5e854efb0581591504e286d23486d. --- .../src/bag3d/common/resources/__init__.py | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index dfda86ad..5f1ec704 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -75,7 +75,7 @@ ) -resource_defs = { +RESOURCES_LOCAL = { "gdal": gdal, "file_store": file_store, "file_store_fastssd": file_store_fastssd, @@ -89,57 +89,57 @@ } -# RESOURCES_TEST = { -# "gdal": gdal, -# "file_store": file_store_test, -# "file_store_fastssd": file_store_fastssd_test, -# "db_connection": db_connection, -# "pdal": pdal, -# "lastools": lastools, -# "tyler": tyler, -# "geoflow": geoflow, -# "roofer": roofer, -# "version": version, -# } -# -# RESOURCES_PROD = { -# "gdal": gdal, -# "file_store": file_store_gilfoyle, -# "file_store_fastssd": file_store_gilfoyle_fastssd, -# "db_connection": db_connection, -# "pdal": pdal, -# "lastools": lastools, -# "tyler": tyler, -# "geoflow": geoflow, -# "roofer": roofer, -# "version": version, -# } -# -# RESOURCES_DEFAULT = { -# "gdal": GDALResource(), -# "file_store": FileStoreResource(), -# "file_store_fastssd": FileStoreResource(), -# "db_connection": DatabaseResource(), -# "pdal": PDALResource(), -# "lastools": LASToolsResource(), -# "tyler": TylerResource(), -# "geoflow": GeoflowResource(), -# "roofer": RooferResource(), -# "version": VersionResource(), -# } -# -# -# resource_defs_by_env_name = { -# "prod": RESOURCES_PROD, -# "local": RESOURCES_LOCAL, -# "test": RESOURCES_TEST, -# "default": RESOURCES_DEFAULT, -# } -# -# env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() -# print(f"env_name={env_name}") -# if env_name not in resource_defs_by_env_name: -# logger.warning(f"Invalid environment: {env_name}, setting to default") -# env_name = "default" -# -# resource_defs = resource_defs_by_env_name[env_name] +RESOURCES_TEST = { + "gdal": gdal, + "file_store": file_store_test, + "file_store_fastssd": file_store_fastssd_test, + "db_connection": db_connection, + "pdal": pdal, + "lastools": lastools, + "tyler": tyler, + "geoflow": geoflow, + "roofer": roofer, + "version": version, +} + +RESOURCES_PROD = { + "gdal": gdal, + "file_store": file_store_gilfoyle, + "file_store_fastssd": file_store_gilfoyle_fastssd, + "db_connection": db_connection, + "pdal": pdal, + "lastools": lastools, + "tyler": tyler, + "geoflow": geoflow, + "roofer": roofer, + "version": version, +} + +RESOURCES_DEFAULT = { + "gdal": GDALResource(), + "file_store": FileStoreResource(), + "file_store_fastssd": FileStoreResource(), + "db_connection": DatabaseResource(), + "pdal": PDALResource(), + "lastools": LASToolsResource(), + "tyler": TylerResource(), + "geoflow": GeoflowResource(), + "roofer": RooferResource(), + "version": VersionResource(), +} + + +resource_defs_by_env_name = { + "prod": RESOURCES_PROD, + "local": RESOURCES_LOCAL, + "test": RESOURCES_TEST, + "default": RESOURCES_DEFAULT, +} + +env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() +print(f"env_name={env_name}") +if env_name not in resource_defs_by_env_name: + logger.warning(f"Invalid environment: {env_name}, setting to default") + env_name = "default" + +resource_defs = resource_defs_by_env_name[env_name] From 7ccf40a68d48bdd9ca85f12f0028c458a830c78d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 20:35:42 +0100 Subject: [PATCH 09/40] env name --- packages/common/src/bag3d/common/resources/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 5f1ec704..60fd4e20 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -137,7 +137,6 @@ } env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() -print(f"env_name={env_name}") if env_name not in resource_defs_by_env_name: logger.warning(f"Invalid environment: {env_name}, setting to default") env_name = "default" From dd7528a6727fa540589f36f8233782ebe93f63ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 9 Dec 2024 20:46:00 +0100 Subject: [PATCH 10/40] env name --- .../src/bag3d/common/resources/__init__.py | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 60fd4e20..fb223673 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -75,7 +75,7 @@ ) -RESOURCES_LOCAL = { +resource_defs = { "gdal": gdal, "file_store": file_store, "file_store_fastssd": file_store_fastssd, @@ -89,56 +89,56 @@ } -RESOURCES_TEST = { - "gdal": gdal, - "file_store": file_store_test, - "file_store_fastssd": file_store_fastssd_test, - "db_connection": db_connection, - "pdal": pdal, - "lastools": lastools, - "tyler": tyler, - "geoflow": geoflow, - "roofer": roofer, - "version": version, -} - -RESOURCES_PROD = { - "gdal": gdal, - "file_store": file_store_gilfoyle, - "file_store_fastssd": file_store_gilfoyle_fastssd, - "db_connection": db_connection, - "pdal": pdal, - "lastools": lastools, - "tyler": tyler, - "geoflow": geoflow, - "roofer": roofer, - "version": version, -} - -RESOURCES_DEFAULT = { - "gdal": GDALResource(), - "file_store": FileStoreResource(), - "file_store_fastssd": FileStoreResource(), - "db_connection": DatabaseResource(), - "pdal": PDALResource(), - "lastools": LASToolsResource(), - "tyler": TylerResource(), - "geoflow": GeoflowResource(), - "roofer": RooferResource(), - "version": VersionResource(), -} - - -resource_defs_by_env_name = { - "prod": RESOURCES_PROD, - "local": RESOURCES_LOCAL, - "test": RESOURCES_TEST, - "default": RESOURCES_DEFAULT, -} - -env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() -if env_name not in resource_defs_by_env_name: - logger.warning(f"Invalid environment: {env_name}, setting to default") - env_name = "default" - -resource_defs = resource_defs_by_env_name[env_name] +# RESOURCES_TEST = { +# "gdal": gdal, +# "file_store": file_store_test, +# "file_store_fastssd": file_store_fastssd_test, +# "db_connection": db_connection, +# "pdal": pdal, +# "lastools": lastools, +# "tyler": tyler, +# "geoflow": geoflow, +# "roofer": roofer, +# "version": version, +# } +# +# RESOURCES_PROD = { +# "gdal": gdal, +# "file_store": file_store_gilfoyle, +# "file_store_fastssd": file_store_gilfoyle_fastssd, +# "db_connection": db_connection, +# "pdal": pdal, +# "lastools": lastools, +# "tyler": tyler, +# "geoflow": geoflow, +# "roofer": roofer, +# "version": version, +# } +# +# RESOURCES_DEFAULT = { +# "gdal": GDALResource(), +# "file_store": FileStoreResource(), +# "file_store_fastssd": FileStoreResource(), +# "db_connection": DatabaseResource(), +# "pdal": PDALResource(), +# "lastools": LASToolsResource(), +# "tyler": TylerResource(), +# "geoflow": GeoflowResource(), +# "roofer": RooferResource(), +# "version": VersionResource(), +# } +# +# +# resource_defs_by_env_name = { +# "prod": RESOURCES_PROD, +# "local": RESOURCES_LOCAL, +# "test": RESOURCES_TEST, +# "default": RESOURCES_DEFAULT, +# } +# +# env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower() +# if env_name not in resource_defs_by_env_name: +# logger.warning(f"Invalid environment: {env_name}, setting to default") +# env_name = "default" +# +# resource_defs = resource_defs_by_env_name[env_name] From 412c50c79020173d74e30ee5103ee8d2a0ca36f7 Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Mon, 9 Dec 2024 22:43:52 +0100 Subject: [PATCH 11/40] fix syntax error --- .../src/bag3d/core/assets/reconstruction/reconstruction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index 2e776e69..6c2c0815 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -285,9 +285,9 @@ def create_roofer_config( footprint_file=f"PG:{context.resources.db_connection.connect.dsn} tables={tile_view}", ahn3_files=laz_files_ahn3, ahn4_files=laz_files_ahn4, - ahn5_files=ahn_dir( + ahn5_files=[ahn_dir( context.resources.file_store.file_store.data_dir, ahn_version=5 - ).joinpath("as_downloaded/LAZ"), + ).joinpath("as_downloaded/LAZ")], output_path=output_dir, ) path_toml = output_dir / "roofer.toml" From cd62acf923f004ed33cbdd701b52171acad1d138 Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Tue, 10 Dec 2024 09:38:01 +0100 Subject: [PATCH 12/40] log roofer toml file --- .../core/src/bag3d/core/assets/reconstruction/reconstruction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index 6c2c0815..b234ac48 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -290,6 +290,7 @@ def create_roofer_config( ).joinpath("as_downloaded/LAZ")], output_path=output_dir, ) + context.log.info(output_toml) path_toml = output_dir / "roofer.toml" with path_toml.open("w") as of: of.write(output_toml) From c6920298768bbe72ae46ff722d4924f0a6f04f2b Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Tue, 10 Dec 2024 09:41:25 +0100 Subject: [PATCH 13/40] convert posixpath to str --- .../src/bag3d/core/assets/reconstruction/reconstruction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index b234ac48..36dcd629 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -285,9 +285,9 @@ def create_roofer_config( footprint_file=f"PG:{context.resources.db_connection.connect.dsn} tables={tile_view}", ahn3_files=laz_files_ahn3, ahn4_files=laz_files_ahn4, - ahn5_files=[ahn_dir( + ahn5_files=[str(ahn_dir( context.resources.file_store.file_store.data_dir, ahn_version=5 - ).joinpath("as_downloaded/LAZ")], + ).joinpath("as_downloaded/LAZ"))], output_path=output_dir, ) context.log.info(output_toml) From f43bcc2a6ccb92683f8a1f3b83e981dcf11176ca Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Tue, 10 Dec 2024 09:46:08 +0100 Subject: [PATCH 14/40] remove log of toml contents --- .../core/src/bag3d/core/assets/reconstruction/reconstruction.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index 36dcd629..4195f276 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -290,7 +290,6 @@ def create_roofer_config( ).joinpath("as_downloaded/LAZ"))], output_path=output_dir, ) - context.log.info(output_toml) path_toml = output_dir / "roofer.toml" with path_toml.open("w") as of: of.write(output_toml) From 563d8b301b3a23559cf0985817788f229d644d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Tue, 10 Dec 2024 10:04:10 +0100 Subject: [PATCH 15/40] ruff fix --- .../assets/reconstruction/reconstruction.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index 4195f276..31eb771f 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -249,15 +249,6 @@ def create_roofer_config( laz_files_ahn4 = [ str(out_dir_ahn4 / f"t_{tile_id_ahn[0]}.laz") for tile_id_ahn in res ] - if dir_tiles_200m_ahn5 is not None: - out_dir_ahn5 = Path(dir_tiles_200m_ahn5) - else: - out_dir_ahn5 = ahn_dir( - context.resources.file_store.file_store.data_dir, ahn_version=5 - ).joinpath("tiles_200m") - laz_files_ahn5 = [ - str(out_dir_ahn5 / f"t_{tile_id_ahn[0]}.laz") for tile_id_ahn in res - ] # Would be neater if we could use -sql in the OGR connection to do this query, # instead of creating a view. tile_view = PostgresTableIdentifier(tiles.schema, f"t_{tile_id}") @@ -285,9 +276,13 @@ def create_roofer_config( footprint_file=f"PG:{context.resources.db_connection.connect.dsn} tables={tile_view}", ahn3_files=laz_files_ahn3, ahn4_files=laz_files_ahn4, - ahn5_files=[str(ahn_dir( - context.resources.file_store.file_store.data_dir, ahn_version=5 - ).joinpath("as_downloaded/LAZ"))], + ahn5_files=[ + str( + ahn_dir( + context.resources.file_store.file_store.data_dir, ahn_version=5 + ).joinpath("as_downloaded/LAZ") + ) + ], output_path=output_dir, ) path_toml = output_dir / "roofer.toml" From aac8e5a5a8ddd600ec95bb06a8ce4d673e48c678 Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Wed, 11 Dec 2024 21:26:05 +0100 Subject: [PATCH 16/40] set cj-scale to 0.001 --- .../core/src/bag3d/core/assets/reconstruction/reconstruction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py index 31eb771f..0e5bbd59 100644 --- a/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py +++ b/packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py @@ -162,6 +162,7 @@ def create_roofer_config( split-cjseq = true omit-metadata = true cj-translate = [171800.0,472700.0,0.0] + cj-scale = [0.001, 0.001, 0.001] output-directory = "{output_path}" [[pointclouds]] From 226e6c56f3c5ef37af54d18c202d79f671dd6fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 12 Dec 2024 06:45:01 +0100 Subject: [PATCH 17/40] Update images --- docker/pipeline/bag3d-core.dockerfile | 2 +- docker/pipeline/bag3d-floors-estimation.dockerfile | 2 +- docker/pipeline/bag3d-party-walls.dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index b8b73094..bf485f19 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.06 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.12 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 378aa458..06be6e7d 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.06 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.12 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index 9c4dddd5..2297387f 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.06 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.12 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " From e8927ab522436c4c1b57b949b4de9a0cb3f06ddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 12 Dec 2024 10:19:04 +0100 Subject: [PATCH 18/40] Parameterize container name --- docker/compose.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/compose.yaml b/docker/compose.yaml index 9a6c1d95..fc7e39a0 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -4,7 +4,7 @@ services: # and event log storage. dagster-postgresql: image: postgres:16 - container_name: bag3d-dagster-postgresql + container_name: bag3d-dagster-postgresql-${BAG3D_DOCKER_IMAGE_TAG} environment: POSTGRES_USER: $DAGSTER_POSTGRES_USER POSTGRES_PASSWORD: $DAGSTER_POSTGRES_PASSWORD @@ -24,7 +24,7 @@ services: - pg_db=$BAG3D_PG_DATABASE command: ["-c", "shared_buffers=8GB", "-c", "max_connections=200", "-c", "work_mem=30MB", "-c", "logging_collector=on", "-c", "log_statement=all", "-c", "log_directory=/var/log/postgresql/"] platform: linux/amd64 - container_name: bag3d-data-postgresql + container_name: bag3d-data-postgresql-${BAG3D_DOCKER_IMAGE_TAG} healthcheck: test: ["CMD-SHELL", "pg_isready -d ${BAG3D_PG_DATABASE}"] interval: 3s @@ -62,9 +62,9 @@ services: dockerfile: docker/pipeline/bag3d-core.dockerfile args: - VERSION=develop - container_name: 3dbag-pipeline-core image: 3dbag-pipeline-core:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build + container_name: 3dbag-pipeline-core-${BAG3D_DOCKER_IMAGE_TAG} restart: always environment: DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-core:${BAG3D_DOCKER_IMAGE_TAG}" @@ -106,9 +106,9 @@ services: dockerfile: docker/pipeline/bag3d-floors-estimation.dockerfile args: - VERSION=develop - container_name: 3dbag-pipeline-floors-estimation image: 3dbag-pipeline-floors-estimation:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build + container_name: 3dbag-pipeline-floors-estimation-${BAG3D_DOCKER_IMAGE_TAG} restart: always environment: DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-floors-estimation:${BAG3D_DOCKER_IMAGE_TAG}" @@ -137,15 +137,16 @@ services: - action: rebuild path: ../packages/floors_estimation/pyproject.toml target: /opt/3dbag-pipeline/packages/floors_estimation/pyproject.toml + bag3d-party-walls: build: context: ../ dockerfile: docker/pipeline/bag3d-party-walls.dockerfile args: - VERSION=develop - container_name: 3dbag-pipeline-party-walls image: 3dbag-pipeline-party-walls:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build + container_name: 3dbag-pipeline-party-walls-${BAG3D_DOCKER_IMAGE_TAG} restart: always environment: DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-party-walls:${BAG3D_DOCKER_IMAGE_TAG}" @@ -183,6 +184,7 @@ services: context: ./dagster image: 3dgi/3dbag-pipeline-dagster:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build + container_name: bag3d-dagster-webserver-${BAG3D_DOCKER_IMAGE_TAG} entrypoint: - dagster-webserver - -h @@ -191,7 +193,6 @@ services: - "3000" - -w - workspace.yaml - container_name: bag3d-dagster-webserver expose: - "3000" ports: @@ -216,10 +217,10 @@ services: context: ./dagster image: 3dgi/3dbag-pipeline-dagster:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build + container_name: bag3d-dagster-daemon-${BAG3D_DOCKER_IMAGE_TAG} entrypoint: - dagster-daemon - run - container_name: bag3d-dagster-daemon restart: on-failure env_file: - ./.env From 7438cca340679c0fea16137e0f4c0710a9d0a418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 12 Dec 2024 10:21:02 +0100 Subject: [PATCH 19/40] Remove amd platform from data-postgresql It seems that was creating the startup issues --- docker/compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/compose.yaml b/docker/compose.yaml index fc7e39a0..3bae08af 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -23,7 +23,6 @@ services: - pg_pswd=$BAG3D_PG_PASSWORD - pg_db=$BAG3D_PG_DATABASE command: ["-c", "shared_buffers=8GB", "-c", "max_connections=200", "-c", "work_mem=30MB", "-c", "logging_collector=on", "-c", "log_statement=all", "-c", "log_directory=/var/log/postgresql/"] - platform: linux/amd64 container_name: bag3d-data-postgresql-${BAG3D_DOCKER_IMAGE_TAG} healthcheck: test: ["CMD-SHELL", "pg_isready -d ${BAG3D_PG_DATABASE}"] From 7d96fdb475425e8ca61a9613467785ca81bea068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 12 Dec 2024 12:41:12 +0100 Subject: [PATCH 20/40] Fix healthchecks --- docker/compose.yaml | 57 +++++++++++++++++++++++++------- scripts/monitor-code-location.py | 46 ++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 scripts/monitor-code-location.py diff --git a/docker/compose.yaml b/docker/compose.yaml index 3bae08af..da1f3c4f 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -5,6 +5,10 @@ services: dagster-postgresql: image: postgres:16 container_name: bag3d-dagster-postgresql-${BAG3D_DOCKER_IMAGE_TAG} + healthcheck: + test: [ "CMD-SHELL", "pg_isready -d ${DAGSTER_POSTGRES_DB} -U ${DAGSTER_POSTGRES_USER}" ] + interval: 3s + retries: 10 environment: POSTGRES_USER: $DAGSTER_POSTGRES_USER POSTGRES_PASSWORD: $DAGSTER_POSTGRES_PASSWORD @@ -25,11 +29,9 @@ services: command: ["-c", "shared_buffers=8GB", "-c", "max_connections=200", "-c", "work_mem=30MB", "-c", "logging_collector=on", "-c", "log_statement=all", "-c", "log_directory=/var/log/postgresql/"] container_name: bag3d-data-postgresql-${BAG3D_DOCKER_IMAGE_TAG} healthcheck: - test: ["CMD-SHELL", "pg_isready -d ${BAG3D_PG_DATABASE}"] + test: ["CMD-SHELL", "pg_isready -d ${BAG3D_PG_DATABASE} -U {BAG3D_PG_USER}"] interval: 3s - retries: 10 - start_period: 30s - timeout: 10s + retries: 20 shm_size: '1gb' ports: - "${BAG3D_PG_PORT_ON_HOST}:5432" @@ -64,6 +66,11 @@ services: image: 3dbag-pipeline-core:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build container_name: 3dbag-pipeline-core-${BAG3D_DOCKER_IMAGE_TAG} + # the port is built into the image + healthcheck: + test: ["CMD-SHELL", "dagster api grpc-health-check -p 4000"] + interval: 2s + retries: 10 restart: always environment: DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-core:${BAG3D_DOCKER_IMAGE_TAG}" @@ -108,6 +115,11 @@ services: image: 3dbag-pipeline-floors-estimation:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build container_name: 3dbag-pipeline-floors-estimation-${BAG3D_DOCKER_IMAGE_TAG} + # the port is built into the image + healthcheck: + test: [ "CMD-SHELL", "dagster api grpc-health-check -p 4001" ] + interval: 2s + retries: 10 restart: always environment: DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-floors-estimation:${BAG3D_DOCKER_IMAGE_TAG}" @@ -146,6 +158,11 @@ services: image: 3dbag-pipeline-party-walls:${BAG3D_DOCKER_IMAGE_TAG} pull_policy: build container_name: 3dbag-pipeline-party-walls-${BAG3D_DOCKER_IMAGE_TAG} + # the port is built into the image + healthcheck: + test: [ "CMD-SHELL", "dagster api grpc-health-check -p 4002" ] + interval: 2s + retries: 10 restart: always environment: DAGSTER_CURRENT_IMAGE: "3dbag-pipeline-party-walls:${BAG3D_DOCKER_IMAGE_TAG}" @@ -204,10 +221,18 @@ services: networks: - bag3d-network depends_on: - - dagster-postgresql - - bag3d-core - - bag3d-floors-estimation - - bag3d-party-walls + dagster-postgresql: + condition: service_healthy + restart: true + bag3d-core: + condition: service_healthy + restart: true + bag3d-floors-estimation: + condition: service_healthy + restart: true + bag3d-party-walls: + condition: service_healthy + restart: true # This service runs the dagster-daemon process, which is responsible for taking runs # off of the queue and launching them, as well as creating runs from schedules or sensors. @@ -229,10 +254,18 @@ services: networks: - bag3d-network depends_on: - - dagster-postgresql - - bag3d-core - - bag3d-floors-estimation - - bag3d-party-walls + dagster-postgresql: + condition: service_healthy + restart: true + bag3d-core: + condition: service_healthy + restart: true + bag3d-floors-estimation: + condition: service_healthy + restart: true + bag3d-party-walls: + condition: service_healthy + restart: true volumes: bag3d-data-postgresql: diff --git a/scripts/monitor-code-location.py b/scripts/monitor-code-location.py new file mode 100644 index 00000000..9d83fc5e --- /dev/null +++ b/scripts/monitor-code-location.py @@ -0,0 +1,46 @@ +"""Check if a code location has loaded without errors.""" + +import argparse +import json + +import requests + +parser = argparse.ArgumentParser() +parser.add_argument("--host", help="dagster host", default="localhost") +parser.add_argument("--port", help="dagster port", type=int, default=3000) +parser.add_argument("--code-location", help="dagster code location", default="core") + +QUERY_CODE_LOCATIONS = """ +query LocationStatusesQuery($codeLocationName: String!) { + workspaceLocationEntryOrError(name: $codeLocationName) { + ... on WorkspaceLocationEntry { + id + name + loadStatus + locationOrLoadError + } + } +} +""" + +if __name__ == "__main__": + args = parser.parse_args() + + variables = { + "codeLocationName": args.code_location, + } + reload_res = requests.post( + "http://{dagster_host}:{dagster_port}/graphql?query={query_string}&variables={variables}".format( + dagster_host=args.host, + dagster_port=args.port, + query_string=QUERY_CODE_LOCATIONS, + variables=json.dumps(variables), + ) + ).json() + + location_entry = reload_res["data"]["workspaceLocationEntry"] + load_status = location_entry.get("loadStatus") + location_or_error = location_entry.get("locationOrError").get("__typename") + is_loaded = load_status == "LOADED" and location_or_error == "RepositoryLocation" + + exit(not is_loaded) From 07f99463bb4a86b441efb24731ad97ff273df405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 12 Dec 2024 12:41:23 +0100 Subject: [PATCH 21/40] Fix env vars in docker --- docker/.env | 6 +++--- docker/pipeline/bag3d-core.dockerfile | 3 +-- docker/pipeline/bag3d-floors-estimation.dockerfile | 3 +-- docker/pipeline/bag3d-party-walls.dockerfile | 3 +-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docker/.env b/docker/.env index 3845e3b7..024eb99e 100644 --- a/docker/.env +++ b/docker/.env @@ -12,9 +12,9 @@ TEMP_CONTAINER="${COMPOSE_PROJECT_NAME}-temp-container" # Dagster instance configuration DAGSTER_HOME="/opt/dagster/dagster_home" DAGSTER_SERVER_PORT=3003 -DAGSTER_POSTGRES_USER="postgres_user" -DAGSTER_POSTGRES_PASSWORD="postgres_password" -DAGSTER_POSTGRES_DB="postgres_db" +DAGSTER_POSTGRES_USER="dagster" +DAGSTER_POSTGRES_PASSWORD="dagster_pwd" +DAGSTER_POSTGRES_DB="dagster" # 3dbag-pipeline resource configuration DAGSTER_ENVIRONMENT="test" diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index bf485f19..d472b6f1 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -20,7 +20,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --python $VIRTUAL_ENV/bin/python COPY . $BAG3D_PIPELINE_LOCATION -COPY ./docker/.env $BAG3D_PIPELINE_LOCATION/.env # Install the workflow package RUN --mount=type=cache,target=/root/.cache/uv \ @@ -32,4 +31,4 @@ EXPOSE 4000 # CMD allows this to be overridden from run launchers or executors that want # to run other commands against your repository -CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "bag3d.core.code_location", "--inject-env-vars-from-instance"] +CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-m", "bag3d.core.code_location"] diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 06be6e7d..5337920b 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -20,7 +20,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --python $VIRTUAL_ENV/bin/python COPY . $BAG3D_PIPELINE_LOCATION -COPY ./docker/.env $BAG3D_PIPELINE_LOCATION/.env # Install the workflow package RUN --mount=type=cache,target=/root/.cache/uv \ @@ -32,4 +31,4 @@ EXPOSE 4001 # CMD allows this to be overridden from run launchers or executors that want # to run other commands against your repository -CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4001", "-m", "bag3d.floors_estimation.code_location", "--inject-env-vars-from-instance"] +CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4001", "-m", "bag3d.floors_estimation.code_location"] diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index 2297387f..7e50c7d5 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -33,7 +33,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --python $VIRTUAL_ENV/bin/python COPY . $BAG3D_PIPELINE_LOCATION -COPY ./docker/.env $BAG3D_PIPELINE_LOCATION/.env # Install the workflow package RUN --mount=type=cache,target=/root/.cache/uv \ @@ -45,4 +44,4 @@ EXPOSE 4002 # CMD allows this to be overridden from run launchers or executors that want # to run other commands against your repository -CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4002", "-m", "bag3d.party_walls.code_location", "--inject-env-vars-from-instance"] +CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4002", "-m", "bag3d.party_walls.code_location"] From 99f9fecbc7f8e562997bd3cd1a760c5d585becaa Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Fri, 13 Dec 2024 08:59:30 +0100 Subject: [PATCH 22/40] capitalise AHN in attribute names, add AHN5 attributes --- .../src/bag3d/core/assets/export/metadata.py | 18 ++++--- .../bag3d/core/sqlfiles/webservice_lod.sql | 50 +++++++++---------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/export/metadata.py b/packages/core/src/bag3d/core/assets/export/metadata.py index 956c0c7c..594e004c 100644 --- a/packages/core/src/bag3d/core/assets/export/metadata.py +++ b/packages/core/src/bag3d/core/assets/export/metadata.py @@ -99,13 +99,17 @@ def feature_evaluation(context): attributes_to_include = ( "b3_pw_selectie_reden", "b3_pw_bron", - "b3_puntdichtheid_ahn3", - "b3_puntdichtheid_ahn4", - "b3_mutatie_ahn3_ahn4", - "b3_nodata_fractie_ahn3", - "b3_nodata_fractie_ahn4", - "b3_nodata_radius_ahn3", - "b3_nodata_radius_ahn4", + "b3_puntdichtheid_AHN3", + "b3_puntdichtheid_AHN4", + "b3_puntdichtheid_AHN5", + "b3_mutatie_AHN3_AHN4", + "b3_mutatie_AHN4_AHN5", + "b3_nodata_fractie_AHN3", + "b3_nodata_fractie_AHN4", + "b3_nodata_fractie_AHN5", + "b3_nodata_radius_AHN3", + "b3_nodata_radius_AHN4", + "b3_nodata_radius_AHN5", ) cityobject_info = {lod: 0 for lod in lods} cityobject_info["has_geometry"] = False diff --git a/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql b/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql index a0eb8bad..8675771d 100644 --- a/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql +++ b/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql @@ -36,9 +36,9 @@ SELECT p.b3_opp_dak_schuin, p.b3_opp_grond, p.b3_opp_scheidingsmuur, - p.b3_puntdichtheid_ahn3, - p.b3_puntdichtheid_ahn4, - p.b3_puntdichtheid_ahn5, + p.b3_puntdichtheid_AHN3, + p.b3_puntdichtheid_AHN4, + p.b3_puntdichtheid_AHN5, p.b3_pw_bron, p.b3_pw_datum, p.b3_pw_selectie_reden, @@ -89,22 +89,22 @@ SELECT p.b3_dak_type, p.b3_h_maaiveld, p.b3_kas_warenhuis, - p.b3_mutatie_ahn3_ahn4, - p.b3_mutatie_ahn4_ahn5, - p.b3_nodata_fractie_ahn3, - p.b3_nodata_fractie_ahn4, - p.b3_nodata_fractie_ahn5, - p.b3_nodata_radius_ahn3, - p.b3_nodata_radius_ahn4, - p.b3_nodata_radius_ahn5, + p.b3_mutatie_AHN3_AHN4, + p.b3_mutatie_AHN4_AHN5, + p.b3_nodata_fractie_AHN3, + p.b3_nodata_fractie_AHN4, + p.b3_nodata_fractie_AHN5, + p.b3_nodata_radius_AHN3, + p.b3_nodata_radius_AHN4, + p.b3_nodata_radius_AHN5, p.b3_opp_buitenmuur, p.b3_opp_dak_plat, p.b3_opp_dak_schuin, p.b3_opp_grond, p.b3_opp_scheidingsmuur, - p.b3_puntdichtheid_ahn3, - p.b3_puntdichtheid_ahn4, - p.b3_puntdichtheid_ahn5, + p.b3_puntdichtheid_AHN3, + p.b3_puntdichtheid_AHN4, + p.b3_puntdichtheid_AHN5, p.b3_pw_bron, p.b3_pw_datum, p.b3_pw_selectie_reden, @@ -155,22 +155,22 @@ SELECT p.b3_dak_type, p.b3_h_maaiveld, p.b3_kas_warenhuis, - p.b3_mutatie_ahn3_ahn4, - p.b3_mutatie_ahn4_ahn5, - p.b3_nodata_fractie_ahn3, - p.b3_nodata_fractie_ahn4, - p.b3_nodata_fractie_ahn5, - p.b3_nodata_radius_ahn3, - p.b3_nodata_radius_ahn4, - p.b3_nodata_radius_ahn5, + p.b3_mutatie_AHN3_AHN4, + p.b3_mutatie_AHN4_AHN5, + p.b3_nodata_fractie_AHN3, + p.b3_nodata_fractie_AHN4, + p.b3_nodata_fractie_AHN5, + p.b3_nodata_radius_AHN3, + p.b3_nodata_radius_AHN4, + p.b3_nodata_radius_AHN5, p.b3_opp_buitenmuur, p.b3_opp_dak_plat, p.b3_opp_dak_schuin, p.b3_opp_grond, p.b3_opp_scheidingsmuur, - p.b3_puntdichtheid_ahn3, - p.b3_puntdichtheid_ahn4, - p.b3_puntdichtheid_ahn5, + p.b3_puntdichtheid_AHN3, + p.b3_puntdichtheid_AHN4, + p.b3_puntdichtheid_AHN5, p.b3_pw_bron, p.b3_pw_datum, p.b3_pw_selectie_reden, From 5cc3a52e08993eaec0c602dd3e78e280eb9b0717 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 13 Dec 2024 10:05:40 +0100 Subject: [PATCH 23/40] Capitalized AHN in webservice_lod --- .../src/bag3d/core/sqlfiles/webservice_lod.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql b/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql index 8675771d..0cce8c0e 100644 --- a/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql +++ b/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql @@ -23,14 +23,14 @@ SELECT p.b3_dak_type, p.b3_h_maaiveld, p.b3_kas_warenhuis, - p.b3_mutatie_ahn3_ahn4, - p.b3_mutatie_ahn4_ahn5, - p.b3_nodata_fractie_ahn3, - p.b3_nodata_fractie_ahn4, - p.b3_nodata_fractie_ahn5, - p.b3_nodata_radius_ahn3, - p.b3_nodata_radius_ahn4, - p.b3_nodata_radius_ahn5, + p.b3_mutatie_AHN3_AHN4, + p.b3_mutatie_AHN4_AHN5, + p.b3_nodata_fractie_AHN3, + p.b3_nodata_fractie_AHN4, + p.b3_nodata_fractie_AHN5, + p.b3_nodata_radius_AHN3, + p.b3_nodata_radius_AHN4, + p.b3_nodata_radius_AHN5, p.b3_opp_buitenmuur, p.b3_opp_dak_plat, p.b3_opp_dak_schuin, From 2de9a7dd89bab669a29dc75909c54f9e4bdf644c Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 13 Dec 2024 10:05:59 +0100 Subject: [PATCH 24/40] missing attributes in floor_estimation --- .../floors_estimation/assets/Attributes.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py b/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py index 220e4768..114f400e 100644 --- a/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py +++ b/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py @@ -5,6 +5,12 @@ @dataclass class Attributes: b3_bag_bag_overlap: float + b3_n_vlakken: int + b3_pw_onvoldoende: bool + b3_t_run: int + b3_succes: bool + b3_extrusie: int + b3_is_glas_dak: bool b3_dak_type: str b3_h_dak_50p: float b3_h_dak_70p: float @@ -12,13 +18,17 @@ class Attributes: b3_h_dak_min: float b3_h_maaiveld: float b3_kas_warenhuis: bool - b3_mutatie_ahn3_ahn4: bool - b3_nodata_fractie_ahn3: float - b3_nodata_fractie_ahn4: float - b3_nodata_radius_ahn3: float - b3_nodata_radius_ahn4: float - b3_puntdichtheid_ahn3: float - b3_puntdichtheid_ahn4: float + b3_mutatie_AHN3_AHN4: bool + b3_mutatie_AHN4_AHN5: bool + b3_nodata_fractie_AHN3: float + b3_nodata_fractie_AHN4: float + b3_nodata_fractie_AHN5: float + b3_nodata_radius_AHN3: float + b3_nodata_radius_AHN4: float + b3_nodata_radius_AHN5: float + b3_puntdichtheid_AHN3: float + b3_puntdichtheid_AHN4: float + b3_puntdichtheid_AHN5: float b3_pw_bron: str b3_pw_datum: int b3_pw_selectie_reden: str From 11b3bf5f464f9512b57ae218ce916c12f1b5e45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Fri, 13 Dec 2024 13:39:48 +0100 Subject: [PATCH 25/40] Update tools image --- docker/pipeline/bag3d-core.dockerfile | 2 +- docker/pipeline/bag3d-floors-estimation.dockerfile | 2 +- docker/pipeline/bag3d-party-walls.dockerfile | 2 +- docker/tools/Dockerfile | 14 +++++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index d472b6f1..634ca142 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.12 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.13 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 5337920b..3199e887 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.12 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.13 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index 7e50c7d5..dd1fb5a0 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.12 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.13 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index 8f9b5560..7634a035 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -100,13 +100,13 @@ FROM roofer AS geoflow-old ARG JOBS=2 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /export /export -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/LASlib /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/gdalplugins /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /export /export +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/LASlib /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/gdalplugins /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins ENV GF_PLUGIN_FOLDER=$BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins RUN echo /export/usr/local/lib >> /etc/ld.so.conf.d/geoflow.conf; \ From 8c4110c3f574f5810431a8e53a7be915ed384cad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Fri, 13 Dec 2024 14:04:15 +0100 Subject: [PATCH 26/40] GH action tools builder --- .github/workflows/build-docker-tools.yaml | 39 +++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/build-docker-tools.yaml diff --git a/.github/workflows/build-docker-tools.yaml b/.github/workflows/build-docker-tools.yaml new file mode 100644 index 00000000..6e19b247 --- /dev/null +++ b/.github/workflows/build-docker-tools.yaml @@ -0,0 +1,39 @@ +name: Build and Publish The Tools Docker Image + +on: + push: + tags: + - '*' + branches: + - "develop" + - "pc-*" + paths: + - docker/tools/Dockerfile + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set docker image tag + run: echo "DOCKER_TAG=$(date +'%Y.%m.%d')" >> $GITHUB_ENV + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push tools image + uses: docker/build-push-action@v6 + with: + context: ./ + file: ./docker/tools/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + build-args: | + JOBS=2 + VERSION=${{ env.DOCKER_TAG }} + push: true + tags: 3dgi/3dbag-pipeline-tools:${{ env.DOCKER_TAG }} + From d904127d0b85cc11b10d6c6c660f9091426eb530 Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Sat, 14 Dec 2024 11:17:52 +0100 Subject: [PATCH 27/40] update images --- docker/pipeline/bag3d-core.dockerfile | 2 +- docker/pipeline/bag3d-floors-estimation.dockerfile | 2 +- docker/pipeline/bag3d-party-walls.dockerfile | 2 +- docker/tools/Dockerfile | 14 +++++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index 634ca142..2a3184e0 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.13 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 3199e887..9870974e 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.13 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index dd1fb5a0..860dd64f 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.13 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index 7634a035..b6a9ea0d 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -100,13 +100,13 @@ FROM roofer AS geoflow-old ARG JOBS=2 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /export /export -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/LASlib /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/gdalplugins /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.13 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /export /export +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/LASlib /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/gdalplugins /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins ENV GF_PLUGIN_FOLDER=$BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins RUN echo /export/usr/local/lib >> /etc/ld.so.conf.d/geoflow.conf; \ From b1f8f07742b203ce146cf477ff0b605b8d9c2cfd Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Sat, 14 Dec 2024 11:46:55 +0100 Subject: [PATCH 28/40] increase tyler n threads --- packages/core/src/bag3d/core/assets/export/tile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/bag3d/core/assets/export/tile.py b/packages/core/src/bag3d/core/assets/export/tile.py index b061cbe4..93b3c1ca 100644 --- a/packages/core/src/bag3d/core/assets/export/tile.py +++ b/packages/core/src/bag3d/core/assets/export/tile.py @@ -44,7 +44,7 @@ def reconstruction_output_tiles_func(context, format: str, **kwargs: dict): ) # # Set the parallelism in tyler from the dagster instance configuration (the dagster.yaml in $DAGSTER_HOME) # num_threads = context.instance.run_coordinator.inst_data.config_dict["max_concurrent_runs"] - num_threads = 4 + num_threads = 40 cmd = [ f"RAYON_NUM_THREADS={num_threads}", "RUST_LOG=info", From 0aa928bf7459d4c8f992e3328d29a5f0d15760ea Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Sat, 14 Dec 2024 13:30:45 +0100 Subject: [PATCH 29/40] update images --- docker/pipeline/bag3d-core.dockerfile | 2 +- docker/pipeline/bag3d-floors-estimation.dockerfile | 2 +- docker/pipeline/bag3d-party-walls.dockerfile | 2 +- docker/tools/Dockerfile | 14 +++++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index 2a3184e0..6e0841df 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.15 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 9870974e..8b77b0d5 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.15 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index 860dd64f..a54bedab 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.15 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index b6a9ea0d..5b1fcdf8 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -100,13 +100,13 @@ FROM roofer AS geoflow-old ARG JOBS=2 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /export /export -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/LASlib /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/gdalplugins /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.14 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /export /export +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/LASlib /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/gdalplugins /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins ENV GF_PLUGIN_FOLDER=$BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins RUN echo /export/usr/local/lib >> /etc/ld.so.conf.d/geoflow.conf; \ From a85359b7baa0265a39943684028825d2de1819ac Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Sat, 14 Dec 2024 13:50:14 +0100 Subject: [PATCH 30/40] change back tools image date --- docker/pipeline/bag3d-core.dockerfile | 2 +- docker/pipeline/bag3d-floors-estimation.dockerfile | 2 +- docker/pipeline/bag3d-party-walls.dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index 6e0841df..2a3184e0 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.15 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 8b77b0d5..9870974e 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.15 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index a54bedab..860dd64f 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2024.12.15 AS develop +FROM 3dgi/3dbag-pipeline-tools:2024.12.14 AS develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline LABEL org.opencontainers.image.authors="Balázs Dukai " From d5291a17559fed4ce4c03e3d433eb604b8db45b9 Mon Sep 17 00:00:00 2001 From: Ravi Peters Date: Sat, 14 Dec 2024 17:52:55 +0100 Subject: [PATCH 31/40] bump geoflow-bundle version --- docker/tools/Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index 5b1fcdf8..6b0124b0 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -100,13 +100,13 @@ FROM roofer AS geoflow-old ARG JOBS=2 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /export /export -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/LASlib /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/gdalplugins /export/usr/local/lib/ -COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.15 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /export /export +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/LASlib /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/gdalplugins /export/usr/local/lib/ +COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.16 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins ENV GF_PLUGIN_FOLDER=$BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins RUN echo /export/usr/local/lib >> /etc/ld.so.conf.d/geoflow.conf; \ From 176b1507e6af5e3476d2b23b8a3b29590c1f0799 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Mon, 16 Dec 2024 13:54:03 +0100 Subject: [PATCH 32/40] fixed attributes --- .../bag3d/floors_estimation/assets/Attributes.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py b/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py index 114f400e..7c90d4fc 100644 --- a/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py +++ b/packages/floors_estimation/src/bag3d/floors_estimation/assets/Attributes.py @@ -5,21 +5,18 @@ @dataclass class Attributes: b3_bag_bag_overlap: float - b3_n_vlakken: int - b3_pw_onvoldoende: bool - b3_t_run: int - b3_succes: bool - b3_extrusie: int - b3_is_glas_dak: bool b3_dak_type: str + b3_extrusie: int b3_h_dak_50p: float b3_h_dak_70p: float b3_h_dak_max: float b3_h_dak_min: float b3_h_maaiveld: float + b3_is_glas_dak: bool b3_kas_warenhuis: bool b3_mutatie_AHN3_AHN4: bool b3_mutatie_AHN4_AHN5: bool + b3_n_vlakken: int b3_nodata_fractie_AHN3: float b3_nodata_fractie_AHN4: float b3_nodata_fractie_AHN5: float @@ -31,11 +28,13 @@ class Attributes: b3_puntdichtheid_AHN5: float b3_pw_bron: str b3_pw_datum: int + b3_pw_onvoldoende: bool b3_pw_selectie_reden: str - b3_reconstructie_onvolledig: bool b3_rmse_lod12: float b3_rmse_lod13: float b3_rmse_lod22: float + b3_succes: bool + b3_t_run: int b3_val3dity_lod12: str b3_val3dity_lod13: str b3_val3dity_lod22: str @@ -47,9 +46,11 @@ class Attributes: documentnummer: str eindgeldigheid: Optional[str] eindregistratie: Optional[str] + fid: int geconstateerd: bool identificatie: str oorspronkelijkbouwjaar: int + rf_force_lod11: bool status: str tijdstipeindregistratielv: str tijdstipinactief: str From ac9a0798d1d3eccd7e74453bc0d4c9f80ec92c67 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 17 Dec 2024 18:06:46 +0100 Subject: [PATCH 33/40] missing / --- packages/core/src/bag3d/core/assets/export/validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/export/validate.py b/packages/core/src/bag3d/core/assets/export/validate.py index 4d2903a7..3ef2b182 100644 --- a/packages/core/src/bag3d/core/assets/export/validate.py +++ b/packages/core/src/bag3d/core/assets/export/validate.py @@ -564,7 +564,7 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe "LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH", "/opt/bin/ogrinfo", sql_buildingpart_count, - f"/vsigzip/{inputzipfile}", + f"/vsigzip//{inputzipfile}", ] ) output, returncode = execute_shell_command_silent( @@ -585,7 +585,7 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe "LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH", "/opt/bin/ogrinfo", sql_building_count, - f"/vsigzip/{inputzipfile}", + f"/vsigzip//{inputzipfile}", ] ) output, returncode = execute_shell_command_silent( From 6bfeed4150e2c4ad74946ed5e4b79b2279bd3338 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 17 Dec 2024 18:20:22 +0100 Subject: [PATCH 34/40] logging --- packages/core/src/bag3d/core/assets/export/validate.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/core/src/bag3d/core/assets/export/validate.py b/packages/core/src/bag3d/core/assets/export/validate.py index 3ef2b182..7b10f151 100644 --- a/packages/core/src/bag3d/core/assets/export/validate.py +++ b/packages/core/src/bag3d/core/assets/export/validate.py @@ -6,7 +6,7 @@ import ast from dataclasses import dataclass, field -from dagster import asset, AssetIn, AssetKey +from dagster import asset, AssetIn, AssetKey, get_dagster_logger from bag3d.common.resources.executables import execute_shell_command_silent from bag3d.common.utils.files import bag3d_export_dir @@ -575,8 +575,12 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe ) re_buildingpart_count = r"(?<=count\(identificatie\) \(Integer\) = )\d+" try: + if returncode != 0: + get_dagster_logger().error(f"Ogrinfo failed: {output}") + raise Exception("ogrinfo failed") n = int(re.search(re_buildingpart_count, output).group(0)) nr_buildingpart_all.append(n) + except Exception: n = None @@ -595,6 +599,9 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe r"(?<=count\(distinct identificatie\) \(Integer\) = )\d+" ) try: + if returncode != 0: + get_dagster_logger().error(f"ogrinfo failed: {output}") + raise Exception("ogrinfo failed") n = int(re.search(re_building_count, output).group(0)) nr_building_all.append(n) except Exception: From b8e55f54839745506319c841e16ea0fe80404819 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 17 Dec 2024 18:42:07 +0100 Subject: [PATCH 35/40] logging --- packages/core/src/bag3d/core/assets/export/validate.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/export/validate.py b/packages/core/src/bag3d/core/assets/export/validate.py index 7b10f151..d6db0155 100644 --- a/packages/core/src/bag3d/core/assets/export/validate.py +++ b/packages/core/src/bag3d/core/assets/export/validate.py @@ -570,14 +570,13 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe output, returncode = execute_shell_command_silent( shell_command=cmd, cwd=str(dirpath) ) + get_dagster_logger().info(f"{output=}") results.file_ok = ( False if returncode != 0 or "error" in output.lower() else True ) re_buildingpart_count = r"(?<=count\(identificatie\) \(Integer\) = )\d+" + try: - if returncode != 0: - get_dagster_logger().error(f"Ogrinfo failed: {output}") - raise Exception("ogrinfo failed") n = int(re.search(re_buildingpart_count, output).group(0)) nr_buildingpart_all.append(n) @@ -595,13 +594,11 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe output, returncode = execute_shell_command_silent( shell_command=cmd, cwd=str(dirpath) ) + get_dagster_logger().info(f"{output=}") re_building_count = ( r"(?<=count\(distinct identificatie\) \(Integer\) = )\d+" ) try: - if returncode != 0: - get_dagster_logger().error(f"ogrinfo failed: {output}") - raise Exception("ogrinfo failed") n = int(re.search(re_building_count, output).group(0)) nr_building_all.append(n) except Exception: From 6afd5e40878b02bb2f30655048d8025550074e84 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 17 Dec 2024 22:37:07 +0100 Subject: [PATCH 36/40] validate update --- .../src/bag3d/core/assets/export/validate.py | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/export/validate.py b/packages/core/src/bag3d/core/assets/export/validate.py index d6db0155..82018e95 100644 --- a/packages/core/src/bag3d/core/assets/export/validate.py +++ b/packages/core/src/bag3d/core/assets/export/validate.py @@ -6,7 +6,7 @@ import ast from dataclasses import dataclass, field -from dagster import asset, AssetIn, AssetKey, get_dagster_logger +from dagster import asset, AssetIn, AssetKey, OpExecutionContext from bag3d.common.resources.executables import execute_shell_command_silent from bag3d.common.utils.files import bag3d_export_dir @@ -502,7 +502,13 @@ def obj( return results -def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileResults: +def gpkg( + context: OpExecutionContext, + dirpath: Path, + file_id: str, + url_root: str, + version: str, +) -> GPKGFileResults: results = { "gpkg_zip_ok": None, "gpkg_ok": None, @@ -552,6 +558,7 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe # ogrinfo nr_building_all = [] nr_buildingpart_all = [] + gdal = context.resources.gdal.app for layer in ["lod12_3d", "lod13_3d", "lod22_3d"]: try: sql_buildingpart_count = f"-sql 'select count(identificatie) from {layer}'" @@ -562,7 +569,7 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe cmd = " ".join( [ "LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH", - "/opt/bin/ogrinfo", + "{exe}", sql_buildingpart_count, f"/vsigzip//{inputzipfile}", ] @@ -570,12 +577,15 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe output, returncode = execute_shell_command_silent( shell_command=cmd, cwd=str(dirpath) ) - get_dagster_logger().info(f"{output=}") + returncode, output = gdal.execute( + "ogrinfo", command=cmd, local_path=str(dirpath) + ) results.file_ok = ( False if returncode != 0 or "error" in output.lower() else True ) + context.log.debug(f"{results.file_ok=}") re_buildingpart_count = r"(?<=count\(identificatie\) \(Integer\) = )\d+" - + try: n = int(re.search(re_buildingpart_count, output).group(0)) nr_buildingpart_all.append(n) @@ -586,15 +596,14 @@ def gpkg(dirpath: Path, file_id: str, url_root: str, version: str) -> GPKGFileRe cmd = " ".join( [ "LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH", - "/opt/bin/ogrinfo", + "{exe}", sql_building_count, f"/vsigzip//{inputzipfile}", ] ) - output, returncode = execute_shell_command_silent( - shell_command=cmd, cwd=str(dirpath) + returncode, output = gdal.execute( + "ogrinfo", command=cmd, local_path=str(dirpath) ) - get_dagster_logger().info(f"{output=}") re_building_count = ( r"(?<=count\(distinct identificatie\) \(Integer\) = )\d+" ) @@ -628,7 +637,7 @@ def create_download_link(url_root: str, format: str, file_id: str, version: str) return link -def check_formats(input) -> TileResults: +def check_formats(context: OpExecutionContext, input) -> TileResults: dirpath, tile_id, url_root, version = input file_id = tile_id.replace("/", "-") planarity_n_tol = 20.0 @@ -649,7 +658,7 @@ def check_formats(input) -> TileResults: url_root=url_root, version=version, ) - gpkg_results = gpkg(dirpath, file_id, url_root=url_root, version=version) + gpkg_results = gpkg(context, dirpath, file_id, url_root=url_root, version=version) return TileResults(tile_id, cj_results, obj_results, gpkg_results) @@ -659,7 +668,7 @@ def check_formats(input) -> TileResults: "metadata": AssetIn(key_prefix="export"), }, deps=[AssetKey(("export", "compressed_tiles"))], - required_resource_keys={"file_store", "version"}, + required_resource_keys={"file_store", "version", "gdal"}, ) def compressed_tiles_validation(context, export_index: Path, metadata: Path) -> Path: """Validates the compressed distribution tiles, for each format. @@ -694,7 +703,13 @@ def compressed_tiles_validation(context, export_index: Path, metadata: Path) -> csvreader = csv.reader(fo) _ = next(csvreader) # header tileids = [ - (path_export_dir.joinpath("tiles", row[0]), row[0], url_root, version) + ( + context, + path_export_dir.joinpath("tiles", row[0]), + row[0], + url_root, + version, + ) for row in csvreader ] From 7d507e7b8f2af5a4d2aab6de4698add52a1d1680 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 17 Dec 2024 22:49:55 +0100 Subject: [PATCH 37/40] AppImage --- .../core/src/bag3d/core/assets/export/validate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/export/validate.py b/packages/core/src/bag3d/core/assets/export/validate.py index 82018e95..135f8c9a 100644 --- a/packages/core/src/bag3d/core/assets/export/validate.py +++ b/packages/core/src/bag3d/core/assets/export/validate.py @@ -8,7 +8,7 @@ from dagster import asset, AssetIn, AssetKey, OpExecutionContext -from bag3d.common.resources.executables import execute_shell_command_silent +from bag3d.common.resources.executables import execute_shell_command_silent, AppImage from bag3d.common.utils.files import bag3d_export_dir @@ -503,7 +503,7 @@ def obj( def gpkg( - context: OpExecutionContext, + gdal: AppImage, dirpath: Path, file_id: str, url_root: str, @@ -558,7 +558,6 @@ def gpkg( # ogrinfo nr_building_all = [] nr_buildingpart_all = [] - gdal = context.resources.gdal.app for layer in ["lod12_3d", "lod13_3d", "lod22_3d"]: try: sql_buildingpart_count = f"-sql 'select count(identificatie) from {layer}'" @@ -637,7 +636,7 @@ def create_download_link(url_root: str, format: str, file_id: str, version: str) return link -def check_formats(context: OpExecutionContext, input) -> TileResults: +def check_formats(gdal: AppImage, input) -> TileResults: dirpath, tile_id, url_root, version = input file_id = tile_id.replace("/", "-") planarity_n_tol = 20.0 @@ -658,7 +657,7 @@ def check_formats(context: OpExecutionContext, input) -> TileResults: url_root=url_root, version=version, ) - gpkg_results = gpkg(context, dirpath, file_id, url_root=url_root, version=version) + gpkg_results = gpkg(gdal, dirpath, file_id, url_root=url_root, version=version) return TileResults(tile_id, cj_results, obj_results, gpkg_results) @@ -699,12 +698,13 @@ def compressed_tiles_validation(context, export_index: Path, metadata: Path) -> metadata_json = json.load(fo) version = metadata_json["identificationInfo"]["citation"]["edition"] context.log.debug(f"{version=}") + gdal = context.resources.gdal.app with export_index.open("r") as fo: csvreader = csv.reader(fo) _ = next(csvreader) # header tileids = [ ( - context, + gdal, path_export_dir.joinpath("tiles", row[0]), row[0], url_root, From 99913dea8ceae71e1ed0c3d4b8bcf25e343d3773 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 17 Dec 2024 22:54:00 +0100 Subject: [PATCH 38/40] gdal in input --- packages/core/src/bag3d/core/assets/export/validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/export/validate.py b/packages/core/src/bag3d/core/assets/export/validate.py index 135f8c9a..9b0863b8 100644 --- a/packages/core/src/bag3d/core/assets/export/validate.py +++ b/packages/core/src/bag3d/core/assets/export/validate.py @@ -636,8 +636,8 @@ def create_download_link(url_root: str, format: str, file_id: str, version: str) return link -def check_formats(gdal: AppImage, input) -> TileResults: - dirpath, tile_id, url_root, version = input +def check_formats(input) -> TileResults: + gdal, dirpath, tile_id, url_root, version = input file_id = tile_id.replace("/", "-") planarity_n_tol = 20.0 planarity_d2p_tol = 0.001 From 6ae0a8337bc4536c536b2c9b7510790e7562742e Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 17 Dec 2024 23:06:34 +0100 Subject: [PATCH 39/40] removed context log --- packages/core/src/bag3d/core/assets/export/validate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/export/validate.py b/packages/core/src/bag3d/core/assets/export/validate.py index 9b0863b8..6a08e3a4 100644 --- a/packages/core/src/bag3d/core/assets/export/validate.py +++ b/packages/core/src/bag3d/core/assets/export/validate.py @@ -582,7 +582,6 @@ def gpkg( results.file_ok = ( False if returncode != 0 or "error" in output.lower() else True ) - context.log.debug(f"{results.file_ok=}") re_buildingpart_count = r"(?<=count\(identificatie\) \(Integer\) = )\d+" try: @@ -669,7 +668,7 @@ def check_formats(input) -> TileResults: deps=[AssetKey(("export", "compressed_tiles"))], required_resource_keys={"file_store", "version", "gdal"}, ) -def compressed_tiles_validation(context, export_index: Path, metadata: Path) -> Path: +def compressed_tiles_validation(context: OpExecutionContext, export_index: Path, metadata: Path) -> Path: """Validates the compressed distribution tiles, for each format. Save the validation results to a CSV. Validation is done concurrently per tile. From 31586506d0c8b3f8ca4949cdc610894998f7ca76 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 19 Dec 2024 18:40:29 +0100 Subject: [PATCH 40/40] mmissing comma --- packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql b/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql index a0eb8bad..3d47061c 100644 --- a/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql +++ b/packages/core/src/bag3d/core/sqlfiles/webservice_lod.sql @@ -18,7 +18,7 @@ SELECT p.b3_pw_onvoldoende, p.b3_t_run, p.b3_succes, - p.b3_extrusie + p.b3_extrusie, p.b3_is_glas_dak, p.b3_dak_type, p.b3_h_maaiveld, @@ -84,7 +84,7 @@ SELECT p.b3_pw_onvoldoende, p.b3_t_run, p.b3_succes, - p.b3_extrusie + p.b3_extrusie, p.b3_is_glas_dak, p.b3_dak_type, p.b3_h_maaiveld, @@ -150,7 +150,7 @@ SELECT p.b3_pw_onvoldoende, p.b3_t_run, p.b3_succes, - p.b3_extrusie + p.b3_extrusie, p.b3_is_glas_dak, p.b3_dak_type, p.b3_h_maaiveld,