From 2a1c3cc15ff4692912320e06e747f5e1df7d8b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 14 Aug 2025 14:49:05 +0200 Subject: [PATCH 01/28] Wip --- makefile | 8 ++-- packages/core/tests/conftest.py | 24 ++++++++++ packages/core/tests/test_assets_ahn.py | 48 +++++++++---------- packages/core/tests/test_integration.py | 3 -- packages/core/tests/test_validate.py | 16 +++---- packages/party_walls/tests/conftest.py | 2 +- .../party_walls/tests/test_integration.py | 42 ++++++---------- .../party_walls/tests/test_party_walls.py | 2 +- 8 files changed, 77 insertions(+), 68 deletions(-) diff --git a/makefile b/makefile index c4cca2a6..90b8af00 100644 --- a/makefile +++ b/makefile @@ -102,10 +102,10 @@ test_deploy: test_all: @set -e; \ FAILED=0; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/common/tests/ -v --run-slow --run-all || FAILED=1; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/ -v --run-slow --run-all || FAILED=1; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/ -v --run-slow --run-all || FAILED=1; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/ -v --run-slow --run-all || FAILED=1; \ +# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/common/tests/ -v --run-slow --run-all || FAILED=1; \ +# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/ -v --run-slow --run-all || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/ -v --run-slow --run-all -k"test_job_party_walls" || FAILED=1; \ +# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/ -v --run-slow --run-all || FAILED=1; \ exit $$FAILED include .env diff --git a/packages/core/tests/conftest.py b/packages/core/tests/conftest.py index a0bc0428..d5803b67 100644 --- a/packages/core/tests/conftest.py +++ b/packages/core/tests/conftest.py @@ -168,6 +168,30 @@ def context( ) +@pytest.fixture +def context_ahn( + database, + file_store, + gdal, + validation, + godzilla_server, + podzilla_server, +): + yield build_op_context( + partition_key="01cz1", + resources={ + "gdal": gdal, + "validation": validation, + "db_connection": database, + "file_store": file_store, + "version": VersionResource("test_version"), + "godzilla_server": godzilla_server, + "podzilla_server": podzilla_server, + "specs": Specs3DBAGResource(), + }, + ) + + @pytest.fixture def context_missing( database, wkt_testarea, file_store, gdal_missing, validation_missing diff --git a/packages/core/tests/test_assets_ahn.py b/packages/core/tests/test_assets_ahn.py index b4656a2a..e16a8e0c 100644 --- a/packages/core/tests/test_assets_ahn.py +++ b/packages/core/tests/test_assets_ahn.py @@ -48,77 +48,77 @@ def test_get_checksums(url): assert sha is not None -def test_checksums_for_ahn(context): - res = md5_ahn3(context) +def test_checksums_for_ahn(context_ahn): + res = md5_ahn3(context_ahn) assert len(res) > 0 for k, sha in list(res.items())[:5]: assert sha is not None - res = md5_ahn4(context) + res = md5_ahn4(context_ahn) assert len(res) > 0 for k, sha in list(res.items())[:5]: assert sha is not None - res = sha256_ahn5(context) + res = sha256_ahn5(context_ahn) assert len(res) > 0 for k, sha in list(res.items())[:5]: assert sha is not None -def test_tile_index_ahn(context): - res = tile_index_ahn(context) +def test_tile_index_ahn(context_ahn): + res = tile_index_ahn(context_ahn) assert len(res) == 1407 assert res[list(res.keys())[0]] is not None @pytest.mark.slow -def test_laz_files_ahn3(context, md5_ahn3_fix, tile_index_ahn_fix): - laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 3) +def test_laz_files_ahn3(context_ahn, md5_ahn3_fix, tile_index_ahn_fix): + laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 3) laz_dir.mkdir(exist_ok=True, parents=True) config = LazFilesConfig(force_download=False, check_hash=False) - res = laz_files_ahn3(context, config, md5_ahn3_fix, tile_index_ahn_fix) + res = laz_files_ahn3(context_ahn, config, md5_ahn3_fix, tile_index_ahn_fix) assert res.value.url is not None assert res is not None print(res.value) @pytest.mark.slow -def test_laz_files_ahn4(context, md5_ahn4_fix, tile_index_ahn_fix): - laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 4) +def test_laz_files_ahn4(context_ahn, md5_ahn4_fix, tile_index_ahn_fix): + laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 4) laz_dir.mkdir(exist_ok=True, parents=True) config = LazFilesConfig(force_download=False, check_hash=False) - res = laz_files_ahn4(context, config, md5_ahn4_fix, tile_index_ahn_fix) + res = laz_files_ahn4(context_ahn, config, md5_ahn4_fix, tile_index_ahn_fix) assert res.value.url is not None assert res is not None @pytest.mark.slow -def test_laz_files_ahn5(context, sha256_ahn5_fix, tile_index_ahn_fix): - laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 5) +def test_laz_files_ahn5(context_ahn, sha256_ahn5_fix, tile_index_ahn_fix): + laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 5) laz_dir.mkdir(exist_ok=True, parents=True) config = LazFilesConfig(force_download=False, check_hash=False) - res = laz_files_ahn5(context, config, sha256_ahn5_fix, tile_index_ahn_fix) + res = laz_files_ahn5(context_ahn, config, sha256_ahn5_fix, tile_index_ahn_fix) assert res.value.url is not None assert res is not None -def test_metadata_table_ahn3(context): - metadata = metadata_table_ahn3(context) +def test_metadata_table_ahn3(context_ahn): + metadata = metadata_table_ahn3(context_ahn) tbl = PostgresTableIdentifier("ahn", "metadata_ahn3") - assert table_exists(context, tbl) + assert table_exists(context_ahn, tbl) assert isinstance(metadata, PostgresTableIdentifier) assert str(metadata) == f"{tbl.schema}.{tbl.table}" -def test_metadata_table_ahn4(context): - metadata = metadata_table_ahn4(context) +def test_metadata_table_ahn4(context_ahn): + metadata = metadata_table_ahn4(context_ahn) tbl = PostgresTableIdentifier("ahn", "metadata_ahn4") - assert table_exists(context, tbl) + assert table_exists(context_ahn, tbl) assert isinstance(metadata, PostgresTableIdentifier) assert str(metadata) == f"{tbl.schema}.{tbl.table}" -def test_metadata_table_ahn5(context): - metadata = metadata_table_ahn5(context) +def test_metadata_table_ahn5(context_ahn): + metadata = metadata_table_ahn5(context_ahn) tbl = PostgresTableIdentifier("ahn", "metadata_ahn5") - assert table_exists(context, tbl) + assert table_exists(context_ahn, tbl) assert isinstance(metadata, PostgresTableIdentifier) assert str(metadata) == f"{tbl.schema}.{tbl.table}" diff --git a/packages/core/tests/test_integration.py b/packages/core/tests/test_integration.py index 95e78e48..c4aad07a 100644 --- a/packages/core/tests/test_integration.py +++ b/packages/core/tests/test_integration.py @@ -203,7 +203,6 @@ def test_integration_reconstruction_and_export( } }, ) - assert isinstance(result, ExecuteInProcessResult) assert result.success @@ -219,13 +218,11 @@ def test_integration_reconstruction_and_export( } }, ) - assert isinstance(result, ExecuteInProcessResult) assert result.success resolved_job = defs.get_job_def("nl_export_after_floors") result = resolved_job.execute_in_process(instance=instance, resources=resources) - assert isinstance(result, ExecuteInProcessResult) assert result.success diff --git a/packages/core/tests/test_validate.py b/packages/core/tests/test_validate.py index 8c48a8f1..a54999dc 100644 --- a/packages/core/tests/test_validate.py +++ b/packages/core/tests/test_validate.py @@ -16,7 +16,7 @@ def test_obj(context, test_data_dir): res = obj( context.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, snap_tol=0.0001, @@ -31,15 +31,15 @@ def test_gpkg(context, test_data_dir): res = gpkg( context.resources.gdal.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", "https://data.3dbag.nl", "test", specs=context.resources.specs, ) assert res.zip_ok assert res.file_ok - assert res.nr_building == 419 - assert res.nr_buildingpart == 422 + assert res.nr_building == 413 + assert res.nr_buildingpart == 414 assert res.nr_invalid_2d_geom == 0 @@ -47,7 +47,7 @@ def test_cityjson(context, test_data_dir): res = cityjson( context.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, url_root="https://data.3dbag.nl", @@ -65,7 +65,7 @@ def test_obj_missing(context_missing, test_data_dir): _ = obj( context_missing.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, snap_tol=0.0001, @@ -79,7 +79,7 @@ def test_gpkg_missing(context_missing, test_data_dir): _ = gpkg( context_missing.resources.gdal.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", "https://data.3dbag.nl", "test", specs=context_missing.resources.specs, @@ -91,7 +91,7 @@ def test_cityjson_missing(context_missing, test_data_dir): _ = cityjson( context_missing.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, url_root="https://data.3dbag.nl", diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index 6306cd10..e2c2c1e6 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -58,7 +58,7 @@ def database(): @pytest.fixture def context(database, input_data_dir, fastssd_data_dir): yield build_op_context( - partition_key="10/564/624", + partition_key="0/0/0", resources={ "db_connection": database, "file_store": FileStoreResource(data_dir=str(input_data_dir)), diff --git a/packages/party_walls/tests/test_integration.py b/packages/party_walls/tests/test_integration.py index 4024c5c0..9ca87956 100644 --- a/packages/party_walls/tests/test_integration.py +++ b/packages/party_walls/tests/test_integration.py @@ -2,12 +2,12 @@ from bag3d.common.resources.files import FileStoreResource from bag3d.common.resources.version import VersionResource from bag3d.party_walls import assets -from bag3d.party_walls.jobs import job_nl_party_walls +from bag3d.party_walls.jobs import job_nl_party_walls, job_nl_party_walls_index from dagster import ( - AssetKey, Definitions, ExecuteInProcessResult, load_assets_from_package_module, + DagsterInstance, ) @@ -16,8 +16,6 @@ def test_job_party_walls( database, input_data_dir, fastssd_data_dir, - mock_asset_distribution_tiles_files_index, - mock_asset_features_file_index, ): resources = { "db_connection": database, @@ -28,35 +26,25 @@ def test_job_party_walls( all_party_assets = load_assets_from_package_module( assets, key_prefix="party_walls", group_name="party_walls" ) - # Filter the assets to include only the ones we need - party_assets = [ - asset - for asset in all_party_assets - if asset.key - in { - AssetKey(["party_walls", "cityjsonfeatures_with_party_walls_nl"]), - AssetKey(["party_walls", "party_walls_nl"]), - } - ] defs = Definitions( resources=resources, - assets=[ - mock_asset_distribution_tiles_files_index, - mock_asset_features_file_index, - party_assets[0], - party_assets[1], - ], + assets=all_party_assets, jobs=[ + job_nl_party_walls_index, job_nl_party_walls, ], ) - resolved_job = defs.get_job_def("nl_party_walls") - - result = resolved_job.execute_in_process( - resources=resources, partition_key="10/564/624" - ) + with DagsterInstance.ephemeral() as instance: + resolved_job = defs.get_job_def("nl_party_walls_index") + result = resolved_job.execute_in_process(instance=instance, resources=resources) + assert isinstance(result, ExecuteInProcessResult) + assert result.success - assert isinstance(result, ExecuteInProcessResult) - assert result.success + resolved_job = defs.get_job_def("nl_party_walls") + result = resolved_job.execute_in_process( + instance=instance, resources=resources, partition_key="0/0/0" + ) + assert isinstance(result, ExecuteInProcessResult) + assert result.success diff --git a/packages/party_walls/tests/test_party_walls.py b/packages/party_walls/tests/test_party_walls.py index 3257d178..847aa35b 100644 --- a/packages/party_walls/tests/test_party_walls.py +++ b/packages/party_walls/tests/test_party_walls.py @@ -6,7 +6,7 @@ party_walls_nl, ) -TILE_IDS = ("10/564/624", "10/564/626", "10/566/624", "10/566/626", "9/560/624") +TILE_IDS = "0/0/0" def test_distribution_tiles_files_index(context): From e50dd64e3e4894d29e0173a12a76c9e239f4b579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 14 Aug 2025 14:53:55 +0200 Subject: [PATCH 02/28] Update to test_data_14-wip --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 90b8af00..76644d65 100644 --- a/makefile +++ b/makefile @@ -113,7 +113,7 @@ include .env download: rm -rf $(BAG3D_TEST_DATA) mkdir -p $(BAG3D_TEST_DATA) - cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v13.zip ; unzip -q test_data_v13.zip ; rm test_data_v13.zip + cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v14-wip.zip ; unzip -q test_data_v14-wip.zip ; rm test_data_v14-wip.zip install_uv: From 170a604d59c93633ce7b57596c06d799c1762c9f Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 15:56:05 +0200 Subject: [PATCH 03/28] tile name --- packages/party_walls/tests/conftest.py | 2 +- packages/party_walls/tests/test_integration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index e2c2c1e6..6306cd10 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -58,7 +58,7 @@ def database(): @pytest.fixture def context(database, input_data_dir, fastssd_data_dir): yield build_op_context( - partition_key="0/0/0", + partition_key="10/564/624", resources={ "db_connection": database, "file_store": FileStoreResource(data_dir=str(input_data_dir)), diff --git a/packages/party_walls/tests/test_integration.py b/packages/party_walls/tests/test_integration.py index 9ca87956..9a68c167 100644 --- a/packages/party_walls/tests/test_integration.py +++ b/packages/party_walls/tests/test_integration.py @@ -44,7 +44,7 @@ def test_job_party_walls( resolved_job = defs.get_job_def("nl_party_walls") result = resolved_job.execute_in_process( - instance=instance, resources=resources, partition_key="0/0/0" + instance=instance, resources=resources, partition_key="10/564/624" ) assert isinstance(result, ExecuteInProcessResult) assert result.success From 0cd56838432f7c85bf274752883efb902286d75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 14 Aug 2025 16:10:49 +0200 Subject: [PATCH 04/28] This works for me with test_data_v14-wip --- packages/party_walls/tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/party_walls/tests/test_integration.py b/packages/party_walls/tests/test_integration.py index 9a68c167..9ca87956 100644 --- a/packages/party_walls/tests/test_integration.py +++ b/packages/party_walls/tests/test_integration.py @@ -44,7 +44,7 @@ def test_job_party_walls( resolved_job = defs.get_job_def("nl_party_walls") result = resolved_job.execute_in_process( - instance=instance, resources=resources, partition_key="10/564/624" + instance=instance, resources=resources, partition_key="0/0/0" ) assert isinstance(result, ExecuteInProcessResult) assert result.success From 32df188efec8ef4d7bfbd71524fe57951b28d00b Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 21:29:21 +0200 Subject: [PATCH 05/28] clean init --- .../src/bag3d/common/resources/__init__.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 92bb3575..865e0c65 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -60,18 +60,11 @@ file_store = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE")) file_store_fastssd = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE_FASTSSD")) -file_store_test = FileStoreResource( - data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input") -) -file_store_fastssd_test = FileStoreResource( - data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "integration_core") -) # Configure for gilfoyle file_store_gilfoyle = FileStoreResource(data_dir="/data") file_store_gilfoyle_fastssd = FileStoreResource(data_dir="/fastssd/data") - lastools = LASToolsResource( exe_lasindex=os.getenv("EXE_PATH_LASINDEX"), exe_las2las=os.getenv("EXE_PATH_LAS2LAS"), @@ -118,23 +111,6 @@ } -# RESOURCES_TEST = { -# "gdal": gdal, -# "file_store": file_store_test, -# "file_store_fastssd": file_store_fastssd_test, -# "db_connection": db_connection, -# "pdal": pdal, -# "lastools": lastools, -# "tyler": tyler, -# "geoflow": geoflow, -# "validation": validation, -# "roofer": roofer, -# "version": version, -# "specs": specs, -# "godzilla_server": godzilla_server, -# "podzilla_server": podzilla_server, -# } -# # RESOURCES_PROD = { # "gdal": gdal, # "file_store": file_store_gilfoyle, From f34d888322ea7ac99bc64c316ade79d7e9d53f77 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 23:23:24 +0200 Subject: [PATCH 06/28] fix core integration --- packages/core/tests/conftest.py | 17 +++++++++++++++++ packages/core/tests/test_integration.py | 12 +++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/core/tests/conftest.py b/packages/core/tests/conftest.py index d5803b67..0be3be68 100644 --- a/packages/core/tests/conftest.py +++ b/packages/core/tests/conftest.py @@ -289,6 +289,23 @@ def test_data_dir(): yield Path(LOCAL_DIR) +@pytest.fixture(scope="session") +def core_integration_test_dir(test_data_dir): + yield test_data_dir / "integration_core" + + +@pytest.fixture(scope="session") +def core_file_store_fastssd(core_integration_test_dir) -> Path: + """Root directory path for test data""" + return core_integration_test_dir / "file_store_fastssd" + + +@pytest.fixture(scope="session") +def core_file_store(core_integration_test_dir) -> Path: + """Root directory path for test data""" + return core_integration_test_dir / "file_store" + + @pytest.fixture(scope="session") def md5_ahn3_fix(): yield {"C_01CZ1.LAZ": "063b23d038f97576d279fb7d8a1481ad"} diff --git a/packages/core/tests/test_integration.py b/packages/core/tests/test_integration.py index c4aad07a..0b3ab17f 100644 --- a/packages/core/tests/test_integration.py +++ b/packages/core/tests/test_integration.py @@ -36,7 +36,7 @@ @pytest.mark.needs_tools -def test_integration_ahn(database, test_data_dir): +def test_integration_ahn(database, core_file_store): """Test the ahn jobs.""" resources = { "lastools": LASToolsResource( @@ -48,7 +48,7 @@ def test_integration_ahn(database, test_data_dir): ), "db_connection": database, "file_store": FileStoreResource( - data_dir=str(test_data_dir / "reconstruction_input") + data_dir=str(core_file_store) ), } @@ -108,6 +108,8 @@ def test_integration_ahn(database, test_data_dir): def test_integration_reconstruction_and_export( database, test_data_dir, + core_file_store, + core_file_store_fastssd, mock_asset_reconstruction_input, mock_asset_tiles, mock_asset_index, @@ -118,7 +120,7 @@ def test_integration_reconstruction_and_export( # update quadtree og_quadtree = test_data_dir / "quadtree.tsv" export_dir = ( - test_data_dir / "reconstruction_input" / "3DBAG" / "export_test_version" + core_file_store / "3DBAG" / "export_test_version" ) export_dir.mkdir(exist_ok=True) os.system(f"cp {og_quadtree} {export_dir}") @@ -144,10 +146,10 @@ def test_integration_reconstruction_and_export( ), "db_connection": database, "file_store": FileStoreResource( - data_dir=str(test_data_dir / "reconstruction_input") + data_dir=str(core_file_store) ), "file_store_fastssd": FileStoreResource( - data_dir=str(test_data_dir / "integration_core") + data_dir=str(core_file_store_fastssd) ), "version": VersionResource("test_version"), "validation": ValidationResource( From 03d198d6bc28e1528605e07d13057fbd16e6eef3 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 23:34:59 +0200 Subject: [PATCH 07/28] formatting --- packages/core/tests/test_integration.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/packages/core/tests/test_integration.py b/packages/core/tests/test_integration.py index 0b3ab17f..4c8f4b3f 100644 --- a/packages/core/tests/test_integration.py +++ b/packages/core/tests/test_integration.py @@ -47,9 +47,7 @@ def test_integration_ahn(database, core_file_store): exe_pdal=os.getenv("EXE_PATH_PDAL"), ), "db_connection": database, - "file_store": FileStoreResource( - data_dir=str(core_file_store) - ), + "file_store": FileStoreResource(data_dir=str(core_file_store)), } all_ahn_assets = load_assets_from_package_module( @@ -119,9 +117,7 @@ def test_integration_reconstruction_and_export( ): # update quadtree og_quadtree = test_data_dir / "quadtree.tsv" - export_dir = ( - core_file_store / "3DBAG" / "export_test_version" - ) + export_dir = core_file_store / "3DBAG" / "export_test_version" export_dir.mkdir(exist_ok=True) os.system(f"cp {og_quadtree} {export_dir}") @@ -145,12 +141,8 @@ def test_integration_reconstruction_and_export( exe_sozip=os.getenv("EXE_PATH_SOZIP"), ), "db_connection": database, - "file_store": FileStoreResource( - data_dir=str(core_file_store) - ), - "file_store_fastssd": FileStoreResource( - data_dir=str(core_file_store_fastssd) - ), + "file_store": FileStoreResource(data_dir=str(core_file_store)), + "file_store_fastssd": FileStoreResource(data_dir=str(core_file_store_fastssd)), "version": VersionResource("test_version"), "validation": ValidationResource( exe_val3dity=os.getenv("EXE_PATH_VAL3DITY"), From f92bc1cc552f355a137dafaf99e51435c508b0f3 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 23:35:21 +0200 Subject: [PATCH 08/28] fixed party_walls integration test --- packages/party_walls/tests/conftest.py | 27 +++++++++++-------- .../party_walls/tests/test_integration.py | 10 ++++--- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index 6306cd10..c065b4e4 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -30,15 +30,20 @@ def test_data_dir(): @pytest.fixture(scope="session") -def input_data_dir(test_data_dir) -> Path: +def party_walls_integration_test_dir(test_data_dir): + yield test_data_dir / "integration_party_walls" + + +@pytest.fixture(scope="session") +def party_walls_file_store_fastssd(party_walls_integration_test_dir) -> Path: """Root directory path for test data""" - return test_data_dir / "reconstruction_input" + return party_walls_integration_test_dir / "file_store_fastssd" @pytest.fixture(scope="session") -def fastssd_data_dir(test_data_dir) -> Path: +def party_walls_file_store(party_walls_integration_test_dir) -> Path: """Root directory path for test data""" - return test_data_dir / "integration_party_walls" + return party_walls_integration_test_dir / "file_store" @pytest.fixture(scope="session") @@ -56,12 +61,12 @@ def database(): @pytest.fixture -def context(database, input_data_dir, fastssd_data_dir): +def context(database, party_walls_file_store, fastssd_data_dir): yield build_op_context( - partition_key="10/564/624", + partition_key="0/0/0", resources={ "db_connection": database, - "file_store": FileStoreResource(data_dir=str(input_data_dir)), + "file_store": FileStoreResource(data_dir=str(party_walls_file_store)), "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)), "version": VersionResource(VERSION), }, @@ -119,27 +124,27 @@ def mock_features_file_index(intermediate_data_dir, fastssd_data_dir): @pytest.fixture(scope="session") -def mock_distribution_tiles_files_index(intermediate_data_dir, input_data_dir): +def mock_distribution_tiles_files_index(intermediate_data_dir, party_walls_file_store): data = pickle.load( open(intermediate_data_dir / "distribution_tiles_files_index.pkl", "rb") ) for i, d in enumerate(data.paths_array): data.paths_array[i] = Path( str(d) - .replace(str(d.parents[6]), str(input_data_dir)) + .replace(str(d.parents[6]), str(party_walls_file_store)) .replace("export", "export_test_version") ) for k, v in data.export_results.items(): cj_path = data.export_results[k].cityjson_path data.export_results[k].cityjson_path = Path( str(cj_path) - .replace(str(cj_path.parents[6]), str(input_data_dir)) + .replace(str(cj_path.parents[6]), str(party_walls_file_store)) .replace("export", "export_test_version") ) gpkg_path = data.export_results[k].gpkg_path data.export_results[k].gpkg_path = Path( str(gpkg_path) - .replace(str(gpkg_path.parents[6]), str(input_data_dir)) + .replace(str(gpkg_path.parents[6]), str(party_walls_file_store)) .replace("export", "export_test_version") ) # TODO: fix data.export_results[k].obj_paths diff --git a/packages/party_walls/tests/test_integration.py b/packages/party_walls/tests/test_integration.py index 9ca87956..97147b38 100644 --- a/packages/party_walls/tests/test_integration.py +++ b/packages/party_walls/tests/test_integration.py @@ -14,13 +14,15 @@ @pytest.mark.needs_tools def test_job_party_walls( database, - input_data_dir, - fastssd_data_dir, + party_walls_file_store, + party_walls_file_store_fastssd, ): resources = { "db_connection": database, - "file_store": FileStoreResource(data_dir=str(input_data_dir)), - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)), + "file_store": FileStoreResource(data_dir=str(party_walls_file_store)), + "file_store_fastssd": FileStoreResource( + data_dir=str(party_walls_file_store_fastssd) + ), "version": VersionResource("test_version"), } all_party_assets = load_assets_from_package_module( From 19cfc6371425e98996930943317fb0480ed9f48a Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 23:35:46 +0200 Subject: [PATCH 09/28] removed unecessary data removal --- packages/core/tests/test_assets_top10nl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/tests/test_assets_top10nl.py b/packages/core/tests/test_assets_top10nl.py index e17e37f0..c7a8385c 100644 --- a/packages/core/tests/test_assets_top10nl.py +++ b/packages/core/tests/test_assets_top10nl.py @@ -7,4 +7,3 @@ def test_extract_top10nl(context_top10nl): """Does the complete asset work?""" res = download.extract_top10nl(context_top10nl) assert res.value.exists() - context_top10nl.resources.file_store.file_store.rm(force=True) From a442f94977f1848f6f5212b504d81fb7e157cc67 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 23:36:19 +0200 Subject: [PATCH 10/28] removed query print --- packages/core/src/bag3d/core/assets/ahn/metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/bag3d/core/assets/ahn/metadata.py b/packages/core/src/bag3d/core/assets/ahn/metadata.py index b3542200..71e26084 100644 --- a/packages/core/src/bag3d/core/assets/ahn/metadata.py +++ b/packages/core/src/bag3d/core/assets/ahn/metadata.py @@ -247,7 +247,6 @@ def compute_load_metadata( ST_SetSRID(ST_GeomFromGeoJSON({boundary}), 28992) ); """).format(**query_params) - context.log.info(conn.print_query(query)) conn.send_query(query) # Cannot index the table here, because this is a partitioned assed. This means that # this function is called for each partition, which would index the table after From 77a7299f2c9f4243216ec0aeefd0da2a98483fe3 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Thu, 14 Aug 2025 23:37:05 +0200 Subject: [PATCH 11/28] clean up init --- packages/common/src/bag3d/common/resources/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 865e0c65..76fb28a4 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from bag3d.common.resources.executables import ( GDALResource, @@ -61,10 +60,6 @@ file_store_fastssd = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE_FASTSSD")) -# Configure for gilfoyle -file_store_gilfoyle = FileStoreResource(data_dir="/data") -file_store_gilfoyle_fastssd = FileStoreResource(data_dir="/fastssd/data") - lastools = LASToolsResource( exe_lasindex=os.getenv("EXE_PATH_LASINDEX"), exe_las2las=os.getenv("EXE_PATH_LAS2LAS"), From 26b0d858d76f9dee542e687585596e71215ed9ea Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 00:08:35 +0200 Subject: [PATCH 12/28] sample laz file --- packages/common/tests/conftest.py | 7 +++++-- packages/common/tests/test_resources.py | 11 ++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/common/tests/conftest.py b/packages/common/tests/conftest.py index c267cf22..b0bf59dc 100644 --- a/packages/common/tests/conftest.py +++ b/packages/common/tests/conftest.py @@ -109,5 +109,8 @@ def test_data_dir(): @pytest.fixture(scope="session") -def laz_files_ahn3_dir(test_data_dir): - yield test_data_dir / "reconstruction_input/pointcloud/AHN3/as_downloaded/LAZ" +def sample_laz_file(test_data_dir): + yield ( + test_data_dir + / "integration_core/file_store/pointcloud/AHN3/as_downloaded/LAZ/C_32BZ2.LAZ" + ) diff --git a/packages/common/tests/test_resources.py b/packages/common/tests/test_resources.py index 730175e3..e01d264e 100644 --- a/packages/common/tests/test_resources.py +++ b/packages/common/tests/test_resources.py @@ -37,16 +37,15 @@ def test_gdal_local(test_data_dir): assert return_code == 0 -def test_pdal_local(laz_files_ahn3_dir): +def test_pdal_local(sample_laz_file): """Use local PDAL installation""" pdal = PDALResource(exe_pdal=EnvVar("EXE_PATH_PDAL").get_value()) assert not pdal.with_docker - filepath = laz_files_ahn3_dir / "C_32BZ2.LAZ" - return_code, output = pdal_info(pdal.app, filepath, with_all=True) + return_code, output = pdal_info(pdal.app, sample_laz_file, with_all=True) assert return_code == 0 -def test_lastools(laz_files_ahn3_dir): +def test_lastools(sample_laz_file): lastools_resource = LASToolsResource( exe_lasindex=EnvVar("EXE_PATH_LASINDEX").get_value(), exe_las2las=EnvVar("EXE_PATH_LAS2LAS").get_value(), @@ -55,8 +54,6 @@ def test_lastools(laz_files_ahn3_dir): lastools = lastools_resource.app - filepath = laz_files_ahn3_dir / "C_32BZ2.LAZ" - cmd_list = [ "{exe}", "-i {local_path}", @@ -66,7 +63,7 @@ def test_lastools(laz_files_ahn3_dir): "-dont_reindex", ] return_code, output = lastools.execute( - "lasindex", " ".join(cmd_list), local_path=filepath + "lasindex", " ".join(cmd_list), local_path=sample_laz_file ) assert return_code == 0 From c174c039083357e9284af998bbd7bdfc01ef7bdb Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 00:23:28 +0200 Subject: [PATCH 13/28] fixed floors-estimation integration -test --- packages/floors_estimation/tests/conftest.py | 20 ++++++++++--------- .../tests/test_integration.py | 6 ++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/floors_estimation/tests/conftest.py b/packages/floors_estimation/tests/conftest.py index 055d5c2c..3ed7affe 100644 --- a/packages/floors_estimation/tests/conftest.py +++ b/packages/floors_estimation/tests/conftest.py @@ -21,15 +21,16 @@ def test_data_dir(): @pytest.fixture(scope="session") -def input_data_dir(test_data_dir) -> Path: - """Root directory path for test data""" - return test_data_dir / "reconstruction_input" +def floors_estimation_integration_test_dir(test_data_dir): + yield test_data_dir / "integration_floors_estimation" @pytest.fixture(scope="session") -def fastssd_data_dir(test_data_dir) -> Path: +def floors_estimation_file_store_fastssd( + floors_estimation_integration_test_dir, +) -> Path: """Root directory path for test data""" - return test_data_dir / "integration_floors_estimation" + return floors_estimation_integration_test_dir / "file_store_fastssd" @pytest.fixture(scope="session") @@ -53,13 +54,14 @@ def database(): @pytest.fixture -def context(database, input_data_dir, model, fastssd_data_dir): +def context(database, model, floors_estimation_file_store_fastssd): yield build_op_context( - partition_key="10/564/624", + partition_key="0/0/0", resources={ "db_connection": database, - "file_store": FileStoreResource(data_dir=str(input_data_dir)), - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)), + "file_store_fastssd": FileStoreResource( + data_dir=str(floors_estimation_file_store_fastssd) + ), "model_store": model, "version": "test_version", }, diff --git a/packages/floors_estimation/tests/test_integration.py b/packages/floors_estimation/tests/test_integration.py index 3e8f182d..780712e1 100644 --- a/packages/floors_estimation/tests/test_integration.py +++ b/packages/floors_estimation/tests/test_integration.py @@ -5,11 +5,13 @@ @pytest.mark.needs_tools -def test_job_floors_estimation(fastssd_data_dir): +def test_job_floors_estimation(floors_estimation_file_store_fastssd): resolved_job = defs.get_job_def("floors_estimation") resources = { - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)) + "file_store_fastssd": FileStoreResource( + data_dir=str(floors_estimation_file_store_fastssd) + ) } result = resolved_job.execute_in_process(resources=resources) From 00dbfa81ee57eb46149e5fa8b1fcae32858e43a9 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 00:30:32 +0200 Subject: [PATCH 14/28] fixes for floors --- packages/floors_estimation/tests/conftest.py | 8 ++++++-- .../floors_estimation/tests/test_floors_estimation.py | 11 +++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/floors_estimation/tests/conftest.py b/packages/floors_estimation/tests/conftest.py index 3ed7affe..be548a33 100644 --- a/packages/floors_estimation/tests/conftest.py +++ b/packages/floors_estimation/tests/conftest.py @@ -107,7 +107,9 @@ def mock_preprocessed_features(intermediate_data_dir): @pytest.fixture(scope="session") -def mock_features_file_index(intermediate_data_dir, fastssd_data_dir): +def mock_features_file_index( + intermediate_data_dir, floors_estimation_file_store_fastssd +): data = pickle.load( open( intermediate_data_dir / "features_file_index_floors_estimation.pkl", @@ -115,7 +117,9 @@ def mock_features_file_index(intermediate_data_dir, fastssd_data_dir): ) ) for k, v in data.items(): - data[k] = Path(str(v).replace(str(v.parents[5]), str(fastssd_data_dir))) + data[k] = Path( + str(v).replace(str(v.parents[5]), str(floors_estimation_file_store_fastssd)) + ) return data diff --git a/packages/floors_estimation/tests/test_floors_estimation.py b/packages/floors_estimation/tests/test_floors_estimation.py index 9a0deaf1..12749597 100644 --- a/packages/floors_estimation/tests/test_floors_estimation.py +++ b/packages/floors_estimation/tests/test_floors_estimation.py @@ -18,7 +18,7 @@ def test_features_file_index(context): """""" result = features_file_index(context=context) - assert len(result) == 419 + assert len(result) == 413 assert "NL.IMBAG.Pand.0307100000377456" in result.keys() assert "party_walls_features" in str(result["NL.IMBAG.Pand.0307100000377456"]) @@ -103,7 +103,7 @@ def test_preprocessed_features(context): assert table_exists(context, all_features_table) is True data = preprocessed_features(context, all_features_table) assert data is not None - assert data.shape[0] == 274 + assert data.shape[0] == 273 def test_inferenced_floors(context, mock_preprocessed_features): @@ -121,7 +121,10 @@ def test_predictions_table(context, mock_inferenced_floors): def test_save_cjfiles( - context, fastssd_data_dir, mock_inferenced_floors, mock_features_file_index + context, + floors_estimation_file_store_fastssd, + mock_inferenced_floors, + mock_features_file_index, ): save_cjfiles( context, @@ -129,6 +132,6 @@ def test_save_cjfiles( mock_features_file_index, ) assert ( - fastssd_data_dir + floors_estimation_file_store_fastssd / "3DBAG/bouwlagen_features/10/564/624/NL.IMBAG.Pand.0307100000308298.city.jsonl" ).exists() From 1977555b0fdffed9a59cdecfcaad3a999d5650a8 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 00:30:59 +0200 Subject: [PATCH 15/28] fix for party walls --- packages/party_walls/tests/conftest.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index c065b4e4..4d3f362d 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -17,12 +17,6 @@ DB_NAME = os.getenv("BAG3D_PG_DATABASE") VERSION = "test_version" -# update quadtree -og_quadtree = Path(LOCAL_DIR) / "quadtree.tsv" -export_dir = Path(LOCAL_DIR) / "reconstruction_input" / "3DBAG" / ("export_" + VERSION) -export_dir.mkdir(exist_ok=True, parents=True) -os.system(f"cp {og_quadtree} {export_dir}") - @pytest.fixture(scope="session") def test_data_dir(): @@ -61,13 +55,15 @@ def database(): @pytest.fixture -def context(database, party_walls_file_store, fastssd_data_dir): +def context(database, party_walls_file_store, party_walls_file_store_fastssd): yield build_op_context( partition_key="0/0/0", resources={ "db_connection": database, "file_store": FileStoreResource(data_dir=str(party_walls_file_store)), - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)), + "file_store_fastssd": FileStoreResource( + data_dir=str(party_walls_file_store_fastssd) + ), "version": VersionResource(VERSION), }, ) @@ -112,12 +108,12 @@ def mock_party_walls_nl(intermediate_data_dir) -> pd.DataFrame: @pytest.fixture(scope="session") -def mock_features_file_index(intermediate_data_dir, fastssd_data_dir): +def mock_features_file_index(intermediate_data_dir, party_walls_file_store_fastssd): data = pickle.load(open(intermediate_data_dir / "features_file_index.pkl", "rb")) for k, v in data.items(): data[k] = Path( str(v) - .replace(str(v.parents[8]), str(fastssd_data_dir)) + .replace(str(v.parents[8]), str(party_walls_file_store_fastssd)) .replace("export", "export_test_version") ) return data From 7f5dd46be0368fbdc672c632c66a912f907d9961 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 02:22:02 +0200 Subject: [PATCH 16/28] party walls unit tests --- packages/party_walls/tests/test_party_walls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/party_walls/tests/test_party_walls.py b/packages/party_walls/tests/test_party_walls.py index 847aa35b..87df502e 100644 --- a/packages/party_walls/tests/test_party_walls.py +++ b/packages/party_walls/tests/test_party_walls.py @@ -6,7 +6,7 @@ party_walls_nl, ) -TILE_IDS = "0/0/0" +TILE_IDS = ("0/0/0",) def test_distribution_tiles_files_index(context): @@ -30,7 +30,7 @@ def test_party_walls(context, mock_distribution_tiles_files_index): def test_features_file_index(context): """Can we find and map all the 5825 cityjson feature files of the test data?""" result = features_file_index(context=context) - assert len(result) == 407 + assert len(result) == 415 @pytest.mark.slow From 24bbdf978e756b454492e84d056bdd09234e9618 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 08:47:03 +0200 Subject: [PATCH 17/28] floors-estimation unit test --- packages/floors_estimation/tests/conftest.py | 80 +++++++++++++------ .../tests/test_floors_estimation.py | 14 ++-- 2 files changed, 61 insertions(+), 33 deletions(-) diff --git a/packages/floors_estimation/tests/conftest.py b/packages/floors_estimation/tests/conftest.py index be548a33..0b4da718 100644 --- a/packages/floors_estimation/tests/conftest.py +++ b/packages/floors_estimation/tests/conftest.py @@ -1,12 +1,15 @@ import os import pickle from pathlib import Path +from io import StringIO +import pandas as pd import pytest from bag3d.common.resources.database import DatabaseResource from bag3d.common.resources.files import FileStoreResource from dagster import build_op_context + LOCAL_DIR = os.getenv("BAG3D_TEST_DATA") HOST = os.getenv("BAG3D_PG_HOST") PORT = os.getenv("BAG3D_PG_PORT") @@ -33,12 +36,6 @@ def floors_estimation_file_store_fastssd( return floors_estimation_integration_test_dir / "file_store_fastssd" -@pytest.fixture(scope="session") -def intermediate_data_dir(test_data_dir) -> Path: - """Root directory path for test data""" - return test_data_dir / "intermediate_data" - - @pytest.fixture(scope="session") def model(test_data_dir) -> Path: """Directory for the floors estimation model""" @@ -52,15 +49,29 @@ def database(): ) yield db +@pytest.fixture +def context_with_data(floors_estimation_file_store_fastssd): + yield build_op_context( + partition_key="0/0/0", + resources={ + "file_store_fastssd": FileStoreResource( + data_dir=str(floors_estimation_file_store_fastssd) + ), + }, + ) + +@pytest.fixture +def file_store_tmp(tmp_path): + yield tmp_path @pytest.fixture -def context(database, model, floors_estimation_file_store_fastssd): +def context(database, model, file_store_tmp): yield build_op_context( partition_key="0/0/0", resources={ "db_connection": database, "file_store_fastssd": FileStoreResource( - data_dir=str(floors_estimation_file_store_fastssd) + data_dir=str(file_store_tmp) ), "model_store": model, "version": "test_version", @@ -102,27 +113,44 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture(scope="session") -def mock_preprocessed_features(intermediate_data_dir): - return pickle.load(open(intermediate_data_dir / "preprocessed_features.pkl", "rb")) - +def mock_preprocessed_features(): + csv_text = """ + identificatie,no_vertices,perimeter,no_units,net_area,building_function,no_neighbours_100,no_adjacent_neighbours,cbs_percent_multihousehold,cbs_pop_per_km2,cbs_dist_to_horeca,buildingtype,construction_year,roof_type,h_roof_50p,h_roof_70p,h_roof_max,h_roof_min,area_roof,area_ext_walls,area_party_walls,area_ground,volume_lod22,volume_lod12 + NL.IMBAG.Pand.0307100000340455,9,55.44252687545277,3,198,1,37,4.0,96,5887,50,2,1927,1,11.029999732971191,11.043000221252441,11.10099983215332,8.204999923706055,160.08,234.48,186.19,156.61,1123.7303466796875,1244.3280029296875 + NL.IMBAG.Pand.0307100000378340,7,21.663911358502567,1,70,0,39,2.0,96,5887,50,4,1922,1,10.729000091552734,10.956000328063965,11.395000457763672,9.892000198364258,34.1,35.98,127.68,23.14,192.20730590820312,204.35214233398438 + NL.IMBAG.Pand.0307100000522025,6,41.48225146934291,1,138,0,124,4.0,77,9573,45,4,1760,1,8.456000328063965,9.131999969482422,10.48900032043457,6.5229997634887695,118.67,91.83,120.32,92.79,526.0918579101562,614.935791015625 + NL.IMBAG.Pand.0307100000351286,19,311.01548191691313,128,6929,1,5,,77,7350,20,2,1979,1,18.472000122070312,24.054000854492188,27.615999221801758,18.336999893188477,2098.48,5430.01,0.0,1976.27,33547.6953125,39757.82421875 + NL.IMBAG.Pand.0307100000353630,11,47.068882927884324,1,194,0,92,4.0,70,9628,120,4,1800,1,11.687000274658203,12.315999984741211,13.361000061035156,6.389999866485596,132.48,147.19,174.05,101.76,762.90283203125,923.0503540039062 + """ + + return pd.read_csv(StringIO(csv_text.strip())) + @pytest.fixture(scope="session") def mock_features_file_index( - intermediate_data_dir, floors_estimation_file_store_fastssd + floors_estimation_file_store_fastssd ): - data = pickle.load( - open( - intermediate_data_dir / "features_file_index_floors_estimation.pkl", - "rb", - ) - ) - for k, v in data.items(): - data[k] = Path( - str(v).replace(str(v.parents[5]), str(floors_estimation_file_store_fastssd)) - ) - return data - + return { + 'NL.IMBAG.Pand.0307100000340455': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000340455.city.jsonl", + 'NL.IMBAG.Pand.0307100000364333': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000364333.city.jsonl", + 'NL.IMBAG.Pand.0307100000378340': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000378340.city.jsonl", + 'NL.IMBAG.Pand.0307100000522025': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522025.city.jsonl", + 'NL.IMBAG.Pand.0307100000351286': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000351286.city.jsonl", + 'NL.IMBAG.Pand.0307100000522233': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522233.city.jsonl", + 'NL.IMBAG.Pand.0307100000353630': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000353630.city.jsonl", + 'NL.IMBAG.Pand.0307100000312499': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000312499.city.jsonl" + } @pytest.fixture(scope="session") -def mock_inferenced_floors(intermediate_data_dir): - return pickle.load(open(intermediate_data_dir / "inferenced_floors.pkl", "rb")) +def mock_inferenced_floors(): + + csv_text = """ + identificatie,no_vertices,perimeter,no_units,net_area,building_function,no_neighbours_100,no_adjacent_neighbours,cbs_percent_multihousehold,cbs_pop_per_km2,cbs_dist_to_horeca,buildingtype,construction_year,roof_type,h_roof_50p,h_roof_70p,h_roof_max,h_roof_min,area_roof,area_ext_walls,area_party_walls,area_ground,volume_lod22,volume_lod12,floors,floors_int + NL.IMBAG.Pand.0307100000340455,9,55.44252687545277,3,198,1,37,4.0,96,5887,50,2,1927,1,11.029999732971191,11.04300022125244,11.10099983215332,8.204999923706055,160.08,234.48,186.19,156.61,1123.7303466796875,1244.3280029296875,3.178904466477873,3.0 + NL.IMBAG.Pand.0307100000378340,7,21.663911358502567,1,70,0,39,2.0,96,5887,50,4,1922,1,10.729000091552734,10.956000328063965,11.395000457763672,9.892000198364258,34.1,35.98,127.68,23.14,192.20730590820312,204.3521423339844,2.7363718856599504,3.0 + NL.IMBAG.Pand.0307100000522025,6,41.48225146934291,1,138,0,124,4.0,77,9573,45,4,1760,1,8.456000328063965,9.131999969482422,10.48900032043457,6.5229997634887695,118.67,91.83,120.32,92.79,526.0918579101562,614.935791015625,2.2665033871332767,2.0 + NL.IMBAG.Pand.0307100000351286,19,311.0154819169132,128,6929,1,5,,77,7350,20,2,1979,1,18.472000122070312,24.054000854492188,27.61599922180176,18.33699989318848,2098.48,5430.01,0.0,1976.27,33547.6953125,39757.82421875,7.52333945870127,8.0 + NL.IMBAG.Pand.0307100000353630,11,47.06888292788432,1,194,0,92,4.0,70,9628,120,4,1800,1,11.687000274658203,12.315999984741213,13.361000061035156,6.389999866485596,132.48,147.19,174.05,101.76,762.90283203125,923.0503540039062,3.1525948513138182,3.0 + """ + + return pd.read_csv(StringIO(csv_text.strip())) \ No newline at end of file diff --git a/packages/floors_estimation/tests/test_floors_estimation.py b/packages/floors_estimation/tests/test_floors_estimation.py index 12749597..8f708249 100644 --- a/packages/floors_estimation/tests/test_floors_estimation.py +++ b/packages/floors_estimation/tests/test_floors_estimation.py @@ -15,9 +15,9 @@ ) -def test_features_file_index(context): +def test_features_file_index(context_with_data): """""" - result = features_file_index(context=context) + result = features_file_index(context=context_with_data) assert len(result) == 413 assert "NL.IMBAG.Pand.0307100000377456" in result.keys() assert "party_walls_features" in str(result["NL.IMBAG.Pand.0307100000377456"]) @@ -103,7 +103,8 @@ def test_preprocessed_features(context): assert table_exists(context, all_features_table) is True data = preprocessed_features(context, all_features_table) assert data is not None - assert data.shape[0] == 273 + assert data.shape[0] == 6 + def test_inferenced_floors(context, mock_preprocessed_features): @@ -112,7 +113,6 @@ def test_inferenced_floors(context, mock_preprocessed_features): assert "floors" in res.columns assert "floors_int" in res.columns - def test_predictions_table(context, mock_inferenced_floors): res = predictions_table(context, inferenced_floors=mock_inferenced_floors) assert res.value is not None @@ -122,7 +122,7 @@ def test_predictions_table(context, mock_inferenced_floors): def test_save_cjfiles( context, - floors_estimation_file_store_fastssd, + file_store_tmp, mock_inferenced_floors, mock_features_file_index, ): @@ -132,6 +132,6 @@ def test_save_cjfiles( mock_features_file_index, ) assert ( - floors_estimation_file_store_fastssd - / "3DBAG/bouwlagen_features/10/564/624/NL.IMBAG.Pand.0307100000308298.city.jsonl" + file_store_tmp + / "3DBAG/bouwlagen_features/0/0/0/NL.IMBAG.Pand.0307100000364333.city.jsonl" ).exists() From 87f2cf47e67c57d6fc1246b416f3b770f6644fae Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 08:47:54 +0200 Subject: [PATCH 18/28] formatting --- packages/floors_estimation/tests/conftest.py | 42 +++++++++++-------- .../tests/test_floors_estimation.py | 2 +- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/packages/floors_estimation/tests/conftest.py b/packages/floors_estimation/tests/conftest.py index 0b4da718..d4f2d734 100644 --- a/packages/floors_estimation/tests/conftest.py +++ b/packages/floors_estimation/tests/conftest.py @@ -1,5 +1,4 @@ import os -import pickle from pathlib import Path from io import StringIO import pandas as pd @@ -49,6 +48,7 @@ def database(): ) yield db + @pytest.fixture def context_with_data(floors_estimation_file_store_fastssd): yield build_op_context( @@ -60,19 +60,19 @@ def context_with_data(floors_estimation_file_store_fastssd): }, ) + @pytest.fixture def file_store_tmp(tmp_path): yield tmp_path + @pytest.fixture def context(database, model, file_store_tmp): yield build_op_context( partition_key="0/0/0", resources={ "db_connection": database, - "file_store_fastssd": FileStoreResource( - data_dir=str(file_store_tmp) - ), + "file_store_fastssd": FileStoreResource(data_dir=str(file_store_tmp)), "model_store": model, "version": "test_version", }, @@ -124,26 +124,32 @@ def mock_preprocessed_features(): """ return pd.read_csv(StringIO(csv_text.strip())) - + @pytest.fixture(scope="session") -def mock_features_file_index( - floors_estimation_file_store_fastssd -): +def mock_features_file_index(floors_estimation_file_store_fastssd): return { - 'NL.IMBAG.Pand.0307100000340455': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000340455.city.jsonl", - 'NL.IMBAG.Pand.0307100000364333': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000364333.city.jsonl", - 'NL.IMBAG.Pand.0307100000378340': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000378340.city.jsonl", - 'NL.IMBAG.Pand.0307100000522025': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522025.city.jsonl", - 'NL.IMBAG.Pand.0307100000351286': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000351286.city.jsonl", - 'NL.IMBAG.Pand.0307100000522233': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522233.city.jsonl", - 'NL.IMBAG.Pand.0307100000353630': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000353630.city.jsonl", - 'NL.IMBAG.Pand.0307100000312499': floors_estimation_file_store_fastssd / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000312499.city.jsonl" + "NL.IMBAG.Pand.0307100000340455": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000340455.city.jsonl", + "NL.IMBAG.Pand.0307100000364333": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000364333.city.jsonl", + "NL.IMBAG.Pand.0307100000378340": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000378340.city.jsonl", + "NL.IMBAG.Pand.0307100000522025": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522025.city.jsonl", + "NL.IMBAG.Pand.0307100000351286": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000351286.city.jsonl", + "NL.IMBAG.Pand.0307100000522233": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522233.city.jsonl", + "NL.IMBAG.Pand.0307100000353630": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000353630.city.jsonl", + "NL.IMBAG.Pand.0307100000312499": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000312499.city.jsonl", } + @pytest.fixture(scope="session") def mock_inferenced_floors(): - csv_text = """ identificatie,no_vertices,perimeter,no_units,net_area,building_function,no_neighbours_100,no_adjacent_neighbours,cbs_percent_multihousehold,cbs_pop_per_km2,cbs_dist_to_horeca,buildingtype,construction_year,roof_type,h_roof_50p,h_roof_70p,h_roof_max,h_roof_min,area_roof,area_ext_walls,area_party_walls,area_ground,volume_lod22,volume_lod12,floors,floors_int NL.IMBAG.Pand.0307100000340455,9,55.44252687545277,3,198,1,37,4.0,96,5887,50,2,1927,1,11.029999732971191,11.04300022125244,11.10099983215332,8.204999923706055,160.08,234.48,186.19,156.61,1123.7303466796875,1244.3280029296875,3.178904466477873,3.0 @@ -153,4 +159,4 @@ def mock_inferenced_floors(): NL.IMBAG.Pand.0307100000353630,11,47.06888292788432,1,194,0,92,4.0,70,9628,120,4,1800,1,11.687000274658203,12.315999984741213,13.361000061035156,6.389999866485596,132.48,147.19,174.05,101.76,762.90283203125,923.0503540039062,3.1525948513138182,3.0 """ - return pd.read_csv(StringIO(csv_text.strip())) \ No newline at end of file + return pd.read_csv(StringIO(csv_text.strip())) diff --git a/packages/floors_estimation/tests/test_floors_estimation.py b/packages/floors_estimation/tests/test_floors_estimation.py index 8f708249..79c73a28 100644 --- a/packages/floors_estimation/tests/test_floors_estimation.py +++ b/packages/floors_estimation/tests/test_floors_estimation.py @@ -106,13 +106,13 @@ def test_preprocessed_features(context): assert data.shape[0] == 6 - def test_inferenced_floors(context, mock_preprocessed_features): res = inferenced_floors(context, preprocessed_features=mock_preprocessed_features) assert res is not None assert "floors" in res.columns assert "floors_int" in res.columns + def test_predictions_table(context, mock_inferenced_floors): res = predictions_table(context, inferenced_floors=mock_inferenced_floors) assert res.value is not None From cdcaee3fc2c1d0234af3ac05793bb6b1f6613ecd Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 09:04:45 +0200 Subject: [PATCH 19/28] make file update with new data --- makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/makefile b/makefile index 76644d65..8220d7e8 100644 --- a/makefile +++ b/makefile @@ -102,10 +102,10 @@ test_deploy: test_all: @set -e; \ FAILED=0; \ -# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/common/tests/ -v --run-slow --run-all || FAILED=1; \ -# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/ -v --run-slow --run-all || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/common/tests/ -v --run-slow --run-all || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/ -v --run-slow --run-all || FAILED=1; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/ -v --run-slow --run-all -k"test_job_party_walls" || FAILED=1; \ -# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/ -v --run-slow --run-all || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/ -v --run-slow --run-all || FAILED=1; \ exit $$FAILED include .env @@ -113,7 +113,7 @@ include .env download: rm -rf $(BAG3D_TEST_DATA) mkdir -p $(BAG3D_TEST_DATA) - cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v14-wip.zip ; unzip -q test_data_v14-wip.zip ; rm test_data_v14-wip.zip + cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v14-wip-2.zip ; unzip -q test_data_v14-wip-2.zip ; rm test_data_v14-wip-2.zip install_uv: From 66ae39facb51292b2b4b41017f78a8ed1ec3d2b1 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 12:18:58 +0200 Subject: [PATCH 20/28] not elegant but this is for now --- packages/common/src/bag3d/common/utils/files.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/common/src/bag3d/common/utils/files.py b/packages/common/src/bag3d/common/utils/files.py index aa497707..b25f3d9a 100644 --- a/packages/common/src/bag3d/common/utils/files.py +++ b/packages/common/src/bag3d/common/utils/files.py @@ -101,7 +101,9 @@ def get_export_tile_ids() -> Sequence[str]: env = os.getenv("DAGSTER_ENVIRONMENT", "test") if env == "test": - root_dir = Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input" + root_dir = ( + Path(os.getenv("BAG3D_FILESTORE")) / "integration_party_walls/file_store" + ) version = "test_version" else: root_dir = Path(os.getenv("BAG3D_FILESTORE", "/data")) From b9b1e557c45bcac494519db457378bc83e75db52 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 12:28:47 +0200 Subject: [PATCH 21/28] typo in makefile --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 8220d7e8..b8125193 100644 --- a/makefile +++ b/makefile @@ -104,7 +104,7 @@ test_all: FAILED=0; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/common/tests/ -v --run-slow --run-all || FAILED=1; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/ -v --run-slow --run-all || FAILED=1; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/ -v --run-slow --run-all -k"test_job_party_walls" || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/ -v --run-slow --run-all -k "test_job_party_walls" || FAILED=1; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/ -v --run-slow --run-all || FAILED=1; \ exit $$FAILED From 74e760cfd330b2da47f957fa686f7afdd4275edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Fri, 15 Aug 2025 16:14:58 +0200 Subject: [PATCH 22/28] Enable all tests --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index b8125193..bf51bdf0 100644 --- a/makefile +++ b/makefile @@ -104,7 +104,7 @@ test_all: FAILED=0; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/common/tests/ -v --run-slow --run-all || FAILED=1; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/ -v --run-slow --run-all || FAILED=1; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/ -v --run-slow --run-all -k "test_job_party_walls" || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/ -v --run-slow --run-all || FAILED=1; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/ -v --run-slow --run-all || FAILED=1; \ exit $$FAILED From a7299b43db181c84dc048f2e56510c5f9631cf17 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 18:21:29 +0200 Subject: [PATCH 23/28] add back logging --- packages/core/src/bag3d/core/assets/ahn/metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/bag3d/core/assets/ahn/metadata.py b/packages/core/src/bag3d/core/assets/ahn/metadata.py index 71e26084..370e6787 100644 --- a/packages/core/src/bag3d/core/assets/ahn/metadata.py +++ b/packages/core/src/bag3d/core/assets/ahn/metadata.py @@ -247,6 +247,7 @@ def compute_load_metadata( ST_SetSRID(ST_GeomFromGeoJSON({boundary}), 28992) ); """).format(**query_params) + context.log.debug(conn.print_query(query)) conn.send_query(query) # Cannot index the table here, because this is a partitioned assed. This means that # this function is called for each partition, which would index the table after From f5d35869d3388ece5e1c5e1845f0af9dd5588e60 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Fri, 15 Aug 2025 18:24:31 +0200 Subject: [PATCH 24/28] test data version --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index bf51bdf0..65381af3 100644 --- a/makefile +++ b/makefile @@ -113,7 +113,7 @@ include .env download: rm -rf $(BAG3D_TEST_DATA) mkdir -p $(BAG3D_TEST_DATA) - cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v14-wip-2.zip ; unzip -q test_data_v14-wip-2.zip ; rm test_data_v14-wip-2.zip + cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v14.zip ; unzip -q test_data_v14.zip ; rm test_data_v14.zip install_uv: From 4d8446318c81fd617123deccf285d0678badf96d Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 19 Aug 2025 16:07:29 +0200 Subject: [PATCH 25/28] small doc fix --- packages/party_walls/tests/test_party_walls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/party_walls/tests/test_party_walls.py b/packages/party_walls/tests/test_party_walls.py index 87df502e..43c1ba0d 100644 --- a/packages/party_walls/tests/test_party_walls.py +++ b/packages/party_walls/tests/test_party_walls.py @@ -28,7 +28,7 @@ def test_party_walls(context, mock_distribution_tiles_files_index): def test_features_file_index(context): - """Can we find and map all the 5825 cityjson feature files of the test data?""" + """Can we find and map all the cityjson feature files of the test data?""" result = features_file_index(context=context) assert len(result) == 415 From 75e009758c03edba240ae01105e07381f1dd1c3f Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 19 Aug 2025 16:58:35 +0200 Subject: [PATCH 26/28] removed pkl file for mock_distribution_tiles_files_index --- packages/party_walls/tests/conftest.py | 62 +++++++++++++++----------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index 4d3f362d..4957bf30 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -1,13 +1,21 @@ import os import pickle -from pathlib import Path +from pathlib import Path, PosixPath import pytest from bag3d.common.resources.database import DatabaseResource from bag3d.common.resources.files import FileStoreResource from bag3d.common.resources.version import VersionResource +from bag3d.common.types import ExportResult +from bag3d.party_walls.assets.party_walls import ( + TilesFilesIndex, +) from dagster import AssetKey, IOManager, SourceAsset, build_op_context import pandas as pd +from shapely import STRtree, from_wkt +import numpy as np + + LOCAL_DIR = os.getenv("BAG3D_TEST_DATA") HOST = os.getenv("BAG3D_PG_HOST") @@ -120,31 +128,35 @@ def mock_features_file_index(intermediate_data_dir, party_walls_file_store_fasts @pytest.fixture(scope="session") -def mock_distribution_tiles_files_index(intermediate_data_dir, party_walls_file_store): - data = pickle.load( - open(intermediate_data_dir / "distribution_tiles_files_index.pkl", "rb") - ) - for i, d in enumerate(data.paths_array): - data.paths_array[i] = Path( - str(d) - .replace(str(d.parents[6]), str(party_walls_file_store)) - .replace("export", "export_test_version") - ) - for k, v in data.export_results.items(): - cj_path = data.export_results[k].cityjson_path - data.export_results[k].cityjson_path = Path( - str(cj_path) - .replace(str(cj_path.parents[6]), str(party_walls_file_store)) - .replace("export", "export_test_version") - ) - gpkg_path = data.export_results[k].gpkg_path - data.export_results[k].gpkg_path = Path( - str(gpkg_path) - .replace(str(gpkg_path.parents[6]), str(party_walls_file_store)) - .replace("export", "export_test_version") +def mock_distribution_tiles_files_index(party_walls_file_store): + export_results = { + "0/0/0": ExportResult( + tile_id="0/0/0", + cityjson_path=PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0.city.json" + ), + gpkg_path=PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0.gpkg" + ), + obj_paths=( + PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0-LoD13-3D.obj" + ), + PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0-LoD12-3D.obj" + ), + PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0-LoD22-3D.obj" + ), + ), + wkt="POLYGON((154565.241 462855.414, 155565.241 462855.414, 155565.241 463855.414, 154565.241 463855.414, 154565.241 462855.414))", ) - # TODO: fix data.export_results[k].obj_paths - return data + } + tree = STRtree(tuple(from_wkt(t.wkt) for t in export_results.values())) + paths_array = np.array(tuple(t.cityjson_path for t in export_results.values())) + return TilesFilesIndex( + export_results=export_results, tree=tree, paths_array=paths_array + ) @pytest.fixture(scope="session") From 65b33ae5d0789c2ce7615da8b8e85848eb78bd1e Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 19 Aug 2025 17:10:51 +0200 Subject: [PATCH 27/28] removed features_file_index pkl --- packages/party_walls/tests/conftest.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index 4957bf30..221e20de 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -1,5 +1,4 @@ import os -import pickle from pathlib import Path, PosixPath import pytest @@ -16,7 +15,6 @@ import numpy as np - LOCAL_DIR = os.getenv("BAG3D_TEST_DATA") HOST = os.getenv("BAG3D_PG_HOST") PORT = os.getenv("BAG3D_PG_PORT") @@ -116,15 +114,21 @@ def mock_party_walls_nl(intermediate_data_dir) -> pd.DataFrame: @pytest.fixture(scope="session") -def mock_features_file_index(intermediate_data_dir, party_walls_file_store_fastssd): - data = pickle.load(open(intermediate_data_dir / "features_file_index.pkl", "rb")) - for k, v in data.items(): - data[k] = Path( - str(v) - .replace(str(v.parents[8]), str(party_walls_file_store_fastssd)) - .replace("export", "export_test_version") - ) - return data +def mock_features_file_index(party_walls_file_store_fastssd): + return { + "NL.IMBAG.Pand.0307100000308298": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000308298/reconstruct/NL.IMBAG.Pand.0307100000308298.city.jsonl", + "NL.IMBAG.Pand.0307100000368987": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000368987/reconstruct/NL.IMBAG.Pand.0307100000368987.city.jsonl", + "NL.IMBAG.Pand.0307100000547663": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000547663/reconstruct/NL.IMBAG.Pand.0307100000547663.city.jsonl", + "NL.IMBAG.Pand.0307100000536600": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000536600/reconstruct/NL.IMBAG.Pand.0307100000536600.city.jsonl", + "NL.IMBAG.Pand.0307100000313420": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000313420/reconstruct/NL.IMBAG.Pand.0307100000313420.city.jsonl", + "NL.IMBAG.Pand.0307100000332591": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000332591/reconstruct/NL.IMBAG.Pand.0307100000332591.city.jsonl", + } @pytest.fixture(scope="session") From 86dce57a3a19893832ba3d359498519aed5f9972 Mon Sep 17 00:00:00 2001 From: GinaStavropoulou Date: Tue, 19 Aug 2025 17:13:50 +0200 Subject: [PATCH 28/28] more party wall test fixes --- packages/party_walls/tests/conftest.py | 32 +------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index 221e20de..d29ace64 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -9,7 +9,7 @@ from bag3d.party_walls.assets.party_walls import ( TilesFilesIndex, ) -from dagster import AssetKey, IOManager, SourceAsset, build_op_context +from dagster import build_op_context import pandas as pd from shapely import STRtree, from_wkt import numpy as np @@ -161,33 +161,3 @@ def mock_distribution_tiles_files_index(party_walls_file_store): return TilesFilesIndex( export_results=export_results, tree=tree, paths_array=paths_array ) - - -@pytest.fixture(scope="session") -def mock_asset_features_file_index(mock_features_file_index): - class MockIOManager(IOManager): - def load_input(self, context): - return mock_features_file_index - - def handle_output(self, context, obj): # pragma: no cover - raise NotImplementedError() - - return SourceAsset( - key=AssetKey(["party_walls", "features_file_index"]), - io_manager_def=MockIOManager(), - ) - - -@pytest.fixture(scope="session") -def mock_asset_distribution_tiles_files_index(mock_distribution_tiles_files_index): - class MockIOManager(IOManager): - def load_input(self, context): - return mock_distribution_tiles_files_index - - def handle_output(self, context, obj): # pragma: no cover - raise NotImplementedError() - - return SourceAsset( - key=AssetKey(["party_walls", "distribution_tiles_files_index"]), - io_manager_def=MockIOManager(), - )