diff --git a/makefile b/makefile index c4cca2a6..65381af3 100644 --- a/makefile +++ b/makefile @@ -113,7 +113,7 @@ include .env download: rm -rf $(BAG3D_TEST_DATA) mkdir -p $(BAG3D_TEST_DATA) - cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v13.zip ; unzip -q test_data_v13.zip ; rm test_data_v13.zip + cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v14.zip ; unzip -q test_data_v14.zip ; rm test_data_v14.zip install_uv: diff --git a/packages/common/src/bag3d/common/resources/__init__.py b/packages/common/src/bag3d/common/resources/__init__.py index 92bb3575..76fb28a4 100644 --- a/packages/common/src/bag3d/common/resources/__init__.py +++ b/packages/common/src/bag3d/common/resources/__init__.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from bag3d.common.resources.executables import ( GDALResource, @@ -60,17 +59,6 @@ file_store = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE")) file_store_fastssd = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE_FASTSSD")) -file_store_test = FileStoreResource( - data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input") -) -file_store_fastssd_test = FileStoreResource( - data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "integration_core") -) - -# Configure for gilfoyle -file_store_gilfoyle = FileStoreResource(data_dir="/data") -file_store_gilfoyle_fastssd = FileStoreResource(data_dir="/fastssd/data") - lastools = LASToolsResource( exe_lasindex=os.getenv("EXE_PATH_LASINDEX"), @@ -118,23 +106,6 @@ } -# RESOURCES_TEST = { -# "gdal": gdal, -# "file_store": file_store_test, -# "file_store_fastssd": file_store_fastssd_test, -# "db_connection": db_connection, -# "pdal": pdal, -# "lastools": lastools, -# "tyler": tyler, -# "geoflow": geoflow, -# "validation": validation, -# "roofer": roofer, -# "version": version, -# "specs": specs, -# "godzilla_server": godzilla_server, -# "podzilla_server": podzilla_server, -# } -# # RESOURCES_PROD = { # "gdal": gdal, # "file_store": file_store_gilfoyle, diff --git a/packages/common/src/bag3d/common/utils/files.py b/packages/common/src/bag3d/common/utils/files.py index aa497707..b25f3d9a 100644 --- a/packages/common/src/bag3d/common/utils/files.py +++ b/packages/common/src/bag3d/common/utils/files.py @@ -101,7 +101,9 @@ def get_export_tile_ids() -> Sequence[str]: env = os.getenv("DAGSTER_ENVIRONMENT", "test") if env == "test": - root_dir = Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input" + root_dir = ( + Path(os.getenv("BAG3D_FILESTORE")) / "integration_party_walls/file_store" + ) version = "test_version" else: root_dir = Path(os.getenv("BAG3D_FILESTORE", "/data")) diff --git a/packages/common/tests/conftest.py b/packages/common/tests/conftest.py index c267cf22..b0bf59dc 100644 --- a/packages/common/tests/conftest.py +++ b/packages/common/tests/conftest.py @@ -109,5 +109,8 @@ def test_data_dir(): @pytest.fixture(scope="session") -def laz_files_ahn3_dir(test_data_dir): - yield test_data_dir / "reconstruction_input/pointcloud/AHN3/as_downloaded/LAZ" +def sample_laz_file(test_data_dir): + yield ( + test_data_dir + / "integration_core/file_store/pointcloud/AHN3/as_downloaded/LAZ/C_32BZ2.LAZ" + ) diff --git a/packages/common/tests/test_resources.py b/packages/common/tests/test_resources.py index 730175e3..e01d264e 100644 --- a/packages/common/tests/test_resources.py +++ b/packages/common/tests/test_resources.py @@ -37,16 +37,15 @@ def test_gdal_local(test_data_dir): assert return_code == 0 -def test_pdal_local(laz_files_ahn3_dir): +def test_pdal_local(sample_laz_file): """Use local PDAL installation""" pdal = PDALResource(exe_pdal=EnvVar("EXE_PATH_PDAL").get_value()) assert not pdal.with_docker - filepath = laz_files_ahn3_dir / "C_32BZ2.LAZ" - return_code, output = pdal_info(pdal.app, filepath, with_all=True) + return_code, output = pdal_info(pdal.app, sample_laz_file, with_all=True) assert return_code == 0 -def test_lastools(laz_files_ahn3_dir): +def test_lastools(sample_laz_file): lastools_resource = LASToolsResource( exe_lasindex=EnvVar("EXE_PATH_LASINDEX").get_value(), exe_las2las=EnvVar("EXE_PATH_LAS2LAS").get_value(), @@ -55,8 +54,6 @@ def test_lastools(laz_files_ahn3_dir): lastools = lastools_resource.app - filepath = laz_files_ahn3_dir / "C_32BZ2.LAZ" - cmd_list = [ "{exe}", "-i {local_path}", @@ -66,7 +63,7 @@ def test_lastools(laz_files_ahn3_dir): "-dont_reindex", ] return_code, output = lastools.execute( - "lasindex", " ".join(cmd_list), local_path=filepath + "lasindex", " ".join(cmd_list), local_path=sample_laz_file ) assert return_code == 0 diff --git a/packages/core/src/bag3d/core/assets/ahn/metadata.py b/packages/core/src/bag3d/core/assets/ahn/metadata.py index b3542200..370e6787 100644 --- a/packages/core/src/bag3d/core/assets/ahn/metadata.py +++ b/packages/core/src/bag3d/core/assets/ahn/metadata.py @@ -247,7 +247,7 @@ def compute_load_metadata( ST_SetSRID(ST_GeomFromGeoJSON({boundary}), 28992) ); """).format(**query_params) - context.log.info(conn.print_query(query)) + context.log.debug(conn.print_query(query)) conn.send_query(query) # Cannot index the table here, because this is a partitioned assed. This means that # this function is called for each partition, which would index the table after diff --git a/packages/core/tests/conftest.py b/packages/core/tests/conftest.py index a0bc0428..0be3be68 100644 --- a/packages/core/tests/conftest.py +++ b/packages/core/tests/conftest.py @@ -168,6 +168,30 @@ def context( ) +@pytest.fixture +def context_ahn( + database, + file_store, + gdal, + validation, + godzilla_server, + podzilla_server, +): + yield build_op_context( + partition_key="01cz1", + resources={ + "gdal": gdal, + "validation": validation, + "db_connection": database, + "file_store": file_store, + "version": VersionResource("test_version"), + "godzilla_server": godzilla_server, + "podzilla_server": podzilla_server, + "specs": Specs3DBAGResource(), + }, + ) + + @pytest.fixture def context_missing( database, wkt_testarea, file_store, gdal_missing, validation_missing @@ -265,6 +289,23 @@ def test_data_dir(): yield Path(LOCAL_DIR) +@pytest.fixture(scope="session") +def core_integration_test_dir(test_data_dir): + yield test_data_dir / "integration_core" + + +@pytest.fixture(scope="session") +def core_file_store_fastssd(core_integration_test_dir) -> Path: + """Root directory path for test data""" + return core_integration_test_dir / "file_store_fastssd" + + +@pytest.fixture(scope="session") +def core_file_store(core_integration_test_dir) -> Path: + """Root directory path for test data""" + return core_integration_test_dir / "file_store" + + @pytest.fixture(scope="session") def md5_ahn3_fix(): yield {"C_01CZ1.LAZ": "063b23d038f97576d279fb7d8a1481ad"} diff --git a/packages/core/tests/test_assets_ahn.py b/packages/core/tests/test_assets_ahn.py index b4656a2a..e16a8e0c 100644 --- a/packages/core/tests/test_assets_ahn.py +++ b/packages/core/tests/test_assets_ahn.py @@ -48,77 +48,77 @@ def test_get_checksums(url): assert sha is not None -def test_checksums_for_ahn(context): - res = md5_ahn3(context) +def test_checksums_for_ahn(context_ahn): + res = md5_ahn3(context_ahn) assert len(res) > 0 for k, sha in list(res.items())[:5]: assert sha is not None - res = md5_ahn4(context) + res = md5_ahn4(context_ahn) assert len(res) > 0 for k, sha in list(res.items())[:5]: assert sha is not None - res = sha256_ahn5(context) + res = sha256_ahn5(context_ahn) assert len(res) > 0 for k, sha in list(res.items())[:5]: assert sha is not None -def test_tile_index_ahn(context): - res = tile_index_ahn(context) +def test_tile_index_ahn(context_ahn): + res = tile_index_ahn(context_ahn) assert len(res) == 1407 assert res[list(res.keys())[0]] is not None @pytest.mark.slow -def test_laz_files_ahn3(context, md5_ahn3_fix, tile_index_ahn_fix): - laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 3) +def test_laz_files_ahn3(context_ahn, md5_ahn3_fix, tile_index_ahn_fix): + laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 3) laz_dir.mkdir(exist_ok=True, parents=True) config = LazFilesConfig(force_download=False, check_hash=False) - res = laz_files_ahn3(context, config, md5_ahn3_fix, tile_index_ahn_fix) + res = laz_files_ahn3(context_ahn, config, md5_ahn3_fix, tile_index_ahn_fix) assert res.value.url is not None assert res is not None print(res.value) @pytest.mark.slow -def test_laz_files_ahn4(context, md5_ahn4_fix, tile_index_ahn_fix): - laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 4) +def test_laz_files_ahn4(context_ahn, md5_ahn4_fix, tile_index_ahn_fix): + laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 4) laz_dir.mkdir(exist_ok=True, parents=True) config = LazFilesConfig(force_download=False, check_hash=False) - res = laz_files_ahn4(context, config, md5_ahn4_fix, tile_index_ahn_fix) + res = laz_files_ahn4(context_ahn, config, md5_ahn4_fix, tile_index_ahn_fix) assert res.value.url is not None assert res is not None @pytest.mark.slow -def test_laz_files_ahn5(context, sha256_ahn5_fix, tile_index_ahn_fix): - laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 5) +def test_laz_files_ahn5(context_ahn, sha256_ahn5_fix, tile_index_ahn_fix): + laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 5) laz_dir.mkdir(exist_ok=True, parents=True) config = LazFilesConfig(force_download=False, check_hash=False) - res = laz_files_ahn5(context, config, sha256_ahn5_fix, tile_index_ahn_fix) + res = laz_files_ahn5(context_ahn, config, sha256_ahn5_fix, tile_index_ahn_fix) assert res.value.url is not None assert res is not None -def test_metadata_table_ahn3(context): - metadata = metadata_table_ahn3(context) +def test_metadata_table_ahn3(context_ahn): + metadata = metadata_table_ahn3(context_ahn) tbl = PostgresTableIdentifier("ahn", "metadata_ahn3") - assert table_exists(context, tbl) + assert table_exists(context_ahn, tbl) assert isinstance(metadata, PostgresTableIdentifier) assert str(metadata) == f"{tbl.schema}.{tbl.table}" -def test_metadata_table_ahn4(context): - metadata = metadata_table_ahn4(context) +def test_metadata_table_ahn4(context_ahn): + metadata = metadata_table_ahn4(context_ahn) tbl = PostgresTableIdentifier("ahn", "metadata_ahn4") - assert table_exists(context, tbl) + assert table_exists(context_ahn, tbl) assert isinstance(metadata, PostgresTableIdentifier) assert str(metadata) == f"{tbl.schema}.{tbl.table}" -def test_metadata_table_ahn5(context): - metadata = metadata_table_ahn5(context) +def test_metadata_table_ahn5(context_ahn): + metadata = metadata_table_ahn5(context_ahn) tbl = PostgresTableIdentifier("ahn", "metadata_ahn5") - assert table_exists(context, tbl) + assert table_exists(context_ahn, tbl) assert isinstance(metadata, PostgresTableIdentifier) assert str(metadata) == f"{tbl.schema}.{tbl.table}" diff --git a/packages/core/tests/test_assets_top10nl.py b/packages/core/tests/test_assets_top10nl.py index e17e37f0..c7a8385c 100644 --- a/packages/core/tests/test_assets_top10nl.py +++ b/packages/core/tests/test_assets_top10nl.py @@ -7,4 +7,3 @@ def test_extract_top10nl(context_top10nl): """Does the complete asset work?""" res = download.extract_top10nl(context_top10nl) assert res.value.exists() - context_top10nl.resources.file_store.file_store.rm(force=True) diff --git a/packages/core/tests/test_integration.py b/packages/core/tests/test_integration.py index 95e78e48..4c8f4b3f 100644 --- a/packages/core/tests/test_integration.py +++ b/packages/core/tests/test_integration.py @@ -36,7 +36,7 @@ @pytest.mark.needs_tools -def test_integration_ahn(database, test_data_dir): +def test_integration_ahn(database, core_file_store): """Test the ahn jobs.""" resources = { "lastools": LASToolsResource( @@ -47,9 +47,7 @@ def test_integration_ahn(database, test_data_dir): exe_pdal=os.getenv("EXE_PATH_PDAL"), ), "db_connection": database, - "file_store": FileStoreResource( - data_dir=str(test_data_dir / "reconstruction_input") - ), + "file_store": FileStoreResource(data_dir=str(core_file_store)), } all_ahn_assets = load_assets_from_package_module( @@ -108,6 +106,8 @@ def test_integration_ahn(database, test_data_dir): def test_integration_reconstruction_and_export( database, test_data_dir, + core_file_store, + core_file_store_fastssd, mock_asset_reconstruction_input, mock_asset_tiles, mock_asset_index, @@ -117,9 +117,7 @@ def test_integration_reconstruction_and_export( ): # update quadtree og_quadtree = test_data_dir / "quadtree.tsv" - export_dir = ( - test_data_dir / "reconstruction_input" / "3DBAG" / "export_test_version" - ) + export_dir = core_file_store / "3DBAG" / "export_test_version" export_dir.mkdir(exist_ok=True) os.system(f"cp {og_quadtree} {export_dir}") @@ -143,12 +141,8 @@ def test_integration_reconstruction_and_export( exe_sozip=os.getenv("EXE_PATH_SOZIP"), ), "db_connection": database, - "file_store": FileStoreResource( - data_dir=str(test_data_dir / "reconstruction_input") - ), - "file_store_fastssd": FileStoreResource( - data_dir=str(test_data_dir / "integration_core") - ), + "file_store": FileStoreResource(data_dir=str(core_file_store)), + "file_store_fastssd": FileStoreResource(data_dir=str(core_file_store_fastssd)), "version": VersionResource("test_version"), "validation": ValidationResource( exe_val3dity=os.getenv("EXE_PATH_VAL3DITY"), @@ -203,7 +197,6 @@ def test_integration_reconstruction_and_export( } }, ) - assert isinstance(result, ExecuteInProcessResult) assert result.success @@ -219,13 +212,11 @@ def test_integration_reconstruction_and_export( } }, ) - assert isinstance(result, ExecuteInProcessResult) assert result.success resolved_job = defs.get_job_def("nl_export_after_floors") result = resolved_job.execute_in_process(instance=instance, resources=resources) - assert isinstance(result, ExecuteInProcessResult) assert result.success diff --git a/packages/core/tests/test_validate.py b/packages/core/tests/test_validate.py index 8c48a8f1..a54999dc 100644 --- a/packages/core/tests/test_validate.py +++ b/packages/core/tests/test_validate.py @@ -16,7 +16,7 @@ def test_obj(context, test_data_dir): res = obj( context.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, snap_tol=0.0001, @@ -31,15 +31,15 @@ def test_gpkg(context, test_data_dir): res = gpkg( context.resources.gdal.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", "https://data.3dbag.nl", "test", specs=context.resources.specs, ) assert res.zip_ok assert res.file_ok - assert res.nr_building == 419 - assert res.nr_buildingpart == 422 + assert res.nr_building == 413 + assert res.nr_buildingpart == 414 assert res.nr_invalid_2d_geom == 0 @@ -47,7 +47,7 @@ def test_cityjson(context, test_data_dir): res = cityjson( context.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, url_root="https://data.3dbag.nl", @@ -65,7 +65,7 @@ def test_obj_missing(context_missing, test_data_dir): _ = obj( context_missing.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, snap_tol=0.0001, @@ -79,7 +79,7 @@ def test_gpkg_missing(context_missing, test_data_dir): _ = gpkg( context_missing.resources.gdal.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", "https://data.3dbag.nl", "test", specs=context_missing.resources.specs, @@ -91,7 +91,7 @@ def test_cityjson_missing(context_missing, test_data_dir): _ = cityjson( context_missing.resources.validation.app, test_data_dir / "validation_input/", - "10-564-624", + "0-0-0", planarity_n_tol=20.0, planarity_d2p_tol=0.001, url_root="https://data.3dbag.nl", diff --git a/packages/floors_estimation/tests/conftest.py b/packages/floors_estimation/tests/conftest.py index 055d5c2c..d4f2d734 100644 --- a/packages/floors_estimation/tests/conftest.py +++ b/packages/floors_estimation/tests/conftest.py @@ -1,12 +1,14 @@ import os -import pickle from pathlib import Path +from io import StringIO +import pandas as pd import pytest from bag3d.common.resources.database import DatabaseResource from bag3d.common.resources.files import FileStoreResource from dagster import build_op_context + LOCAL_DIR = os.getenv("BAG3D_TEST_DATA") HOST = os.getenv("BAG3D_PG_HOST") PORT = os.getenv("BAG3D_PG_PORT") @@ -21,21 +23,16 @@ def test_data_dir(): @pytest.fixture(scope="session") -def input_data_dir(test_data_dir) -> Path: - """Root directory path for test data""" - return test_data_dir / "reconstruction_input" +def floors_estimation_integration_test_dir(test_data_dir): + yield test_data_dir / "integration_floors_estimation" @pytest.fixture(scope="session") -def fastssd_data_dir(test_data_dir) -> Path: +def floors_estimation_file_store_fastssd( + floors_estimation_integration_test_dir, +) -> Path: """Root directory path for test data""" - return test_data_dir / "integration_floors_estimation" - - -@pytest.fixture(scope="session") -def intermediate_data_dir(test_data_dir) -> Path: - """Root directory path for test data""" - return test_data_dir / "intermediate_data" + return floors_estimation_integration_test_dir / "file_store_fastssd" @pytest.fixture(scope="session") @@ -53,13 +50,29 @@ def database(): @pytest.fixture -def context(database, input_data_dir, model, fastssd_data_dir): +def context_with_data(floors_estimation_file_store_fastssd): + yield build_op_context( + partition_key="0/0/0", + resources={ + "file_store_fastssd": FileStoreResource( + data_dir=str(floors_estimation_file_store_fastssd) + ), + }, + ) + + +@pytest.fixture +def file_store_tmp(tmp_path): + yield tmp_path + + +@pytest.fixture +def context(database, model, file_store_tmp): yield build_op_context( - partition_key="10/564/624", + partition_key="0/0/0", resources={ "db_connection": database, - "file_store": FileStoreResource(data_dir=str(input_data_dir)), - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)), + "file_store_fastssd": FileStoreResource(data_dir=str(file_store_tmp)), "model_store": model, "version": "test_version", }, @@ -100,23 +113,50 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture(scope="session") -def mock_preprocessed_features(intermediate_data_dir): - return pickle.load(open(intermediate_data_dir / "preprocessed_features.pkl", "rb")) +def mock_preprocessed_features(): + csv_text = """ + identificatie,no_vertices,perimeter,no_units,net_area,building_function,no_neighbours_100,no_adjacent_neighbours,cbs_percent_multihousehold,cbs_pop_per_km2,cbs_dist_to_horeca,buildingtype,construction_year,roof_type,h_roof_50p,h_roof_70p,h_roof_max,h_roof_min,area_roof,area_ext_walls,area_party_walls,area_ground,volume_lod22,volume_lod12 + NL.IMBAG.Pand.0307100000340455,9,55.44252687545277,3,198,1,37,4.0,96,5887,50,2,1927,1,11.029999732971191,11.043000221252441,11.10099983215332,8.204999923706055,160.08,234.48,186.19,156.61,1123.7303466796875,1244.3280029296875 + NL.IMBAG.Pand.0307100000378340,7,21.663911358502567,1,70,0,39,2.0,96,5887,50,4,1922,1,10.729000091552734,10.956000328063965,11.395000457763672,9.892000198364258,34.1,35.98,127.68,23.14,192.20730590820312,204.35214233398438 + NL.IMBAG.Pand.0307100000522025,6,41.48225146934291,1,138,0,124,4.0,77,9573,45,4,1760,1,8.456000328063965,9.131999969482422,10.48900032043457,6.5229997634887695,118.67,91.83,120.32,92.79,526.0918579101562,614.935791015625 + NL.IMBAG.Pand.0307100000351286,19,311.01548191691313,128,6929,1,5,,77,7350,20,2,1979,1,18.472000122070312,24.054000854492188,27.615999221801758,18.336999893188477,2098.48,5430.01,0.0,1976.27,33547.6953125,39757.82421875 + NL.IMBAG.Pand.0307100000353630,11,47.068882927884324,1,194,0,92,4.0,70,9628,120,4,1800,1,11.687000274658203,12.315999984741211,13.361000061035156,6.389999866485596,132.48,147.19,174.05,101.76,762.90283203125,923.0503540039062 + """ + + return pd.read_csv(StringIO(csv_text.strip())) @pytest.fixture(scope="session") -def mock_features_file_index(intermediate_data_dir, fastssd_data_dir): - data = pickle.load( - open( - intermediate_data_dir / "features_file_index_floors_estimation.pkl", - "rb", - ) - ) - for k, v in data.items(): - data[k] = Path(str(v).replace(str(v.parents[5]), str(fastssd_data_dir))) - return data +def mock_features_file_index(floors_estimation_file_store_fastssd): + return { + "NL.IMBAG.Pand.0307100000340455": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000340455.city.jsonl", + "NL.IMBAG.Pand.0307100000364333": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000364333.city.jsonl", + "NL.IMBAG.Pand.0307100000378340": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000378340.city.jsonl", + "NL.IMBAG.Pand.0307100000522025": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522025.city.jsonl", + "NL.IMBAG.Pand.0307100000351286": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000351286.city.jsonl", + "NL.IMBAG.Pand.0307100000522233": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000522233.city.jsonl", + "NL.IMBAG.Pand.0307100000353630": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000353630.city.jsonl", + "NL.IMBAG.Pand.0307100000312499": floors_estimation_file_store_fastssd + / "3DBAG/party_walls_features/0/0/0/NL.IMBAG.Pand.0307100000312499.city.jsonl", + } @pytest.fixture(scope="session") -def mock_inferenced_floors(intermediate_data_dir): - return pickle.load(open(intermediate_data_dir / "inferenced_floors.pkl", "rb")) +def mock_inferenced_floors(): + csv_text = """ + identificatie,no_vertices,perimeter,no_units,net_area,building_function,no_neighbours_100,no_adjacent_neighbours,cbs_percent_multihousehold,cbs_pop_per_km2,cbs_dist_to_horeca,buildingtype,construction_year,roof_type,h_roof_50p,h_roof_70p,h_roof_max,h_roof_min,area_roof,area_ext_walls,area_party_walls,area_ground,volume_lod22,volume_lod12,floors,floors_int + NL.IMBAG.Pand.0307100000340455,9,55.44252687545277,3,198,1,37,4.0,96,5887,50,2,1927,1,11.029999732971191,11.04300022125244,11.10099983215332,8.204999923706055,160.08,234.48,186.19,156.61,1123.7303466796875,1244.3280029296875,3.178904466477873,3.0 + NL.IMBAG.Pand.0307100000378340,7,21.663911358502567,1,70,0,39,2.0,96,5887,50,4,1922,1,10.729000091552734,10.956000328063965,11.395000457763672,9.892000198364258,34.1,35.98,127.68,23.14,192.20730590820312,204.3521423339844,2.7363718856599504,3.0 + NL.IMBAG.Pand.0307100000522025,6,41.48225146934291,1,138,0,124,4.0,77,9573,45,4,1760,1,8.456000328063965,9.131999969482422,10.48900032043457,6.5229997634887695,118.67,91.83,120.32,92.79,526.0918579101562,614.935791015625,2.2665033871332767,2.0 + NL.IMBAG.Pand.0307100000351286,19,311.0154819169132,128,6929,1,5,,77,7350,20,2,1979,1,18.472000122070312,24.054000854492188,27.61599922180176,18.33699989318848,2098.48,5430.01,0.0,1976.27,33547.6953125,39757.82421875,7.52333945870127,8.0 + NL.IMBAG.Pand.0307100000353630,11,47.06888292788432,1,194,0,92,4.0,70,9628,120,4,1800,1,11.687000274658203,12.315999984741213,13.361000061035156,6.389999866485596,132.48,147.19,174.05,101.76,762.90283203125,923.0503540039062,3.1525948513138182,3.0 + """ + + return pd.read_csv(StringIO(csv_text.strip())) diff --git a/packages/floors_estimation/tests/test_floors_estimation.py b/packages/floors_estimation/tests/test_floors_estimation.py index 9a0deaf1..79c73a28 100644 --- a/packages/floors_estimation/tests/test_floors_estimation.py +++ b/packages/floors_estimation/tests/test_floors_estimation.py @@ -15,10 +15,10 @@ ) -def test_features_file_index(context): +def test_features_file_index(context_with_data): """""" - result = features_file_index(context=context) - assert len(result) == 419 + result = features_file_index(context=context_with_data) + assert len(result) == 413 assert "NL.IMBAG.Pand.0307100000377456" in result.keys() assert "party_walls_features" in str(result["NL.IMBAG.Pand.0307100000377456"]) @@ -103,7 +103,7 @@ def test_preprocessed_features(context): assert table_exists(context, all_features_table) is True data = preprocessed_features(context, all_features_table) assert data is not None - assert data.shape[0] == 274 + assert data.shape[0] == 6 def test_inferenced_floors(context, mock_preprocessed_features): @@ -121,7 +121,10 @@ def test_predictions_table(context, mock_inferenced_floors): def test_save_cjfiles( - context, fastssd_data_dir, mock_inferenced_floors, mock_features_file_index + context, + file_store_tmp, + mock_inferenced_floors, + mock_features_file_index, ): save_cjfiles( context, @@ -129,6 +132,6 @@ def test_save_cjfiles( mock_features_file_index, ) assert ( - fastssd_data_dir - / "3DBAG/bouwlagen_features/10/564/624/NL.IMBAG.Pand.0307100000308298.city.jsonl" + file_store_tmp + / "3DBAG/bouwlagen_features/0/0/0/NL.IMBAG.Pand.0307100000364333.city.jsonl" ).exists() diff --git a/packages/floors_estimation/tests/test_integration.py b/packages/floors_estimation/tests/test_integration.py index 3e8f182d..780712e1 100644 --- a/packages/floors_estimation/tests/test_integration.py +++ b/packages/floors_estimation/tests/test_integration.py @@ -5,11 +5,13 @@ @pytest.mark.needs_tools -def test_job_floors_estimation(fastssd_data_dir): +def test_job_floors_estimation(floors_estimation_file_store_fastssd): resolved_job = defs.get_job_def("floors_estimation") resources = { - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)) + "file_store_fastssd": FileStoreResource( + data_dir=str(floors_estimation_file_store_fastssd) + ) } result = resolved_job.execute_in_process(resources=resources) diff --git a/packages/party_walls/tests/conftest.py b/packages/party_walls/tests/conftest.py index 6306cd10..d29ace64 100644 --- a/packages/party_walls/tests/conftest.py +++ b/packages/party_walls/tests/conftest.py @@ -1,13 +1,19 @@ import os -import pickle -from pathlib import Path +from pathlib import Path, PosixPath import pytest from bag3d.common.resources.database import DatabaseResource from bag3d.common.resources.files import FileStoreResource from bag3d.common.resources.version import VersionResource -from dagster import AssetKey, IOManager, SourceAsset, build_op_context +from bag3d.common.types import ExportResult +from bag3d.party_walls.assets.party_walls import ( + TilesFilesIndex, +) +from dagster import build_op_context import pandas as pd +from shapely import STRtree, from_wkt +import numpy as np + LOCAL_DIR = os.getenv("BAG3D_TEST_DATA") HOST = os.getenv("BAG3D_PG_HOST") @@ -17,12 +23,6 @@ DB_NAME = os.getenv("BAG3D_PG_DATABASE") VERSION = "test_version" -# update quadtree -og_quadtree = Path(LOCAL_DIR) / "quadtree.tsv" -export_dir = Path(LOCAL_DIR) / "reconstruction_input" / "3DBAG" / ("export_" + VERSION) -export_dir.mkdir(exist_ok=True, parents=True) -os.system(f"cp {og_quadtree} {export_dir}") - @pytest.fixture(scope="session") def test_data_dir(): @@ -30,15 +30,20 @@ def test_data_dir(): @pytest.fixture(scope="session") -def input_data_dir(test_data_dir) -> Path: +def party_walls_integration_test_dir(test_data_dir): + yield test_data_dir / "integration_party_walls" + + +@pytest.fixture(scope="session") +def party_walls_file_store_fastssd(party_walls_integration_test_dir) -> Path: """Root directory path for test data""" - return test_data_dir / "reconstruction_input" + return party_walls_integration_test_dir / "file_store_fastssd" @pytest.fixture(scope="session") -def fastssd_data_dir(test_data_dir) -> Path: +def party_walls_file_store(party_walls_integration_test_dir) -> Path: """Root directory path for test data""" - return test_data_dir / "integration_party_walls" + return party_walls_integration_test_dir / "file_store" @pytest.fixture(scope="session") @@ -56,13 +61,15 @@ def database(): @pytest.fixture -def context(database, input_data_dir, fastssd_data_dir): +def context(database, party_walls_file_store, party_walls_file_store_fastssd): yield build_op_context( - partition_key="10/564/624", + partition_key="0/0/0", resources={ "db_connection": database, - "file_store": FileStoreResource(data_dir=str(input_data_dir)), - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)), + "file_store": FileStoreResource(data_dir=str(party_walls_file_store)), + "file_store_fastssd": FileStoreResource( + data_dir=str(party_walls_file_store_fastssd) + ), "version": VersionResource(VERSION), }, ) @@ -107,70 +114,50 @@ def mock_party_walls_nl(intermediate_data_dir) -> pd.DataFrame: @pytest.fixture(scope="session") -def mock_features_file_index(intermediate_data_dir, fastssd_data_dir): - data = pickle.load(open(intermediate_data_dir / "features_file_index.pkl", "rb")) - for k, v in data.items(): - data[k] = Path( - str(v) - .replace(str(v.parents[8]), str(fastssd_data_dir)) - .replace("export", "export_test_version") - ) - return data +def mock_features_file_index(party_walls_file_store_fastssd): + return { + "NL.IMBAG.Pand.0307100000308298": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000308298/reconstruct/NL.IMBAG.Pand.0307100000308298.city.jsonl", + "NL.IMBAG.Pand.0307100000368987": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000368987/reconstruct/NL.IMBAG.Pand.0307100000368987.city.jsonl", + "NL.IMBAG.Pand.0307100000547663": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000547663/reconstruct/NL.IMBAG.Pand.0307100000547663.city.jsonl", + "NL.IMBAG.Pand.0307100000536600": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000536600/reconstruct/NL.IMBAG.Pand.0307100000536600.city.jsonl", + "NL.IMBAG.Pand.0307100000313420": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000313420/reconstruct/NL.IMBAG.Pand.0307100000313420.city.jsonl", + "NL.IMBAG.Pand.0307100000332591": party_walls_file_store_fastssd + / "3DBAG/crop_reconstruct/10/564/624/objects/NL.IMBAG.Pand.0307100000332591/reconstruct/NL.IMBAG.Pand.0307100000332591.city.jsonl", + } @pytest.fixture(scope="session") -def mock_distribution_tiles_files_index(intermediate_data_dir, input_data_dir): - data = pickle.load( - open(intermediate_data_dir / "distribution_tiles_files_index.pkl", "rb") - ) - for i, d in enumerate(data.paths_array): - data.paths_array[i] = Path( - str(d) - .replace(str(d.parents[6]), str(input_data_dir)) - .replace("export", "export_test_version") - ) - for k, v in data.export_results.items(): - cj_path = data.export_results[k].cityjson_path - data.export_results[k].cityjson_path = Path( - str(cj_path) - .replace(str(cj_path.parents[6]), str(input_data_dir)) - .replace("export", "export_test_version") +def mock_distribution_tiles_files_index(party_walls_file_store): + export_results = { + "0/0/0": ExportResult( + tile_id="0/0/0", + cityjson_path=PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0.city.json" + ), + gpkg_path=PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0.gpkg" + ), + obj_paths=( + PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0-LoD13-3D.obj" + ), + PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0-LoD12-3D.obj" + ), + PosixPath( + f"{party_walls_file_store}/3DBAG/export_test_version/tiles/0/0/0/0-0-0-LoD22-3D.obj" + ), + ), + wkt="POLYGON((154565.241 462855.414, 155565.241 462855.414, 155565.241 463855.414, 154565.241 463855.414, 154565.241 462855.414))", ) - gpkg_path = data.export_results[k].gpkg_path - data.export_results[k].gpkg_path = Path( - str(gpkg_path) - .replace(str(gpkg_path.parents[6]), str(input_data_dir)) - .replace("export", "export_test_version") - ) - # TODO: fix data.export_results[k].obj_paths - return data - - -@pytest.fixture(scope="session") -def mock_asset_features_file_index(mock_features_file_index): - class MockIOManager(IOManager): - def load_input(self, context): - return mock_features_file_index - - def handle_output(self, context, obj): # pragma: no cover - raise NotImplementedError() - - return SourceAsset( - key=AssetKey(["party_walls", "features_file_index"]), - io_manager_def=MockIOManager(), - ) - - -@pytest.fixture(scope="session") -def mock_asset_distribution_tiles_files_index(mock_distribution_tiles_files_index): - class MockIOManager(IOManager): - def load_input(self, context): - return mock_distribution_tiles_files_index - - def handle_output(self, context, obj): # pragma: no cover - raise NotImplementedError() - - return SourceAsset( - key=AssetKey(["party_walls", "distribution_tiles_files_index"]), - io_manager_def=MockIOManager(), + } + tree = STRtree(tuple(from_wkt(t.wkt) for t in export_results.values())) + paths_array = np.array(tuple(t.cityjson_path for t in export_results.values())) + return TilesFilesIndex( + export_results=export_results, tree=tree, paths_array=paths_array ) diff --git a/packages/party_walls/tests/test_integration.py b/packages/party_walls/tests/test_integration.py index 4024c5c0..97147b38 100644 --- a/packages/party_walls/tests/test_integration.py +++ b/packages/party_walls/tests/test_integration.py @@ -2,61 +2,51 @@ from bag3d.common.resources.files import FileStoreResource from bag3d.common.resources.version import VersionResource from bag3d.party_walls import assets -from bag3d.party_walls.jobs import job_nl_party_walls +from bag3d.party_walls.jobs import job_nl_party_walls, job_nl_party_walls_index from dagster import ( - AssetKey, Definitions, ExecuteInProcessResult, load_assets_from_package_module, + DagsterInstance, ) @pytest.mark.needs_tools def test_job_party_walls( database, - input_data_dir, - fastssd_data_dir, - mock_asset_distribution_tiles_files_index, - mock_asset_features_file_index, + party_walls_file_store, + party_walls_file_store_fastssd, ): resources = { "db_connection": database, - "file_store": FileStoreResource(data_dir=str(input_data_dir)), - "file_store_fastssd": FileStoreResource(data_dir=str(fastssd_data_dir)), + "file_store": FileStoreResource(data_dir=str(party_walls_file_store)), + "file_store_fastssd": FileStoreResource( + data_dir=str(party_walls_file_store_fastssd) + ), "version": VersionResource("test_version"), } all_party_assets = load_assets_from_package_module( assets, key_prefix="party_walls", group_name="party_walls" ) - # Filter the assets to include only the ones we need - party_assets = [ - asset - for asset in all_party_assets - if asset.key - in { - AssetKey(["party_walls", "cityjsonfeatures_with_party_walls_nl"]), - AssetKey(["party_walls", "party_walls_nl"]), - } - ] defs = Definitions( resources=resources, - assets=[ - mock_asset_distribution_tiles_files_index, - mock_asset_features_file_index, - party_assets[0], - party_assets[1], - ], + assets=all_party_assets, jobs=[ + job_nl_party_walls_index, job_nl_party_walls, ], ) - resolved_job = defs.get_job_def("nl_party_walls") - - result = resolved_job.execute_in_process( - resources=resources, partition_key="10/564/624" - ) + with DagsterInstance.ephemeral() as instance: + resolved_job = defs.get_job_def("nl_party_walls_index") + result = resolved_job.execute_in_process(instance=instance, resources=resources) + assert isinstance(result, ExecuteInProcessResult) + assert result.success - assert isinstance(result, ExecuteInProcessResult) - assert result.success + resolved_job = defs.get_job_def("nl_party_walls") + result = resolved_job.execute_in_process( + instance=instance, resources=resources, partition_key="0/0/0" + ) + assert isinstance(result, ExecuteInProcessResult) + assert result.success diff --git a/packages/party_walls/tests/test_party_walls.py b/packages/party_walls/tests/test_party_walls.py index 3257d178..43c1ba0d 100644 --- a/packages/party_walls/tests/test_party_walls.py +++ b/packages/party_walls/tests/test_party_walls.py @@ -6,7 +6,7 @@ party_walls_nl, ) -TILE_IDS = ("10/564/624", "10/564/626", "10/566/624", "10/566/626", "9/560/624") +TILE_IDS = ("0/0/0",) def test_distribution_tiles_files_index(context): @@ -28,9 +28,9 @@ def test_party_walls(context, mock_distribution_tiles_files_index): def test_features_file_index(context): - """Can we find and map all the 5825 cityjson feature files of the test data?""" + """Can we find and map all the cityjson feature files of the test data?""" result = features_file_index(context=context) - assert len(result) == 407 + assert len(result) == 415 @pytest.mark.slow