Merge pull request #269 from 3DBAG/fix/235--update-test-data

Fix/235  update test data

Author: balazsdukai

makefile

@@ -113,7 +113,7 @@ include .env
 download:
 	rm -rf $(BAG3D_TEST_DATA)
 	mkdir -p $(BAG3D_TEST_DATA)
-	cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v13.zip ; unzip -q test_data_v13.zip ; rm test_data_v13.zip
+	cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v14.zip ; unzip -q test_data_v14.zip ; rm test_data_v14.zip
 
 
 install_uv:

packages/common/src/bag3d/common/resources/__init__.py

@@ -1,5 +1,4 @@
 import os
-from pathlib import Path
 
 from bag3d.common.resources.executables import (
     GDALResource,
@@ -60,17 +59,6 @@
 file_store = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE"))
 file_store_fastssd = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE_FASTSSD"))
 
-file_store_test = FileStoreResource(
-    data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input")
-)
-file_store_fastssd_test = FileStoreResource(
-    data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "integration_core")
-)
-
-# Configure for  gilfoyle
-file_store_gilfoyle = FileStoreResource(data_dir="/data")
-file_store_gilfoyle_fastssd = FileStoreResource(data_dir="/fastssd/data")
-
 
 lastools = LASToolsResource(
     exe_lasindex=os.getenv("EXE_PATH_LASINDEX"),
@@ -118,23 +106,6 @@
 }
 
 
-# RESOURCES_TEST = {
-#     "gdal": gdal,
-#     "file_store": file_store_test,
-#     "file_store_fastssd": file_store_fastssd_test,
-#     "db_connection": db_connection,
-#     "pdal": pdal,
-#     "lastools": lastools,
-#     "tyler": tyler,
-#     "geoflow": geoflow,
-#     "validation": validation,
-#     "roofer": roofer,
-#     "version": version,
-#     "specs": specs,
-#     "godzilla_server": godzilla_server,
-#     "podzilla_server": podzilla_server,
-# }
-#
 # RESOURCES_PROD = {
 #     "gdal": gdal,
 #     "file_store": file_store_gilfoyle,

packages/common/src/bag3d/common/utils/files.py

@@ -101,7 +101,9 @@ def get_export_tile_ids() -> Sequence[str]:
 
     env = os.getenv("DAGSTER_ENVIRONMENT", "test")
     if env == "test":
-        root_dir = Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input"
+        root_dir = (
+            Path(os.getenv("BAG3D_FILESTORE")) / "integration_party_walls/file_store"
+        )
         version = "test_version"
     else:
         root_dir = Path(os.getenv("BAG3D_FILESTORE", "/data"))

packages/common/tests/conftest.py

@@ -109,5 +109,8 @@ def test_data_dir():
 
 
 @pytest.fixture(scope="session")
-def laz_files_ahn3_dir(test_data_dir):
-    yield test_data_dir / "reconstruction_input/pointcloud/AHN3/as_downloaded/LAZ"
+def sample_laz_file(test_data_dir):
+    yield (
+        test_data_dir
+        / "integration_core/file_store/pointcloud/AHN3/as_downloaded/LAZ/C_32BZ2.LAZ"
+    )

packages/common/tests/test_resources.py

@@ -37,16 +37,15 @@ def test_gdal_local(test_data_dir):
     assert return_code == 0
 
 
-def test_pdal_local(laz_files_ahn3_dir):
+def test_pdal_local(sample_laz_file):
     """Use local PDAL installation"""
     pdal = PDALResource(exe_pdal=EnvVar("EXE_PATH_PDAL").get_value())
     assert not pdal.with_docker
-    filepath = laz_files_ahn3_dir / "C_32BZ2.LAZ"
-    return_code, output = pdal_info(pdal.app, filepath, with_all=True)
+    return_code, output = pdal_info(pdal.app, sample_laz_file, with_all=True)
     assert return_code == 0
 
 
-def test_lastools(laz_files_ahn3_dir):
+def test_lastools(sample_laz_file):
     lastools_resource = LASToolsResource(
         exe_lasindex=EnvVar("EXE_PATH_LASINDEX").get_value(),
         exe_las2las=EnvVar("EXE_PATH_LAS2LAS").get_value(),
@@ -55,8 +54,6 @@ def test_lastools(laz_files_ahn3_dir):
 
     lastools = lastools_resource.app
 
-    filepath = laz_files_ahn3_dir / "C_32BZ2.LAZ"
-
     cmd_list = [
         "{exe}",
         "-i {local_path}",
@@ -66,7 +63,7 @@ def test_lastools(laz_files_ahn3_dir):
         "-dont_reindex",
     ]
     return_code, output = lastools.execute(
-        "lasindex", " ".join(cmd_list), local_path=filepath
+        "lasindex", " ".join(cmd_list), local_path=sample_laz_file
     )
 
     assert return_code == 0

packages/core/src/bag3d/core/assets/ahn/metadata.py

@@ -241,7 +241,7 @@ def compute_load_metadata(
             ST_SetSRID(ST_GeomFromGeoJSON({boundary}), 28992)
         );
         """).format(**query_params)
-    context.log.info(conn.print_query(query))
+    context.log.debug(conn.print_query(query))
     conn.send_query(query)
     # Cannot index the table here, because this is a partitioned assed. This means that
     # this function is called for each partition, which would index the table after

packages/core/tests/conftest.py

@@ -168,6 +168,30 @@ def context(
     )
 
 
+@pytest.fixture
+def context_ahn(
+    database,
+    file_store,
+    gdal,
+    validation,
+    godzilla_server,
+    podzilla_server,
+):
+    yield build_op_context(
+        partition_key="01cz1",
+        resources={
+            "gdal": gdal,
+            "validation": validation,
+            "db_connection": database,
+            "file_store": file_store,
+            "version": VersionResource("test_version"),
+            "godzilla_server": godzilla_server,
+            "podzilla_server": podzilla_server,
+            "specs": Specs3DBAGResource(),
+        },
+    )
+
+
 @pytest.fixture
 def context_missing(
     database, wkt_testarea, file_store, gdal_missing, validation_missing
@@ -265,6 +289,23 @@ def test_data_dir():
     yield Path(LOCAL_DIR)
 
 
+@pytest.fixture(scope="session")
+def core_integration_test_dir(test_data_dir):
+    yield test_data_dir / "integration_core"
+
+
+@pytest.fixture(scope="session")
+def core_file_store_fastssd(core_integration_test_dir) -> Path:
+    """Root directory path for test data"""
+    return core_integration_test_dir / "file_store_fastssd"
+
+
+@pytest.fixture(scope="session")
+def core_file_store(core_integration_test_dir) -> Path:
+    """Root directory path for test data"""
+    return core_integration_test_dir / "file_store"
+
+
 @pytest.fixture(scope="session")
 def md5_ahn3_fix():
     yield {"C_01CZ1.LAZ": "063b23d038f97576d279fb7d8a1481ad"}

packages/core/tests/test_assets_ahn.py

@@ -51,77 +51,77 @@ def test_get_checksums(ahn_version):
         assert sha is not None
 
 
-def test_checksums_for_ahn(context):
-    res = md5_ahn3(context)
+def test_checksums_for_ahn(context_ahn):
+    res = md5_ahn3(context_ahn)
     assert len(res) > 0
     for k, sha in list(res.items())[:5]:
         assert sha is not None
-    res = md5_ahn4(context)
+    res = md5_ahn4(context_ahn)
     assert len(res) > 0
     for k, sha in list(res.items())[:5]:
         assert sha is not None
-    res = sha256_ahn5(context)
+    res = sha256_ahn5(context_ahn)
     assert len(res) > 0
     for k, sha in list(res.items())[:5]:
         assert sha is not None
 
 
-def test_tile_index_ahn(context):
-    res = tile_index_ahn(context)
+def test_tile_index_ahn(context_ahn):
+    res = tile_index_ahn(context_ahn)
     assert len(res) == 1407
     assert res[list(res.keys())[0]] is not None
 
 
 @pytest.mark.slow
-def test_laz_files_ahn3(context, md5_ahn3_fix, tile_index_ahn_fix):
-    laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 3)
+def test_laz_files_ahn3(context_ahn, md5_ahn3_fix, tile_index_ahn_fix):
+    laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 3)
     laz_dir.mkdir(exist_ok=True, parents=True)
     config = LazFilesConfig(force_download=False, check_hash=False)
-    res = laz_files_ahn3(context, config, md5_ahn3_fix, tile_index_ahn_fix)
+    res = laz_files_ahn3(context_ahn, config, md5_ahn3_fix, tile_index_ahn_fix)
     assert res.value.url is not None
     assert res is not None
     print(res.value)
 
 
 @pytest.mark.slow
-def test_laz_files_ahn4(context, md5_ahn4_fix, tile_index_ahn_fix):
-    laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 4)
+def test_laz_files_ahn4(context_ahn, md5_ahn4_fix, tile_index_ahn_fix):
+    laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 4)
     laz_dir.mkdir(exist_ok=True, parents=True)
     config = LazFilesConfig(force_download=False, check_hash=False)
-    res = laz_files_ahn4(context, config, md5_ahn4_fix, tile_index_ahn_fix)
+    res = laz_files_ahn4(context_ahn, config, md5_ahn4_fix, tile_index_ahn_fix)
     assert res.value.url is not None
     assert res is not None
 
 
 @pytest.mark.slow
-def test_laz_files_ahn5(context, sha256_ahn5_fix, tile_index_ahn_fix):
-    laz_dir = ahn_laz_dir(context.resources.file_store.file_store.data_dir, 5)
+def test_laz_files_ahn5(context_ahn, sha256_ahn5_fix, tile_index_ahn_fix):
+    laz_dir = ahn_laz_dir(context_ahn.resources.file_store.file_store.data_dir, 5)
     laz_dir.mkdir(exist_ok=True, parents=True)
     config = LazFilesConfig(force_download=False, check_hash=False)
-    res = laz_files_ahn5(context, config, sha256_ahn5_fix, tile_index_ahn_fix)
+    res = laz_files_ahn5(context_ahn, config, sha256_ahn5_fix, tile_index_ahn_fix)
     assert res.value.url is not None
     assert res is not None
 
 
-def test_metadata_table_ahn3(context):
-    metadata = metadata_table_ahn3(context)
+def test_metadata_table_ahn3(context_ahn):
+    metadata = metadata_table_ahn3(context_ahn)
     tbl = PostgresTableIdentifier("ahn", "metadata_ahn3")
-    assert table_exists(context, tbl)
+    assert table_exists(context_ahn, tbl)
     assert isinstance(metadata, PostgresTableIdentifier)
     assert str(metadata) == f"{tbl.schema}.{tbl.table}"
 
 
-def test_metadata_table_ahn4(context):
-    metadata = metadata_table_ahn4(context)
+def test_metadata_table_ahn4(context_ahn):
+    metadata = metadata_table_ahn4(context_ahn)
     tbl = PostgresTableIdentifier("ahn", "metadata_ahn4")
-    assert table_exists(context, tbl)
+    assert table_exists(context_ahn, tbl)
     assert isinstance(metadata, PostgresTableIdentifier)
     assert str(metadata) == f"{tbl.schema}.{tbl.table}"
 
 
-def test_metadata_table_ahn5(context):
-    metadata = metadata_table_ahn5(context)
+def test_metadata_table_ahn5(context_ahn):
+    metadata = metadata_table_ahn5(context_ahn)
     tbl = PostgresTableIdentifier("ahn", "metadata_ahn5")
-    assert table_exists(context, tbl)
+    assert table_exists(context_ahn, tbl)
     assert isinstance(metadata, PostgresTableIdentifier)
     assert str(metadata) == f"{tbl.schema}.{tbl.table}"

packages/core/tests/test_assets_top10nl.py

@@ -7,4 +7,3 @@ def test_extract_top10nl(context_top10nl):
     """Does the complete asset work?"""
     res = download.extract_top10nl(context_top10nl)
     assert res.value.exists()
-    context_top10nl.resources.file_store.file_store.rm(force=True)

packages/core/tests/test_integration.py

@@ -36,7 +36,7 @@
 
 
 @pytest.mark.needs_tools
-def test_integration_ahn(database, test_data_dir):
+def test_integration_ahn(database, core_file_store):
     """Test the ahn jobs."""
     resources = {
         "lastools": LASToolsResource(
@@ -47,9 +47,7 @@ def test_integration_ahn(database, test_data_dir):
             exe_pdal=os.getenv("EXE_PATH_PDAL"),
         ),
         "db_connection": database,
-        "file_store": FileStoreResource(
-            data_dir=str(test_data_dir / "reconstruction_input")
-        ),
+        "file_store": FileStoreResource(data_dir=str(core_file_store)),
     }
 
     all_ahn_assets = load_assets_from_package_module(
@@ -108,6 +106,8 @@ def test_integration_ahn(database, test_data_dir):
 def test_integration_reconstruction_and_export(
     database,
     test_data_dir,
+    core_file_store,
+    core_file_store_fastssd,
     mock_asset_reconstruction_input,
     mock_asset_tiles,
     mock_asset_index,
@@ -117,9 +117,7 @@ def test_integration_reconstruction_and_export(
 ):
     # update quadtree
     og_quadtree = test_data_dir / "quadtree.tsv"
-    export_dir = (
-        test_data_dir / "reconstruction_input" / "3DBAG" / "export_test_version"
-    )
+    export_dir = core_file_store / "3DBAG" / "export_test_version"
     export_dir.mkdir(exist_ok=True)
     os.system(f"cp {og_quadtree} {export_dir}")
 
@@ -143,12 +141,8 @@ def test_integration_reconstruction_and_export(
             exe_sozip=os.getenv("EXE_PATH_SOZIP"),
         ),
         "db_connection": database,
-        "file_store": FileStoreResource(
-            data_dir=str(test_data_dir / "reconstruction_input")
-        ),
-        "file_store_fastssd": FileStoreResource(
-            data_dir=str(test_data_dir / "integration_core")
-        ),
+        "file_store": FileStoreResource(data_dir=str(core_file_store)),
+        "file_store_fastssd": FileStoreResource(data_dir=str(core_file_store_fastssd)),
         "version": VersionResource("test_version"),
         "validation": ValidationResource(
             exe_val3dity=os.getenv("EXE_PATH_VAL3DITY"),
@@ -203,7 +197,6 @@ def test_integration_reconstruction_and_export(
                 }
             },
         )
-
         assert isinstance(result, ExecuteInProcessResult)
         assert result.success
 
@@ -219,13 +212,11 @@ def test_integration_reconstruction_and_export(
                 }
             },
         )
-
         assert isinstance(result, ExecuteInProcessResult)
         assert result.success
 
         resolved_job = defs.get_job_def("nl_export_after_floors")
         result = resolved_job.execute_in_process(instance=instance, resources=resources)
-
         assert isinstance(result, ExecuteInProcessResult)
         assert result.success