From 96dc4227afb01a9fe562dd8b820173d8437f0cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Fri, 1 Aug 2025 23:49:47 +0200 Subject: [PATCH 01/13] Refactor ServerTransferResource to include more parameters --- .../bag3d/common/resources/server_transfer.py | 50 +++++++++++++++++-- .../src/bag3d/core/assets/deploy/servers.py | 2 +- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/server_transfer.py b/packages/common/src/bag3d/common/resources/server_transfer.py index 277db6ac..a1107f38 100644 --- a/packages/common/src/bag3d/common/resources/server_transfer.py +++ b/packages/common/src/bag3d/common/resources/server_transfer.py @@ -8,14 +8,58 @@ class ServerTransferResource(ConfigurableResource): """ A resource for transferring files to other servers. + + Attributes: + host: Optional[str] + The hostname or IP address of the remote server. + port: Optional[int] + The port to connect to on the remote server. + user: Optional[str] + The username to use for authentication. + password: Optional[str] + The password to use for authentication (if not using key). + key_filename: Optional[str] + The path to the private key file for key-based authentication. + target_dir: Optional[str] + The default target directory on the remote server for file transfers. + public_dir: Optional[str] + The 3DBAG public directory on the remote server. """ host: Optional[str] = None + port: Optional[int] = None user: Optional[str] = None + password: Optional[str] = None + key_filename: Optional[str] = None target_dir: Optional[str] = None public_dir: Optional[str] = None @property - def connect(self): - conn = Connection(host=self.host, user=self.user) - return conn + def connection(self): + connect_kwargs = {} + if self.key_filename: + connect_kwargs["key_filename"] = self.key_filename + elif self.password: + connect_kwargs["password"] = self.password + return Connection( + host=self.host, + port=self.port, + user=self.user, + connect_kwargs=connect_kwargs, + ) + + def transfer_file(self, local_path, remote_path): + """Transfer a file to remote server.""" + with self.connection as conn: + # Upload the file + conn.put(local_path, remote_path) + + # Verify the file was uploaded + result = conn.run(f"ls -la {remote_path}", hide=True) + return result.ok + + def file_exists(self, remote_path): + """Check if file exists on remote server.""" + with self.connection as conn: + result = conn.run(f"test -f {remote_path}", warn=True, hide=True) + return result.ok diff --git a/packages/core/src/bag3d/core/assets/deploy/servers.py b/packages/core/src/bag3d/core/assets/deploy/servers.py index d143e210..ff7fd390 100644 --- a/packages/core/src/bag3d/core/assets/deploy/servers.py +++ b/packages/core/src/bag3d/core/assets/deploy/servers.py @@ -80,7 +80,7 @@ def transfer_to_server( compressed_file = Path(target_dir) / compressed_export_nl.name try: - with server.connect as c: + with server.connection as c: # test connection result = c.run("echo connected", hide=True) assert result.ok, "Connection command failed" From 96b4c61f1008cb792d29d6ade7fa4ea62b2c2bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Fri, 1 Aug 2025 23:50:10 +0200 Subject: [PATCH 02/13] Remove redundant transfer test --- packages/core/tests/test_assets_deploy.py | 93 +++++------------------ 1 file changed, 17 insertions(+), 76 deletions(-) diff --git a/packages/core/tests/test_assets_deploy.py b/packages/core/tests/test_assets_deploy.py index ab31b42e..c6f37d06 100644 --- a/packages/core/tests/test_assets_deploy.py +++ b/packages/core/tests/test_assets_deploy.py @@ -1,17 +1,10 @@ -from bag3d.core.assets.deploy.servers import ( - compressed_export_nl, - transfer_to_godzilla, - transfer_to_podzilla, -) +from bag3d.common.resources import ServerTransferResource +from bag3d.core.assets.deploy.servers import compressed_export_nl, transfer_to_server from pathlib import Path import pytest -@pytest.mark.skip( - reason="Skip until refactor so that it transfers to a docker container instead of our server, because we should not modify the state of the world outside the test environment" -) -@pytest.mark.needs_tools -def test_transfer_to_podzilla(context, test_data_dir): +def test_transfer_to_server(context, test_data_dir): # Create deployment dir export_dir = test_data_dir / "deployment" / "3DBAG" / "export_test_version" export_dir.mkdir(parents=True, exist_ok=True) @@ -34,65 +27,23 @@ def test_transfer_to_podzilla(context, test_data_dir): assert compressed_file.exists() # Check that the file was created # Test the transfer to podzilla - res = transfer_to_podzilla( - context, - compressed_file, - metadata_file, + target_dir = "/data/3DBAG" + server = ServerTransferResource( + host="3dbag.docker.internal", + port=2222, + user="deploy", + password="deploy", + target_dir=target_dir, + public_dir="/data/3DBAG/public", ) - assert ( - res == f"{context.resources.podzilla_server.target_dir}/test_version" - ) # Check that the function returns a value - finally: - # Clean up the test files - compressed_file.unlink(missing_ok=True) - metadata_file.unlink(missing_ok=True) - empty_file.unlink(missing_ok=True) - export_dir.rmdir() - with context.resources.podzilla_server.connect as c: - c.run( - f"rm -rf {context.resources.podzilla_server.target_dir}/test_version", - warn=True, - ) - c.run( - f"rm -f {context.resources.podzilla_server.target_dir}/export_test_version.tar.gz", - warn=True, - ) - - -@pytest.mark.skip( - reason="Skip until refactor so that it transfers to a docker container instead of our server, because we should not modify the state of the world outside the test environment" -) -@pytest.mark.needs_tools -def test_transfer_to_godzilla(context, test_data_dir): - # Create deployment dir - export_dir = test_data_dir / "deployment" / "3DBAG" / "export_test_version" - export_dir.mkdir(parents=True, exist_ok=True) - - # Create an empty file within the directory - empty_file = export_dir / "dummy.txt" - empty_file.touch() - - # Create a mock metadata file - metadata_file = test_data_dir / "deployment" / "3DBAG" / "metadata.json" - metadata_file.touch() - metadata_file.write_text( - '{"identificationInfo": {"citation": {"edition": "test_version"}}}' - ) - try: - # compress the export dir - res = compressed_export_nl(context, export_dir) - - compressed_file = Path(res.metadata["path"].text) - assert compressed_file.exists() # Check that the file was created - - # Test the transfer to godzilla - res = transfer_to_godzilla( - context, - compressed_file, - metadata_file, + res = transfer_to_server( + server=server, + compressed_export_nl=compressed_file, + metadata=metadata_file, + target_dir=target_dir, ) assert ( - res == f"{context.resources.godzilla_server.target_dir}/test_version" + res == f"{target_dir}/test_version" ) # Check that the function returns a value finally: # Clean up the test files @@ -100,13 +51,3 @@ def test_transfer_to_godzilla(context, test_data_dir): metadata_file.unlink(missing_ok=True) empty_file.unlink(missing_ok=True) export_dir.rmdir() - with context.resources.godzilla_server.connect as c: - c.run( - f"rm -rf {context.resources.godzilla_server.target_dir}/test_version", - warn=True, - ) - c.run(f"rm -rf {context.resources.godzilla_server.public_dir}", warn=True) - c.run( - f"rm -f {context.resources.godzilla_server.target_dir}/export_test_version.tar.gz", - warn=True, - ) From 0993a2164867104a8341a4c731e8025da1048e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Fri, 1 Aug 2025 23:52:14 +0200 Subject: [PATCH 03/13] Connect with deployment docker setup --- docker/compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/compose.yaml b/docker/compose.yaml index f9f00c30..622e76d3 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -88,6 +88,8 @@ services: - ~/.ssh:/root/.ssh:ro networks: - bag3d-network + extra_hosts: + - "3dbag.docker.internal:host-gateway" depends_on: data-postgresql: condition: service_healthy From b9db64d0d066ae77ef45616dbefae2d91c33ea78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Sat, 2 Aug 2025 14:29:38 +0200 Subject: [PATCH 04/13] Add needs_deploy pytest marker --- packages/core/tests/conftest.py | 15 +++++++++++++++ packages/core/tests/test_assets_deploy.py | 1 + 2 files changed, 16 insertions(+) diff --git a/packages/core/tests/conftest.py b/packages/core/tests/conftest.py index a55612d5..f833454f 100644 --- a/packages/core/tests/conftest.py +++ b/packages/core/tests/conftest.py @@ -190,6 +190,12 @@ def pytest_addoption(parser): parser.addoption( "--run-slow", action="store_true", default=False, help="run slow tests" ) + parser.addoption( + "--run-deploy", + action="store_true", + default=False, + help="run deployment tests that require the dockerized deployment setup", + ) parser.addoption( "--run-all", action="store_true", @@ -203,6 +209,9 @@ def pytest_configure(config): config.addinivalue_line( "markers", "needs_tools: mark test as needing local builds of tools" ) + config.addinivalue_line( + "markers", "needs_deploy: mark test as needing the dockerized deployment setup" + ) def pytest_collection_modifyitems(config, items): @@ -218,6 +227,12 @@ def pytest_collection_modifyitems(config, items): if "needs_tools" in item.keywords: item.add_marker(skip_needs_tools) + if not config.getoption("--run-deploy"): # pragma: no cover + skip_needs_deploy = pytest.mark.skip(reason="needs the --run-deploy option to run") + for item in items: + if "needs_deploy" in item.keywords: + item.add_marker(skip_needs_deploy) + @pytest.fixture(scope="session") def test_data_dir(): diff --git a/packages/core/tests/test_assets_deploy.py b/packages/core/tests/test_assets_deploy.py index c6f37d06..16433129 100644 --- a/packages/core/tests/test_assets_deploy.py +++ b/packages/core/tests/test_assets_deploy.py @@ -4,6 +4,7 @@ import pytest +@pytest.mark.needs_deploy def test_transfer_to_server(context, test_data_dir): # Create deployment dir export_dir = test_data_dir / "deployment" / "3DBAG" / "export_test_version" From 76e0937c3dae24ae9d0931425253b2e051cd2185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Sat, 2 Aug 2025 15:06:27 +0200 Subject: [PATCH 05/13] Add deployment_server fixture --- .../bag3d/common/resources/server_transfer.py | 8 +++---- packages/core/tests/conftest.py | 22 ++++++++++++++++++- packages/core/tests/test_assets_deploy.py | 18 +++++---------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/common/src/bag3d/common/resources/server_transfer.py b/packages/common/src/bag3d/common/resources/server_transfer.py index a1107f38..35f65c54 100644 --- a/packages/common/src/bag3d/common/resources/server_transfer.py +++ b/packages/common/src/bag3d/common/resources/server_transfer.py @@ -35,7 +35,7 @@ class ServerTransferResource(ConfigurableResource): public_dir: Optional[str] = None @property - def connection(self): + def connection(self) -> Connection: connect_kwargs = {} if self.key_filename: connect_kwargs["key_filename"] = self.key_filename @@ -48,17 +48,17 @@ def connection(self): connect_kwargs=connect_kwargs, ) - def transfer_file(self, local_path, remote_path): + def transfer_file(self, local_path, remote_path) -> bool: """Transfer a file to remote server.""" with self.connection as conn: # Upload the file conn.put(local_path, remote_path) # Verify the file was uploaded - result = conn.run(f"ls -la {remote_path}", hide=True) + result = conn.run(f"test -f {remote_path}", warn=True, hide=True) return result.ok - def file_exists(self, remote_path): + def file_exists(self, remote_path) -> bool: """Check if file exists on remote server.""" with self.connection as conn: result = conn.run(f"test -f {remote_path}", warn=True, hide=True) diff --git a/packages/core/tests/conftest.py b/packages/core/tests/conftest.py index f833454f..8e4cd121 100644 --- a/packages/core/tests/conftest.py +++ b/packages/core/tests/conftest.py @@ -26,6 +26,24 @@ DB_NAME = os.getenv("BAG3D_PG_DATABASE") +@pytest.fixture(scope="session") +def deployment_server(): + """Connection to the dockerized deployment setup. + The dockerized deployment setup is in the 3dbag-admin repo and it needs to be + managed manually, similar to the 3dbag-pipeline docker setup. + These credentials provide access to the ``deployment-server`` service of the + deployment setup. + """ + yield ServerTransferResource( + host="3dbag.docker.internal", + port=2222, + user="deploy", + password="deploy", + target_dir="/data/3DBAG", + public_dir="/data/3DBAG/public", + ) + + @pytest.fixture(scope="session") def godzilla_server(): yield ServerTransferResource( @@ -228,7 +246,9 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip_needs_tools) if not config.getoption("--run-deploy"): # pragma: no cover - skip_needs_deploy = pytest.mark.skip(reason="needs the --run-deploy option to run") + skip_needs_deploy = pytest.mark.skip( + reason="needs the --run-deploy option to run" + ) for item in items: if "needs_deploy" in item.keywords: item.add_marker(skip_needs_deploy) diff --git a/packages/core/tests/test_assets_deploy.py b/packages/core/tests/test_assets_deploy.py index 16433129..524c0043 100644 --- a/packages/core/tests/test_assets_deploy.py +++ b/packages/core/tests/test_assets_deploy.py @@ -5,7 +5,7 @@ @pytest.mark.needs_deploy -def test_transfer_to_server(context, test_data_dir): +def test_transfer_to_server(context, deployment_server, test_data_dir): # Create deployment dir export_dir = test_data_dir / "deployment" / "3DBAG" / "export_test_version" export_dir.mkdir(parents=True, exist_ok=True) @@ -28,24 +28,16 @@ def test_transfer_to_server(context, test_data_dir): assert compressed_file.exists() # Check that the file was created # Test the transfer to podzilla - target_dir = "/data/3DBAG" - server = ServerTransferResource( - host="3dbag.docker.internal", - port=2222, - user="deploy", - password="deploy", - target_dir=target_dir, - public_dir="/data/3DBAG/public", - ) res = transfer_to_server( - server=server, + server=deployment_server, compressed_export_nl=compressed_file, metadata=metadata_file, - target_dir=target_dir, + target_dir=deployment_server.target_dir, ) assert ( - res == f"{target_dir}/test_version" + res == f"{deployment_server.target_dir}/test_version" ) # Check that the function returns a value + assert deployment_server.file_exists(f"{deployment_server.target_dir}/test_version/dummy.txt") finally: # Clean up the test files compressed_file.unlink(missing_ok=True) From 76e70181849071e53882465b2546b1acd72b264b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Sat, 2 Aug 2025 16:39:16 +0200 Subject: [PATCH 06/13] WIP deploy release integration test --- makefile | 6 +- .../src/bag3d/core/assets/export/archive.py | 56 ++--- .../src/bag3d/core/assets/export/metadata.py | 2 +- .../src/bag3d/core/assets/release/publish.py | 4 +- packages/core/src/bag3d/core/jobs.py | 10 +- packages/core/tests/conftest.py | 214 +++++++++++++++++- packages/core/tests/test_assets_deploy.py | 7 +- packages/core/tests/test_integration.py | 72 +++++- 8 files changed, 317 insertions(+), 54 deletions(-) diff --git a/makefile b/makefile index f6772efb..e1b305f8 100644 --- a/makefile +++ b/makefile @@ -88,9 +88,9 @@ test_slow: test_integration: @set -e; \ FAILED=0; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/test_integration.py -v -s --run-all || FAILED=1; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/test_integration.py -v -s --run-all || FAILED=1; \ - docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/test_integration.py -v -s --run-all || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/test_integration.py -v -s --run-all --run-deploy -k 'test_integration_deploy_release' || FAILED=1; \ +# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/test_integration.py -v -s --run-all || FAILED=1; \ +# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/test_integration.py -v -s --run-all || FAILED=1; \ exit $$FAILED test_all: diff --git a/packages/core/src/bag3d/core/assets/export/archive.py b/packages/core/src/bag3d/core/assets/export/archive.py index b4db0964..047a2a7a 100644 --- a/packages/core/src/bag3d/core/assets/export/archive.py +++ b/packages/core/src/bag3d/core/assets/export/archive.py @@ -129,34 +129,6 @@ def create_path_layer(id_layer, path_tiles_dir): return path_lod12_2d -class CompressionConfig(Config): - concurrency: int - - -@asset( - deps={ - AssetKey("geopackage_nl"), - }, - required_resource_keys={"file_store", "version"}, -) -def compressed_tiles(context, config: CompressionConfig, export_index): - """Each format is gzipped individually in each tile, for better transfer over the - web. The OBJ files are collected into a single .zip file.""" - path_export_dir = bag3d_export_dir( - context.resources.file_store.file_store.data_dir, - version=context.resources.version.version, - ) - path_tiles_dir = path_export_dir.joinpath("tiles") - with export_index.open("r") as fo: - csvreader = csv.reader(fo) - _ = next(csvreader) # skip header - tile_ids = tuple((row[0], path_tiles_dir) for row in csvreader) - - with ProcessPoolExecutor(max_workers=config.concurrency) as executor: - for result in executor.map(compress_files, tile_ids): - pass - - def compress_files(input): tile_id, path_tiles_dir = input path_tile_dir = path_tiles_dir.joinpath(tile_id) @@ -188,3 +160,31 @@ def compress_files(input): with gzip.open(gpkg_zip, "wb") as f_out: copyfileobj(f_in, f_out) gpkg_file.unlink() + + +class CompressionConfig(Config): + concurrency: int + + +@asset( + deps={ + AssetKey("geopackage_nl"), + }, + required_resource_keys={"file_store", "version"}, +) +def compressed_tiles(context, config: CompressionConfig, export_index): + """Each format is gzipped individually in each tile, for better transfer over the + web. The OBJ files are collected into a single .zip file.""" + path_export_dir = bag3d_export_dir( + context.resources.file_store.file_store.data_dir, + version=context.resources.version.version, + ) + path_tiles_dir = path_export_dir.joinpath("tiles") + with export_index.open("r") as fo: + csvreader = csv.reader(fo) + _ = next(csvreader) # skip header + tile_ids = tuple((row[0], path_tiles_dir) for row in csvreader) + + with ProcessPoolExecutor(max_workers=config.concurrency) as executor: + for result in executor.map(compress_files, tile_ids): + pass diff --git a/packages/core/src/bag3d/core/assets/export/metadata.py b/packages/core/src/bag3d/core/assets/export/metadata.py index 11981b39..fc1a5c5d 100644 --- a/packages/core/src/bag3d/core/assets/export/metadata.py +++ b/packages/core/src/bag3d/core/assets/export/metadata.py @@ -152,7 +152,7 @@ def feature_evaluation(context): deps={AssetKey(("export", "reconstruction_output_multitiles_nl"))}, required_resource_keys={"file_store", "version"}, ) -def export_index(context): +def export_index(context) -> Path: """Index of the distribution tiles. Parses the quadtree.tsv file output by *tyler* and checks if all formats exist for diff --git a/packages/core/src/bag3d/core/assets/release/publish.py b/packages/core/src/bag3d/core/assets/release/publish.py index dfdece0e..609cada3 100644 --- a/packages/core/src/bag3d/core/assets/release/publish.py +++ b/packages/core/src/bag3d/core/assets/release/publish.py @@ -37,7 +37,7 @@ def publish_data( compressed_file = Path(data_dir) / compressed_export_nl.name try: - with context.resources.godzilla_server.connect as c: + with context.resources.godzilla_server.connection as c: # test connection result = c.run("echo connected", hide=True) assert result.ok, "Connection command failed" @@ -84,7 +84,7 @@ def publish_webservices(context): alter_dev_to_latest = f"ALTER SCHEMA {dev_schema} RENAME TO {latest_schema};" try: - with context.resources.godzilla_server.connect as c: + with context.resources.godzilla_server.connection as c: context.log.debug(alter_latest_to_archive) c.run( f"psql --dbname baseregisters --port 5432 --host localhost --user etl -c '{alter_latest_to_archive}'" diff --git a/packages/core/src/bag3d/core/jobs.py b/packages/core/src/bag3d/core/jobs.py index 28bad0dc..0779b23f 100644 --- a/packages/core/src/bag3d/core/jobs.py +++ b/packages/core/src/bag3d/core/jobs.py @@ -157,15 +157,15 @@ name="nl_deploy", description="Deploy the Netherland data.", selection=AssetSelection.assets(["deploy", "compressed_export_nl"]) - | AssetSelection.assets(["deploy", "transfer_to_godzilla"]) - | AssetSelection.assets(["deploy", "transfer_to_podzilla"]) - | AssetSelection.assets(["deploy", "webservice_godzilla"]), + | AssetSelection.assets(["deploy", "transfer_to_godzilla"]), + # | AssetSelection.assets(["deploy", "transfer_to_podzilla"]) + # | AssetSelection.assets(["deploy", "webservice_godzilla"]), ) job_nl_release = define_asset_job( name="nl_release", description="Perform the final steps for the 3DBAG release.", - selection=AssetSelection.assets(["release", "publish_data"]) - | AssetSelection.assets(["release", "publish_webservices"]), + selection=AssetSelection.assets(["release", "publish_data"]), + # | AssetSelection.assets(["release", "publish_webservices"]), ) diff --git a/packages/core/tests/conftest.py b/packages/core/tests/conftest.py index 8e4cd121..1d06c830 100644 --- a/packages/core/tests/conftest.py +++ b/packages/core/tests/conftest.py @@ -34,7 +34,7 @@ def deployment_server(): These credentials provide access to the ``deployment-server`` service of the deployment setup. """ - yield ServerTransferResource( + server = ServerTransferResource( host="3dbag.docker.internal", port=2222, user="deploy", @@ -43,23 +43,29 @@ def deployment_server(): public_dir="/data/3DBAG/public", ) + yield server + # + # with server.connection as conn: + # conn.run(f"rm -rf {server.target_dir}") + # conn.run(f"rm -rf {server.public_dir}") + # conn.run(f"mkdir -p {server.target_dir}") + # conn.run(f"mkdir -p {server.public_dir}") + @pytest.fixture(scope="session") -def godzilla_server(): - yield ServerTransferResource( - host="godzilla", - user="", - target_dir="/tmp", - public_dir="/tmp/3dbag_public", - ) +def godzilla_server(deployment_server): + yield deployment_server @pytest.fixture(scope="session") def podzilla_server(): yield ServerTransferResource( - host="podzilla", - user="gstavropoulou", - target_dir="/tmp", + host="3dbag.docker.internal", + port=2222, + user="deploy", + password="deploy", + target_dir="/tmp/podzilla", + public_dir="/tmp/podzilla/public", ) @@ -364,3 +370,189 @@ def handle_output(self, context, obj): # pragma: no cover key=AssetKey(["ahn", "regular_grid_200m"]), io_manager_def=MockIOManager(), ) + + +@pytest.fixture(scope="session") +def mock_asset_compressed_tiles(): + class MockIOManager(IOManager): + def load_input(self, context): + return None + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "compressed_tiles"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_compressed_tiles_validation(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "validate_compressed_files.csv" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "compressed_tiles_validation"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_export_index(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "export_index.csv" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "export_index"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_geopackage_nl(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "3dbag_nl.gpkg.zip" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "geopackage_nl"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_metadata(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "metadata.json" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "metadata"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_reconstruction_output_3dtiles_lod12_nl(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "cesium3dtiles" + / "lod12" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "reconstruction_output_3dtiles_lod12_nl"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_reconstruction_output_3dtiles_lod13_nl(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "cesium3dtiles" + / "lod13" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "reconstruction_output_3dtiles_lod13_nl"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_reconstruction_output_3dtiles_lod22_nl(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "cesium3dtiles" + / "lod22" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "reconstruction_output_3dtiles_lod22_nl"]), + io_manager_def=MockIOManager(), + ) + + +@pytest.fixture(scope="session") +def mock_asset_reconstruction_output_multitiles_nl(test_data_dir): + class MockIOManager(IOManager): + def load_input(self, context): + return ( + test_data_dir + / "integration_deploy_release" + / "3DBAG" + / "export_test_version" + / "tiles" + ) + + def handle_output(self, context, obj): # pragma: no cover + raise NotImplementedError() + + return SourceAsset( + key=AssetKey(["export", "reconstruction_output_multitiles_nl"]), + io_manager_def=MockIOManager(), + ) diff --git a/packages/core/tests/test_assets_deploy.py b/packages/core/tests/test_assets_deploy.py index 524c0043..c7245cfb 100644 --- a/packages/core/tests/test_assets_deploy.py +++ b/packages/core/tests/test_assets_deploy.py @@ -1,10 +1,9 @@ -from bag3d.common.resources import ServerTransferResource from bag3d.core.assets.deploy.servers import compressed_export_nl, transfer_to_server from pathlib import Path import pytest -@pytest.mark.needs_deploy +@pytest.mark.skip("included in integration test") def test_transfer_to_server(context, deployment_server, test_data_dir): # Create deployment dir export_dir = test_data_dir / "deployment" / "3DBAG" / "export_test_version" @@ -37,7 +36,9 @@ def test_transfer_to_server(context, deployment_server, test_data_dir): assert ( res == f"{deployment_server.target_dir}/test_version" ) # Check that the function returns a value - assert deployment_server.file_exists(f"{deployment_server.target_dir}/test_version/dummy.txt") + assert deployment_server.file_exists( + f"{deployment_server.target_dir}/test_version/dummy.txt" + ) finally: # Clean up the test files compressed_file.unlink(missing_ok=True) diff --git a/packages/core/tests/test_integration.py b/packages/core/tests/test_integration.py index de7f6e7e..6eacdbc3 100644 --- a/packages/core/tests/test_integration.py +++ b/packages/core/tests/test_integration.py @@ -12,11 +12,13 @@ ) from bag3d.common.resources.files import FileStoreResource from bag3d.common.resources.version import VersionResource -from bag3d.core.assets import export, reconstruction +from bag3d.core.assets import export, reconstruction, deploy, release from bag3d.core.jobs import ( job_nl_export, job_nl_export_after_floors, job_nl_reconstruct, + job_nl_deploy, + job_nl_release, ) from dagster import ( AssetKey, @@ -140,3 +142,71 @@ def test_integration_reconstruction_and_export( assert isinstance(result, ExecuteInProcessResult) assert result.success + + +@pytest.mark.needs_deploy +def test_integration_deploy_release( + test_data_dir, + godzilla_server, + podzilla_server, + database, + mock_asset_compressed_tiles, + mock_asset_compressed_tiles_validation, + mock_asset_export_index, + mock_asset_geopackage_nl, + mock_asset_metadata, + mock_asset_reconstruction_output_3dtiles_lod12_nl, + mock_asset_reconstruction_output_3dtiles_lod13_nl, + mock_asset_reconstruction_output_3dtiles_lod22_nl, + mock_asset_reconstruction_output_multitiles_nl, +): + """Can we deploy and release the 3DBAG, everything included?""" + + resources = { + "version": VersionResource("test_version"), + "godzilla_server": godzilla_server, + "podzilla_server": podzilla_server, + "db_connection": database, + } + + all_deploy_assets = load_assets_from_package_module( + deploy, key_prefix="deploy", group_name="deploy" + ) + + all_release_assets = load_assets_from_package_module( + release, key_prefix="release", group_name="release" + ) + + defs = Definitions( + resources=resources, + assets=[ + mock_asset_compressed_tiles, + mock_asset_compressed_tiles_validation, + mock_asset_export_index, + mock_asset_geopackage_nl, + mock_asset_metadata, + mock_asset_reconstruction_output_3dtiles_lod12_nl, + mock_asset_reconstruction_output_3dtiles_lod13_nl, + mock_asset_reconstruction_output_3dtiles_lod22_nl, + mock_asset_reconstruction_output_multitiles_nl, + *all_deploy_assets, + *all_release_assets, + ], + jobs=[job_nl_deploy, job_nl_release], + ) + + with DagsterInstance.ephemeral() as instance: + resolved_job = defs.get_job_def("nl_deploy") + result = resolved_job.execute_in_process(instance=instance, resources=resources) + + assert isinstance(result, ExecuteInProcessResult) + assert result.success + + resolved_job = defs.get_job_def("nl_release") + result = resolved_job.execute_in_process( + instance=instance, + resources=resources, + ) + + assert isinstance(result, ExecuteInProcessResult) + assert result.success From 0ddfc7762a11403c8a336a14c3ab89039cc4d5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Mon, 4 Aug 2025 21:38:54 +0200 Subject: [PATCH 07/13] Fix file transfer and release --- .../src/bag3d/core/assets/deploy/servers.py | 19 ++++++++++--------- .../src/bag3d/core/assets/release/publish.py | 19 ++++++++++++------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/deploy/servers.py b/packages/core/src/bag3d/core/assets/deploy/servers.py index ff7fd390..4febbb96 100644 --- a/packages/core/src/bag3d/core/assets/deploy/servers.py +++ b/packages/core/src/bag3d/core/assets/deploy/servers.py @@ -16,30 +16,31 @@ @asset( - ins={"reconstruction_output_multitiles_nl": AssetIn(key_prefix="export")}, + ins={"metadata": AssetIn(key_prefix="export")}, deps=[ AssetKey(("export", "geopackage_nl")), AssetKey(("export", "export_index")), - AssetKey(("export", "metadata")), AssetKey(("export", "compressed_tiles")), AssetKey(("export", "compressed_tiles_validation")), + AssetKey(("export", "reconstruction_output_multitiles_nl")), AssetKey(("export", "reconstruction_output_3dtiles_lod12_nl")), AssetKey(("export", "reconstruction_output_3dtiles_lod13_nl")), AssetKey(("export", "reconstruction_output_3dtiles_lod22_nl")), ], required_resource_keys={"version"}, ) -def compressed_export_nl(context, reconstruction_output_multitiles_nl): +def compressed_export_nl(context, metadata): """Create a compressed tar.gz archive containing the complete 3D BAG export. The archive will be named `export_.tar.gz`. + Args: context: Dagster execution context - reconstruction_output_multitiles_nl: Path to the exported data directory + metadata: Path to the 3DBAG metadata file Returns: Output: Path to the created export_{version}.tar.gz file with size metadata """ - export_dir = reconstruction_output_multitiles_nl + export_dir = metadata.parent version = context.resources.version.version output_tarfile = export_dir.parent / f"export_{version}.tar.gz" with tarfile.open(output_tarfile, "w:gz") as tar: @@ -56,7 +57,7 @@ def transfer_to_server( compressed_export_nl: Path, metadata: Path, target_dir: str, -) -> str: +) -> tuple[Path, Path]: """Transfer and extract export file to a remote server. Args: @@ -66,7 +67,7 @@ def transfer_to_server( target_dir: Base directory on remote server for deployment Returns: - str: Path to the deployment directory on the remote server + (Path to the deployment directory on the remote server, Path to the compressed export on the remote server) Raises: AssertionError: If SSH commands fail during transfer or extraction @@ -76,7 +77,7 @@ def transfer_to_server( with metadata.open("r") as fo: metadata_json = json.load(fo) version = metadata_json["identificationInfo"]["citation"]["edition"] - deploy_dir = f"{target_dir}/{version}" + deploy_dir = Path(target_dir) / version compressed_file = Path(target_dir) / compressed_export_nl.name try: @@ -106,7 +107,7 @@ def transfer_to_server( except Exception as e: logger.error(f"SSH connection failed: {e}") raise - return deploy_dir + return deploy_dir, compressed_file @asset( diff --git a/packages/core/src/bag3d/core/assets/release/publish.py b/packages/core/src/bag3d/core/assets/release/publish.py index 609cada3..cacc282e 100644 --- a/packages/core/src/bag3d/core/assets/release/publish.py +++ b/packages/core/src/bag3d/core/assets/release/publish.py @@ -13,28 +13,25 @@ @asset( - deps={AssetKey(("deploy", "transfer_to_godzilla"))}, ins={ "metadata": AssetIn(key_prefix="export"), - "compressed_export_nl": AssetIn(key_prefix="deploy"), + "transfer_to_godzilla": AssetIn(key_prefix="deploy"), }, required_resource_keys={"godzilla_server"}, ) def publish_data( context, - compressed_export_nl: Path, + transfer_to_godzilla: tuple[Path, Path], metadata: Path, ): """On godzilla, create symlink to the 'export' to the current version and add the current version to the tar.gz archive. """ - data_dir: str = context.resources.godzilla_server.target_dir - public_dir: str = context.resources.godzilla_server.target_dir + public_dir: str = context.resources.godzilla_server.public_dir + deploy_dir, compressed_file = transfer_to_godzilla with metadata.open("r") as fo: metadata_json = json.load(fo) version = metadata_json["identificationInfo"]["citation"]["edition"] - deploy_dir = f"{data_dir}/{version}" - compressed_file = Path(data_dir) / compressed_export_nl.name try: with context.resources.godzilla_server.connection as c: @@ -56,6 +53,14 @@ def publish_data( result = c.run(f"ln -s {deploy_dir} {public_dir}/{version_nopoints}") assert result.ok, "Creating symlink failed" + logger.debug( + f"Setting published version {version_nopoints} to latest version" + ) + result = c.run(f"rm -f {public_dir}/latest") + assert result.ok, "Removing public/latest symlink failed" + result = c.run(f"ln -s {public_dir}/{version_nopoints} {public_dir}/latest") + assert result.ok, "Setting latest version failed" + logger.debug(f"Removing compressed file {compressed_file}") result = c.run(f"rm {compressed_file}") assert result.ok, "Removing compressed file failed" From 576e18a01253d6203db1d16be5e50574cc127179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Wed, 6 Aug 2025 11:12:56 +0200 Subject: [PATCH 08/13] Create test_deploy make target --- makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/makefile b/makefile index e1b305f8..bc53cab8 100644 --- a/makefile +++ b/makefile @@ -86,11 +86,17 @@ test_slow: exit $$FAILED test_integration: + @set -e; \ + FAILED=0; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/test_integration.py -v -s --run-all || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/test_integration.py -v -s --run-all || FAILED=1; \ + docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/test_integration.py -v -s --run-all || FAILED=1; \ + exit $$FAILED + +test_deploy: @set -e; \ FAILED=0; \ docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-core pytest /opt/3dbag-pipeline/packages/core/tests/test_integration.py -v -s --run-all --run-deploy -k 'test_integration_deploy_release' || FAILED=1; \ -# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-party-walls pytest /opt/3dbag-pipeline/packages/party_walls/tests/test_integration.py -v -s --run-all || FAILED=1; \ -# docker compose -p $(COMPOSE_PROJECT_NAME) exec bag3d-floors-estimation pytest /opt/3dbag-pipeline/packages/floors_estimation/tests/test_integration.py -v -s --run-all || FAILED=1; \ exit $$FAILED test_all: From 2c364d22690a115c2b634bf0331682d7272b113d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Wed, 6 Aug 2025 11:40:43 +0200 Subject: [PATCH 09/13] Update test data with files for deploy release integration test --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index bc53cab8..65c9ae7d 100644 --- a/makefile +++ b/makefile @@ -113,7 +113,7 @@ include .env download: rm -rf $(BAG3D_TEST_DATA) mkdir -p $(BAG3D_TEST_DATA) - cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v11.zip ; unzip -q test_data_v11.zip ; rm test_data_v11.zip + cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v12.zip ; unzip -q test_data_v12.zip ; rm test_data_v12.zip install_uv: From 92f353f71bf2567ba1e46775ad9ee5e0f76ac49b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Wed, 6 Aug 2025 11:41:52 +0200 Subject: [PATCH 10/13] Enable assets These will probably still fail in the integration test --- packages/core/src/bag3d/core/jobs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/core/src/bag3d/core/jobs.py b/packages/core/src/bag3d/core/jobs.py index 0779b23f..28bad0dc 100644 --- a/packages/core/src/bag3d/core/jobs.py +++ b/packages/core/src/bag3d/core/jobs.py @@ -157,15 +157,15 @@ name="nl_deploy", description="Deploy the Netherland data.", selection=AssetSelection.assets(["deploy", "compressed_export_nl"]) - | AssetSelection.assets(["deploy", "transfer_to_godzilla"]), - # | AssetSelection.assets(["deploy", "transfer_to_podzilla"]) - # | AssetSelection.assets(["deploy", "webservice_godzilla"]), + | AssetSelection.assets(["deploy", "transfer_to_godzilla"]) + | AssetSelection.assets(["deploy", "transfer_to_podzilla"]) + | AssetSelection.assets(["deploy", "webservice_godzilla"]), ) job_nl_release = define_asset_job( name="nl_release", description="Perform the final steps for the 3DBAG release.", - selection=AssetSelection.assets(["release", "publish_data"]), - # | AssetSelection.assets(["release", "publish_webservices"]), + selection=AssetSelection.assets(["release", "publish_data"]) + | AssetSelection.assets(["release", "publish_webservices"]), ) From d958e73d2b6acc4ec42d88b67209e6f19107a8e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 7 Aug 2025 21:28:52 +0200 Subject: [PATCH 11/13] Update tools --- docker/pipeline/bag3d-core.dockerfile | 2 +- docker/pipeline/bag3d-floors-estimation.dockerfile | 2 +- docker/pipeline/bag3d-party-walls.dockerfile | 2 +- docker/tools/Dockerfile | 14 +++++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docker/pipeline/bag3d-core.dockerfile b/docker/pipeline/bag3d-core.dockerfile index 60c7029b..66e33d14 100644 --- a/docker/pipeline/bag3d-core.dockerfile +++ b/docker/pipeline/bag3d-core.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2025.08.04 AS develop +FROM 3dgi/3dbag-pipeline-tools:2025.08.07 AS develop ARG VERSION=develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline diff --git a/docker/pipeline/bag3d-floors-estimation.dockerfile b/docker/pipeline/bag3d-floors-estimation.dockerfile index 38aef290..05d6a424 100644 --- a/docker/pipeline/bag3d-floors-estimation.dockerfile +++ b/docker/pipeline/bag3d-floors-estimation.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2025.08.04 AS develop +FROM 3dgi/3dbag-pipeline-tools:2025.08.07 AS develop ARG VERSION=develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline diff --git a/docker/pipeline/bag3d-party-walls.dockerfile b/docker/pipeline/bag3d-party-walls.dockerfile index 9dd28e6c..65f0cc33 100644 --- a/docker/pipeline/bag3d-party-walls.dockerfile +++ b/docker/pipeline/bag3d-party-walls.dockerfile @@ -1,4 +1,4 @@ -FROM 3dgi/3dbag-pipeline-tools:2025.08.04 AS develop +FROM 3dgi/3dbag-pipeline-tools:2025.08.07 AS develop ARG VERSION=develop ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index 1895e49c..09e06273 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -109,13 +109,13 @@ FROM roofer AS geoflow-old ARG JOBS=2 ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline -COPY --from=3dgi/geoflow-bundle-builder:2025.07.30 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof -COPY --from=3dgi/geoflow-bundle-builder:2025.07.30 /export /export -COPY --from=3dgi/geoflow-bundle-builder:2025.07.30 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib -COPY --from=3dgi/geoflow-bundle-builder:2025.07.30 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so -COPY --from=3dgi/geoflow-bundle-builder:2025.07.30 /usr/local/lib/LASlib /export/usr/local/lib/ -COPY --from=3dgi/geoflow-bundle-builder:2025.07.30 /usr/local/lib/gdalplugins /export/usr/local/lib/ -COPY --from=3dgi/geoflow-bundle-builder:2025.07.30 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins +COPY --from=3dgi/geoflow-bundle-builder:2025.08.07 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof +COPY --from=3dgi/geoflow-bundle-builder:2025.08.07 /export /export +COPY --from=3dgi/geoflow-bundle-builder:2025.08.07 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib +COPY --from=3dgi/geoflow-bundle-builder:2025.08.07 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so +COPY --from=3dgi/geoflow-bundle-builder:2025.08.07 /usr/local/lib/LASlib /export/usr/local/lib/ +COPY --from=3dgi/geoflow-bundle-builder:2025.08.07 /usr/local/lib/gdalplugins /export/usr/local/lib/ +COPY --from=3dgi/geoflow-bundle-builder:2025.08.07 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins ENV GF_PLUGIN_FOLDER=$BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins RUN echo /export/usr/local/lib >> /etc/ld.so.conf.d/geoflow.conf; \ From 22960d27944b28e492023cb33b7d8d34dee3aef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 7 Aug 2025 22:44:32 +0200 Subject: [PATCH 12/13] Fix deploy tests --- .../src/bag3d/core/assets/deploy/servers.py | 67 +++++++++---------- packages/core/tests/conftest.py | 4 +- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/packages/core/src/bag3d/core/assets/deploy/servers.py b/packages/core/src/bag3d/core/assets/deploy/servers.py index 4febbb96..5c7d46b4 100644 --- a/packages/core/src/bag3d/core/assets/deploy/servers.py +++ b/packages/core/src/bag3d/core/assets/deploy/servers.py @@ -8,7 +8,7 @@ from bag3d.common.utils.database import load_sql from bag3d.common.types import PostgresTableIdentifier -from bag3d.common.resources import ServerTransferResource +from bag3d.common.resources import ServerTransferResource, DatabaseResource from dagster import get_dagster_logger @@ -112,45 +112,42 @@ def transfer_to_server( @asset( ins={"metadata": AssetIn(key_prefix="export")}, - required_resource_keys={"podzilla_server"}, ) def transfer_to_podzilla( - context, - compressed_export_nl: Path, - metadata: Path, + compressed_export_nl: Path, metadata: Path, podzilla_server: ServerTransferResource ): """Transfer the 3D BAG export to the podzilla server for API access.""" return transfer_to_server( - context.resources.podzilla_server, + podzilla_server, compressed_export_nl, metadata, - context.resources.podzilla_server.target_dir, + podzilla_server.target_dir, ) @asset( ins={"metadata": AssetIn(key_prefix="export")}, - required_resource_keys={"godzilla_server"}, ) def transfer_to_godzilla( - context, - compressed_export_nl: Path, - metadata: Path, + compressed_export_nl: Path, metadata: Path, godzilla_server: ServerTransferResource ): """Transfer the 3D BAG export to the godzilla server for public downloads and webservices.""" return transfer_to_server( - context.resources.godzilla_server, + godzilla_server, compressed_export_nl, metadata, - context.resources.godzilla_server.target_dir, + godzilla_server.target_dir, ) @asset( deps={AssetKey(("transfer_to_godzilla"))}, - required_resource_keys={"db_connection", "godzilla_server"}, ) -def webservice_godzilla(context, transfer_to_godzilla): +def webservice_godzilla( + transfer_to_godzilla, + db_connection: DatabaseResource, + godzilla_server: ServerTransferResource, +): """ Load the layers for WFS, WMS to the database on Godzilla. The layers will be loaded into the schema `webservice_dev` and @@ -159,8 +156,8 @@ def webservice_godzilla(context, transfer_to_godzilla): """ schema = "webservice_dev" sql = f"drop schema if exists {schema} cascade; create schema {schema};" - with context.resources.godzilla_server.connect as c: - context.log.debug(sql) + with godzilla_server.connection as c: + logger.debug(sql) c.run( f"psql --dbname baseregisters --port 5432 --host localhost --user etl -c '{sql}'" ) @@ -186,10 +183,10 @@ def webservice_godzilla(context, transfer_to_godzilla): layer + "_tmp", ] ) - with context.resources.godzilla_server.connect as c: - context.log.debug(cmd) + with godzilla_server.connection as c: + logger.debug(cmd) r = c.run(cmd) - context.log.debug(r.stdout) + logger.debug(r.stdout) pand_table = PostgresTableIdentifier(schema, "pand_tmp") lod12_2d_tmp = PostgresTableIdentifier(schema, "lod12_2d_tmp") @@ -212,9 +209,9 @@ def webservice_godzilla(context, transfer_to_godzilla): "lod22_2d": lod22_2d, }, ) - sql = context.resources.db_connection.connect.print_query(sql) - with context.resources.godzilla_server.connect as c: - context.log.debug(sql) + sql = db_connection.connect.print_query(sql) + with godzilla_server.connection as c: + logger.debug(sql) c.run( f"psql --dbname baseregisters --port 5432 --host localhost --user etl -c '{sql}'" ) @@ -231,15 +228,15 @@ def webservice_godzilla(context, transfer_to_godzilla): "validate_compressed_files": validate_compressed_files, }, ) - sql = context.resources.db_connection.connect.print_query(sql) - with context.resources.godzilla_server.connect as c: - context.log.debug(sql) + sql = db_connection.connect.print_query(sql) + with godzilla_server.connection as c: + logger.debug(sql) c.run( f"psql --dbname baseregisters --port 5432 --host localhost --user etl -c '{sql}'" ) # Load the CSV files into the intermediary tables - with context.resources.godzilla_server.connect as c: + with godzilla_server.connection as c: filepath = f"{deploy_dir}/export_index.csv" copy_cmd = ( "\copy " @@ -248,7 +245,7 @@ def webservice_godzilla(context, transfer_to_godzilla): + filepath + "' DELIMITER ',' CSV HEADER " ) - context.log.debug(f"{copy_cmd}") + logger.debug(f"{copy_cmd}") c.run( rf'psql --dbname baseregisters --port 5432 --host localhost --user etl -c "{copy_cmd}" ' ) @@ -260,7 +257,7 @@ def webservice_godzilla(context, transfer_to_godzilla): + filepath + "' DELIMITER ',' CSV HEADER " ) - context.log.debug(f"{copy_cmd}") + logger.debug(f"{copy_cmd}") c.run( rf'psql --dbname baseregisters --port 5432 --host localhost --user etl -c "{copy_cmd}" ' ) @@ -275,9 +272,9 @@ def webservice_godzilla(context, transfer_to_godzilla): "validate_compressed_files": validate_compressed_files, }, ) - sql = context.resources.db_connection.connect.print_query(sql) - with context.resources.godzilla_server.connect as c: - context.log.debug(sql) + sql = db_connection.connect.print_query(sql) + with godzilla_server.connection as c: + logger.debug(sql) c.run( f"psql --dbname baseregisters --port 5432 --host localhost --user etl -c '{sql}'" ) @@ -285,12 +282,12 @@ def webservice_godzilla(context, transfer_to_godzilla): grant_usage = f"GRANT USAGE ON SCHEMA {schema} TO bag_geoserver;" grant_select = f"GRANT SELECT ON ALL TABLES IN SCHEMA {schema} TO bag_geoserver;" - with context.resources.godzilla_server.connect as c: - context.log.debug(grant_usage) + with godzilla_server.connection as c: + logger.debug(grant_usage) c.run( f"psql --dbname baseregisters --port 5432 --host localhost --user etl -c '{grant_usage}'" ) - context.log.debug(grant_select) + logger.debug(grant_select) c.run( f"psql --dbname baseregisters --port 5432 --host localhost --user etl -c '{grant_select}'" ) diff --git a/packages/core/tests/conftest.py b/packages/core/tests/conftest.py index 6cff79da..a99c07e8 100644 --- a/packages/core/tests/conftest.py +++ b/packages/core/tests/conftest.py @@ -64,8 +64,8 @@ def podzilla_server(): port=2222, user="deploy", password="deploy", - target_dir="/tmp/podzilla", - public_dir="/tmp/podzilla/public", + target_dir="/tmp", + public_dir="/tmp/podzilla_public", ) From 4e0206ac33e0e3c41a2dd8ff764e806d0d0df311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Thu, 7 Aug 2025 22:45:23 +0200 Subject: [PATCH 13/13] Update test data to v13 to include files for integration deploy release --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 65c9ae7d..c4cca2a6 100644 --- a/makefile +++ b/makefile @@ -113,7 +113,7 @@ include .env download: rm -rf $(BAG3D_TEST_DATA) mkdir -p $(BAG3D_TEST_DATA) - cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v12.zip ; unzip -q test_data_v12.zip ; rm test_data_v12.zip + cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v13.zip ; unzip -q test_data_v13.zip ; rm test_data_v13.zip install_uv: