|
9 | 9 |
|
10 | 10 | from bag3d.common.utils.database import load_sql |
11 | 11 | from bag3d.common.types import PostgresTableIdentifier |
| 12 | +from dagster import get_dagster_logger |
| 13 | + |
| 14 | +logger = get_dagster_logger("deploy") |
12 | 15 |
|
13 | 16 |
|
14 | 17 | @asset( |
|
20 | 23 | AssetKey(("export", "compressed_tiles")), |
21 | 24 | AssetKey(("export", "compressed_tiles_validation")), |
22 | 25 | ], |
| 26 | + required_resource_keys={"version"}, |
23 | 27 | ) |
24 | 28 | def compressed_export_nl(context, reconstruction_output_multitiles_nl): |
25 | 29 | """A .tar.gz compressed full directory tree of the exports""" |
26 | 30 | export_dir = reconstruction_output_multitiles_nl |
27 | | - output_tarfile = export_dir.parent / "export.tar.gz" |
| 31 | + version = context.resources.version.version |
| 32 | + output_tarfile = export_dir.parent / f"export_{version}.tar.gz" |
28 | 33 | with tarfile.open(output_tarfile, "w:gz") as tar: |
29 | | - tar.add(export_dir, arcname="export") |
| 34 | + tar.add(export_dir, arcname=f"export_{version}") |
30 | 35 | metadata_output = { |
31 | 36 | "size [Gb]": output_tarfile.stat().st_size * 1e-9, |
32 | 37 | "path": str(output_tarfile), |
33 | 38 | } |
34 | 39 | return Output(output_tarfile, metadata=metadata_output) |
35 | 40 |
|
36 | 41 |
|
37 | | -@asset(ins={"metadata": AssetIn(key_prefix="export")}) |
38 | | -def downloadable_godzilla(context, compressed_export_nl: Path, metadata: Path): |
| 42 | +@asset( |
| 43 | + ins={"metadata": AssetIn(key_prefix="export")}, required_resource_keys={"version"} |
| 44 | +) |
| 45 | +def downloadable_godzilla( |
| 46 | + context, |
| 47 | + compressed_export_nl: Path, |
| 48 | + metadata: Path, |
| 49 | + data_dir: str = "/data/3DBAG", |
| 50 | + public_dir: str = "/data/3DBAG/public", |
| 51 | +): |
39 | 52 | """Downloadable files hosted on godzilla. |
40 | | - - Transfer the export.tar.gz archive to `godzilla:/data/3DBAG` |
| 53 | + - Transfer the export_<version>.tar.gz archive to `godzilla:/data/3DBAG` |
41 | 54 | - Uncompress the archive and add the current version to the directory name |
42 | 55 | - Symlink to the 'export' to the current version |
43 | 56 | - Add the current version to the tar.gz archive |
44 | 57 | """ |
45 | | - data_dir = "/data/3DBAG" |
46 | 58 | with metadata.open("r") as fo: |
47 | 59 | metadata_json = json.load(fo) |
48 | 60 | version = metadata_json["identificationInfo"]["citation"]["edition"] |
49 | 61 | deploy_dir = f"{data_dir}/{version}" |
50 | | - with Connection(host="godzilla.bk.tudelft.nl", user="dagster") as c: |
51 | | - c.put(compressed_export_nl, remote=data_dir) |
52 | | - # delete symlink here, because the uncompressed tar archive is also 'export', |
53 | | - # so we have a bit of downtime here, but that's ok |
54 | | - c.run(f"mkdir {deploy_dir}") |
55 | | - c.run( |
56 | | - f"tar --strip-components=1 -C {deploy_dir} -xzvf {data_dir}/export.tar.gz" |
57 | | - ) |
58 | | - # symlink to latest version so the fileserver picks up the data |
59 | | - version_nopoints = version.replace(".", "") |
60 | | - c.run(f"ln -s {deploy_dir} {data_dir}/public/{version_nopoints}") |
61 | | - # add version to the tar so that we can archive the data |
62 | | - # c.run(f"mv {data_dir}/export.tar.gz {data_dir}/export_{version}.tar.gz") |
63 | | - # remove archive |
64 | | - c.run(f"rm {data_dir}/export.tar.gz") |
| 62 | + compressed_file = Path(data_dir) / compressed_export_nl.name |
| 63 | + |
| 64 | + try: |
| 65 | + with Connection(host="godzilla.bk.tudelft.nl", user="dagster") as c: |
| 66 | + # test connection |
| 67 | + result = c.run("echo connected", hide=True) |
| 68 | + assert result.ok, "Connection command failed" |
| 69 | + logger.debug("SSH connection successful") |
| 70 | + |
| 71 | + logger.debug(f"Transferring {compressed_export_nl} to {data_dir}") |
| 72 | + result = c.put(compressed_export_nl, remote=data_dir) |
| 73 | + logger.debug(f"Transferred: {result}") |
| 74 | + |
| 75 | + logger.debug(f"Creating deploy_dir {deploy_dir}") |
| 76 | + result = c.run(f"mkdir -p {deploy_dir}") |
| 77 | + assert result.ok, "Creating deploy_dir failed" |
| 78 | + |
| 79 | + logger.debug(f"Decompressing {compressed_file} to {deploy_dir}") |
| 80 | + result = c.run( |
| 81 | + f"tar --strip-components=1 -C {deploy_dir} -xzvf {compressed_file}" |
| 82 | + ) |
| 83 | + assert result.ok, "Decompressing failed" |
| 84 | + |
| 85 | + # symlink to latest version so the fileserver picks up the data |
| 86 | + version_nopoints = version.replace(".", "") |
| 87 | + |
| 88 | + logger.debug(f"Creating public_dir {public_dir}") |
| 89 | + result = c.run(f"mkdir -p {public_dir}") |
| 90 | + assert result.ok, "Creating public_dir failed" |
| 91 | + |
| 92 | + logger.debug( |
| 93 | + f"Creating symlink to {deploy_dir} as {public_dir}/{version_nopoints}" |
| 94 | + ) |
| 95 | + result = c.run(f"ln -s {deploy_dir} {data_dir}/public/{version_nopoints}") |
| 96 | + assert result.ok, "Creating symlink failed" |
| 97 | + |
| 98 | + logger.debug(f"Removing compressed file {compressed_file}") |
| 99 | + result = c.run(f"rm {compressed_file}") |
| 100 | + assert result.ok, "Removing compressed file failed" |
| 101 | + |
| 102 | + logger.info( |
| 103 | + f"Deployment successful: Files transferred to {public_dir}/{version_nopoints} on godzilla" |
| 104 | + ) |
| 105 | + except Exception as e: |
| 106 | + logger.error(f"SSH connection failed: {e}") |
| 107 | + raise |
65 | 108 | return deploy_dir |
66 | 109 |
|
67 | 110 |
|
|
0 commit comments