diff --git a/packages/core/src/bag3d/core/assets/ahn/download.py b/packages/core/src/bag3d/core/assets/ahn/download.py index 8a41fee8..5cad21c1 100644 --- a/packages/core/src/bag3d/core/assets/ahn/download.py +++ b/packages/core/src/bag3d/core/assets/ahn/download.py @@ -187,7 +187,11 @@ def laz_files_ahn3(context, config: LazFilesConfig, md5_ahn3, tile_index_ahn): # Let's try to re-download the file once if not first_validation: - logger.info(format_laz_log(fpath, "Removing")) + logger.info( + format_laz_log( + fpath, "First validation failed. Removing and retrying..." + ) + ) fpath.unlink() lazdownload = download_ahn_laz( fpath=fpath, url_laz=url_laz, verify_ssl=verify_ssl @@ -198,7 +202,7 @@ def laz_files_ahn3(context, config: LazFilesConfig, md5_ahn3, tile_index_ahn): if not second_validation: logger.warning(format_laz_log(fpath, "Checksum failed")) else: - logger.debug(format_laz_log(fpath, "OK")) + logger.debug(format_laz_log(fpath, "Validation OK")) return Output(lazdownload, metadata=lazdownload.asdict()) @@ -237,7 +241,11 @@ def laz_files_ahn4(context, config: LazFilesConfig, md5_ahn4, tile_index_ahn): # Let's try to re-download the file once if not first_validation: - logger.info(format_laz_log(fpath, "Removing")) + logger.info( + format_laz_log( + fpath, "First validation failed. Removing and retrying..." + ) + ) fpath.unlink() lazdownload = download_ahn_laz( fpath=fpath, @@ -250,7 +258,7 @@ def laz_files_ahn4(context, config: LazFilesConfig, md5_ahn4, tile_index_ahn): if not second_validation: logger.warning(format_laz_log(fpath, "Checksum failed")) else: - logger.debug(format_laz_log(fpath, "OK")) + logger.debug(format_laz_log(fpath, "Validation OK")) return Output(lazdownload, metadata=lazdownload.asdict()) @@ -286,7 +294,11 @@ def laz_files_ahn5(context, config: LazFilesConfig, sha256_ahn5, tile_index_ahn) ) # Let's try to re-download the file once if not first_validation: - logger.info(format_laz_log(fpath, "Removing")) + logger.info( + format_laz_log( + fpath, "First validation failed. Removing and retrying..." + ) + ) fpath.unlink() lazdownload = download_ahn_laz( fpath=fpath, @@ -299,7 +311,7 @@ def laz_files_ahn5(context, config: LazFilesConfig, sha256_ahn5, tile_index_ahn) if not second_validation: logger.warning(format_laz_log(fpath, "Checksum failed")) else: - logger.debug(format_laz_log(fpath, "OK")) + logger.debug(format_laz_log(fpath, "Validation OK")) return Output(lazdownload, metadata=lazdownload.asdict()) @@ -370,6 +382,9 @@ def download_ahn_laz( ) else: # pragma: no cover logger.info(format_laz_log(fpath, "File already downloaded")) + success = True + file_size = round(fpath.stat().st_size / 1e6, 2) + is_new = False if force_download: logger.info(format_laz_log(fpath, "Forcing re-download")) file_size, fpath, is_new, success, url_laz = download_laz( diff --git a/packages/core/src/bag3d/core/assets/ahn/metadata.py b/packages/core/src/bag3d/core/assets/ahn/metadata.py index 9e7767bc..c28ea97b 100644 --- a/packages/core/src/bag3d/core/assets/ahn/metadata.py +++ b/packages/core/src/bag3d/core/assets/ahn/metadata.py @@ -200,6 +200,13 @@ def compute_load_metadata( """ tile_id = context.partition_key conn = context.resources.db_connection.connect + if not laz_files_ahn.success or laz_files_ahn.hash_name is None: + context.log.info( + f"LAZ tile {tile_id} has not been successfully downloaded." + f"Skipping metadata computation..." + ) + return Output(None) + if not laz_files_ahn.new: if not context.op_execution_context.op_execution_context.op_config["force"]: context.log.info(