Ahn5 pdal issue

packages/common/src/bag3d/common/utils/geodata.py

@@ -9,12 +9,15 @@
     TableSchema,
     TableColumnConstraints,
     TableColumn,
+    get_dagster_logger,
 )
 from pgutils import PostgresTableIdentifier
 
 from bag3d.common.utils.database import postgrestable_metadata
 from bag3d.common.resources.executables import AppImage
 
+logger = get_dagster_logger()
+
 
 def wkt_from_bbox(bbox):
     minx, miny, maxx, maxy = bbox
@@ -256,9 +259,26 @@ def pdal_info(
     ]
     cmd_list.append("--all") if with_all else cmd_list.append("--metadata")
     cmd_list.append("{local_path}")
-    return_code, output = pdal.execute(
-        "pdal", command=" ".join(cmd_list), local_path=file_path, silent=(not verbose)
-    )
+    try:
+        return_code, output = pdal.execute(
+            "pdal",
+            command=" ".join(cmd_list),
+            local_path=file_path,
+            silent=(not verbose),
+        )
+    except Exception as e:
+        if "Global encoding WKT flag" in str(e):
+            logger.warning(f"Pdal failed for tile {file_path} with error {e}.")
+            logger.warning("Setting --readers.las.nosrs true")
+            cmd_list.append("--readers.las.nosrs true")
+            return_code, output = pdal.execute(
+                "pdal",
+                command=" ".join(cmd_list),
+                local_path=file_path,
+                silent=(not verbose),
+            )
+        else:
+            raise
     output_processed = output.replace("\\u0000", "")
     return return_code, json.loads(output_processed)
 

packages/core/src/bag3d/core/assets/ahn/download.py

@@ -187,7 +187,11 @@ def laz_files_ahn3(context, config: LazFilesConfig, md5_ahn3, tile_index_ahn):
 
         # Let's try to re-download the file once
         if not first_validation:
-            logger.info(format_laz_log(fpath, "Removing"))
+            logger.info(
+                format_laz_log(
+                    fpath, "First validation failed. Removing and retrying..."
+                )
+            )
             fpath.unlink()
             lazdownload = download_ahn_laz(
                 fpath=fpath, url_laz=url_laz, verify_ssl=verify_ssl
@@ -198,7 +202,7 @@ def laz_files_ahn3(context, config: LazFilesConfig, md5_ahn3, tile_index_ahn):
             if not second_validation:
                 logger.warning(format_laz_log(fpath, "Checksum failed"))
         else:
-            logger.debug(format_laz_log(fpath, "OK"))
+            logger.debug(format_laz_log(fpath, "Validation OK"))
 
     return Output(lazdownload, metadata=lazdownload.asdict())
 
@@ -237,7 +241,11 @@ def laz_files_ahn4(context, config: LazFilesConfig, md5_ahn4, tile_index_ahn):
 
         # Let's try to re-download the file once
         if not first_validation:
-            logger.info(format_laz_log(fpath, "Removing"))
+            logger.info(
+                format_laz_log(
+                    fpath, "First validation failed. Removing and retrying..."
+                )
+            )
             fpath.unlink()
             lazdownload = download_ahn_laz(
                 fpath=fpath,
@@ -250,7 +258,7 @@ def laz_files_ahn4(context, config: LazFilesConfig, md5_ahn4, tile_index_ahn):
             if not second_validation:
                 logger.warning(format_laz_log(fpath, "Checksum failed"))
         else:
-            logger.debug(format_laz_log(fpath, "OK"))
+            logger.debug(format_laz_log(fpath, "Validation OK"))
 
     return Output(lazdownload, metadata=lazdownload.asdict())
 
@@ -286,7 +294,11 @@ def laz_files_ahn5(context, config: LazFilesConfig, sha256_ahn5, tile_index_ahn)
         )
         # Let's try to re-download the file once
         if not first_validation:
-            logger.info(format_laz_log(fpath, "Removing"))
+            logger.info(
+                format_laz_log(
+                    fpath, "First validation failed. Removing and retrying..."
+                )
+            )
             fpath.unlink()
             lazdownload = download_ahn_laz(
                 fpath=fpath,
@@ -299,7 +311,7 @@ def laz_files_ahn5(context, config: LazFilesConfig, sha256_ahn5, tile_index_ahn)
             if not second_validation:
                 logger.warning(format_laz_log(fpath, "Checksum failed"))
         else:
-            logger.debug(format_laz_log(fpath, "OK"))
+            logger.debug(format_laz_log(fpath, "Validation OK"))
 
     return Output(lazdownload, metadata=lazdownload.asdict())
 
@@ -370,14 +382,14 @@ def download_ahn_laz(
         )
     else:  # pragma: no cover
         logger.info(format_laz_log(fpath, "File already downloaded"))
+        success = True
+        file_size = round(fpath.stat().st_size / 1e6, 2)
+        is_new = False
         if force_download:
             logger.info(format_laz_log(fpath, "Forcing re-download"))
             file_size, fpath, is_new, success, url_laz = download_laz(
                 file_size, fpath, is_new, nr_retries, success, url, url_laz, verify_ssl
             )
-        success = True
-        file_size = round(fpath.stat().st_size / 1e6, 2)
-        is_new = False
 
     if not success:
         raise Failure(format_laz_log(fpath, "Downloading failed!"))