better control of roofer concurrency

Author: Ylannl

docker/.env

@@ -18,9 +18,10 @@ DAGSTER_POSTGRES_DB="dagster"
 BAG3D_CONCURRENCY_JOB_AHN3=10
 BAG3D_CONCURRENCY_JOB_AHN4=10
 BAG3D_CONCURRENCY_JOB_AHN5=10
-BAG3D_CONCURRENCY_JOB_NL_RECONSTRUCT=10
+BAG3D_CONCURRENCY_JOB_NL_RECONSTRUCT=1
 BAG3D_CONCURRENCY_JOB_ARCHIVE=10
 BAG3D_CONCURRENCY_TOOL_TYLER=10
+BAG3D_CONCURRENCY_TOOL_ROOFER=10
 
 # 3dbag-pipeline resource configuration
 DAGSTER_ENVIRONMENT="test"

packages/core/src/bag3d/core/assets/reconstruction/reconstruction.py

@@ -83,6 +83,12 @@ def __init__(self, schema: str, table_tiles: str, wkt: str = None):
             is_required=False,
             default_value="info",
         ),
+        "concurrency": Field(
+            int,
+            description="Roofer --jobs",
+            is_required=False,
+            default_value=10,
+        ),
     },
 )
 def reconstructed_building_models_nl(
@@ -114,7 +120,7 @@ def reconstructed_building_models_nl(
     try:
         return_code, output = context.resources.roofer.app.execute(
             exe_name="roofer",
-            command=f"{{exe}} --config {{local_path}} {output_dir} --loglevel {context.op_config['loglevel']} --skip-pc-check",
+            command=f"{{exe}} --config {{local_path}} {output_dir} -j {context.op_config['concurrency']} --loglevel {context.op_config['loglevel']} --skip-pc-check",
             local_path=roofer_toml,
             silent=False,
         )

packages/core/src/bag3d/core/jobs.py

@@ -84,6 +84,15 @@
     selection=AssetSelection.assets(
         ["reconstruction", "reconstructed_building_models_nl"]
     ),
+    config={
+        "ops": {
+            "reconstructed_building_models_nl": {
+                "config": {
+                    "concurrency": int(getenv("BAG3D_CONCURRENCY_TOOL_ROOFER", 1))
+                }
+            }
+        }
+    },
 )
 
 job_nl_reconstruct_debug = define_asset_job(
@@ -95,7 +104,11 @@
     config={
         "ops": {
             "reconstructed_building_models_nl": {
-                "config": {"drop_views": False, "loglevel": "debug"}
+                "config": {
+                    "drop_views": False,
+                    "loglevel": "debug",
+                    "concurrency": int(getenv("BAG3D_CONCURRENCY_TOOL_ROOFER", 1)),
+                }
             }
         }
     },