Merge pull request #107 from 3DBAG/59-resources-should-initialize-without-values

59 resources should initialize without values

Author: GinaStavropoulou

docker/.env

@@ -1,3 +1,4 @@
+DAGSTER_ENVIRONMENT="test"
 DAGSTER_HOME=/opt/dagster/dagster_home
 DAGSTER_POSTGRES_USER="postgres_user"
 DAGSTER_POSTGRES_PASSWORD="postgres_password"
@@ -10,7 +11,6 @@ BAG3D_FILESTORE=/data/volume
 BAG3D_RELEASE_VERSION=develop
 BAG3D_TEST_DATA=/data/volume
 BAG3D_FLOORS_ESTIMATION_MODEL=/data/volume/model/pipeline_model1_gbr_untuned.joblib
-BAG3D_EXPORT_DIR=/data/volume/reconstruction_input/3DBAG/export_test_version
 
 BAG3D_PG_DOCKERFILE=/3dbag-pipeline/docker/postgres/Dockerfile
 BAG3D_PG_DOCKERIMAGE=bag3d_image_postgis

makefile

@@ -5,7 +5,7 @@ include .env
 source:
 	set -a ; . ./.env ; set +a
 
-download: source
+download:
 	rm -rf $(BAG3D_TEST_DATA)
 	mkdir -p $(BAG3D_TEST_DATA)
 	cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v5.zip ; unzip -q test_data_v5.zip ; rm test_data_v5.zip

packages/common/src/bag3d/common/resources/__init__.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 from bag3d.common.resources.executables import (
     GDALResource,
@@ -12,21 +13,21 @@
 from bag3d.common.resources.database import DatabaseResource
 from bag3d.common.resources.version import VersionResource
 
-from dagster import EnvVar
+from dagster import EnvVar, get_dagster_logger
 
-# The 'mount_point' is the directory in the container that is bind-mounted on the host
+logger = get_dagster_logger()
 
 version = VersionResource(os.getenv("BAG3D_RELEASE_VERSION"))
 
 
-gdal_local = GDALResource(
+gdal = GDALResource(
     exe_ogr2ogr=os.getenv("EXE_PATH_OGR2OGR"),
     exe_ogrinfo=os.getenv("EXE_PATH_OGRINFO"),
     exe_sozip=os.getenv("EXE_PATH_SOZIP"),
 )
 
 
-pdal_local = PDALResource(exe_pdal=os.getenv("EXE_PATH_PDAL"))
+pdal = PDALResource(exe_pdal=os.getenv("EXE_PATH_PDAL"))
 
 
 db_connection = DatabaseResource(
@@ -47,6 +48,14 @@
     dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
 )
 
+file_store_test = FileStoreResource(
+    data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input"),
+    dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
+)
+file_store_fastssd_test = FileStoreResource(
+    data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "integration_core"),
+    dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
+)
 
 # Configure for gilfoyle
 file_store_gilfoyle = FileStoreResource(data_dir="/data")
@@ -74,11 +83,11 @@
 
 
 RESOURCES_LOCAL = {
-    "gdal": gdal_local,
+    "gdal": gdal,
     "file_store": file_store,
     "file_store_fastssd": file_store_fastssd,
     "db_connection": db_connection,
-    "pdal": pdal_local,
+    "pdal": pdal,
     "lastools": lastools,
     "tyler": tyler,
     "geoflow": geoflow,
@@ -87,12 +96,12 @@
 }
 
 
-RESOURCES_PYTEST = {
-    "gdal": gdal_local,
-    "file_store": file_store,
-    "file_store_fastssd": file_store_fastssd,
+RESOURCES_TEST = {
+    "gdal": gdal,
+    "file_store": file_store_test,
+    "file_store_fastssd": file_store_fastssd_test,
     "db_connection": db_connection,
-    "pdal": pdal_local,
+    "pdal": pdal,
     "lastools": lastools,
     "tyler": tyler,
     "geoflow": geoflow,
@@ -101,24 +110,41 @@
 }
 
 RESOURCES_PROD = {
-    "gdal": gdal_local,
+    "gdal": gdal,
     "file_store": file_store_gilfoyle,
     "file_store_fastssd": file_store_gilfoyle_fastssd,
     "db_connection": db_connection,
-    "pdal": pdal_local,
+    "pdal": pdal,
     "lastools": lastools,
     "tyler": tyler,
     "geoflow": geoflow,
     "roofer": roofer,
     "version": version,
 }
 
-# Resource definitions for import
+RESOURCES_DEFAULT = {
+    "gdal": GDALResource(),
+    "file_store": FileStoreResource(),
+    "file_store_fastssd": FileStoreResource(),
+    "db_connection": DatabaseResource(),
+    "pdal": PDALResource(),
+    "lastools": LASToolsResource(),
+    "tyler": TylerResource(),
+    "geoflow": GeoflowResource(),
+    "roofer": RooferResource(),
+    "version": VersionResource(),
+}
+
 
-resource_defs_by_deployment_name = {
+resource_defs_by_env_name = {
     "prod": RESOURCES_PROD,
     "local": RESOURCES_LOCAL,
-    "pytest": RESOURCES_PYTEST,
+    "test": RESOURCES_TEST,
+    "default": RESOURCES_DEFAULT,
 }
-deployment_name = os.environ.get("DAGSTER_DEPLOYMENT", "local")
-resource_defs = resource_defs_by_deployment_name[deployment_name]
+env_name = os.getenv("DAGSTER_ENVIRONMENT", "default").lower()
+if env_name not in resource_defs_by_env_name.keys():
+    logger.warning(f"Invalid environment: {env_name}, setting to default")
+    env_name = "default"
+
+resource_defs = resource_defs_by_env_name[env_name]

packages/common/src/bag3d/common/resources/database.py

@@ -12,29 +12,29 @@ class DatabaseResource(ConfigurableResource):
     Database connection.
     """
 
-    host: str
-    user: str
-    password: str
-    dbname: str
-    port: str
-    other_params: Permissive()
+    host: Optional[str] = None
+    user: Optional[str] = None
+    password: Optional[str] = None
+    dbname: Optional[str] = None
+    port: Optional[str] = None
+    other_params: Optional[Permissive()] = None
 
     def __init__(
         self,
-        host: str,
-        user: str,
-        password: str,
-        dbname: str,
-        port: str,
+        host: Optional[str] = None,
+        user: Optional[str] = None,
+        password: Optional[str] = None,
+        dbname: Optional[str] = None,
+        port: Optional[str] = None,
         other_params: Optional[Permissive()] = None,
     ):
         super().__init__(
-            host=host,
-            user=user,
-            password=password,
-            dbname=dbname,
-            port=port,
-            other_params=other_params or {},
+            host=host or "data-postgresql",
+            user=user or "baseregisters_test_user",
+            password=password or "baseregisters_test_pswd",
+            dbname=dbname or "baseregisters_test",
+            port=port or "5432",
+            other_params=other_params or {"sslmode": "allow"},
         )
         conn = DatabaseConnection(
             user=self.user,

packages/common/src/bag3d/common/resources/executables.py

@@ -334,25 +334,19 @@ class PDALResource(ConfigurableResource):
         pdal_resource.app
     """
 
-    exe_pdal: str
-    docker_cfg: DockerConfig
-
-    def __init__(
-        self,
-        exe_pdal: Optional[str] = None,
-        docker_cfg: Optional[DockerConfig] = None,
-    ):
-        super().__init__(
-            exe_pdal=exe_pdal or "pdal",
-            docker_cfg=docker_cfg
-            or DockerConfig(image=DOCKER_PDAL_IMAGE, mount_point="/tmp"),
-        )
+    exe_pdal: Optional[str] = None
+    docker_cfg: Optional[DockerConfig] = None
 
     @property
     def exes(self) -> Dict[str, str]:
-        return {
-            "pdal": self.exe_pdal,
-        }
+        if self.docker_cfg is None:
+            return {
+                "pdal": self.exe_pdal,
+            }
+        else:
+            return {
+                "pdal": "pdal",
+            }
 
     @property
     def with_docker(self) -> bool:
@@ -384,8 +378,8 @@ class LASToolsResource(ConfigurableResource):
         lastools_resource.app
     """
 
-    exe_lasindex: str
-    exe_las2las: str
+    exe_lasindex: Optional[str] = None
+    exe_las2las: Optional[str] = None
 
     @property
     def exes(self) -> Dict[str, str]:
@@ -416,8 +410,8 @@ class TylerResource(ConfigurableResource):
         tyler = tyler_resource.app
     """
 
-    exe_tyler: str
-    exe_tyler_db: str
+    exe_tyler: Optional[str] = None
+    exe_tyler_db: Optional[str] = None
 
     @property
     def exes(self) -> Dict[str, str]:
@@ -448,8 +442,8 @@ class RooferResource(ConfigurableResource):
         roofer = roofer_resource.app
     """
 
-    exe_crop: str
-    exe_roofer: str
+    exe_crop: Optional[str] = None
+    exe_roofer: Optional[str] = None
 
     @property
     def exes(self) -> Dict[str, str]:
@@ -484,8 +478,8 @@ class GeoflowResource(ConfigurableResource):
         geoflow = geoflow_resource.app
     """
 
-    exe_geoflow: str
-    flowchart: str
+    exe_geoflow: Optional[str] = None
+    flowchart: Optional[str] = None
 
     @property
     def exes(self) -> Dict[str, str]:

packages/common/src/bag3d/common/resources/files.py

@@ -102,9 +102,9 @@ class FileStoreResource(ConfigurableResource):
     TODO: make the directory functions in .core (bag3d_export_dir etc) members of this
     """
 
-    data_dir: str
-    docker_volume_id: str
-    dir_id: str
+    data_dir: Optional[str] = None
+    docker_volume_id: Optional[str] = None
+    dir_id: Optional[str] = None
 
     def __init__(
         self,
@@ -113,20 +113,20 @@ def __init__(
         dir_id: Optional[str] = None,
     ):
         super().__init__(
-            data_dir=str(data_dir) if data_dir else "",
-            docker_volume_id=docker_volume_id or "",
-            dir_id=dir_id or "",
+            data_dir=str(data_dir) if data_dir else None,
+            docker_volume_id=docker_volume_id,
+            dir_id=dir_id,
         )
 
     @property
     def file_store(self) -> FileStore:
-        if self.data_dir != "" and self.dir_id != "":
+        if self.data_dir and self.dir_id:
             return FileStore(data_dir=self.data_dir, dir_id=self.dir_id)
-        elif self.data_dir != "":
+        elif self.data_dir:
             return FileStore(data_dir=self.data_dir)
-        elif self.docker_volume_id != "":
+        elif self.docker_volume_id:
             return FileStore(docker_volume_id=self.docker_volume_id)
-        elif self.dir_id != "":
+        elif self.dir_id:
             return FileStore(dir_id=self.dir_id)
         else:
             return FileStore()

packages/common/src/bag3d/common/resources/version.py

@@ -10,7 +10,7 @@ class VersionResource(ConfigurableResource):
     A resource for setting up the version release.
     """
 
-    version: str
+    version: Optional[str] = None
 
     def __init__(
         self,

packages/common/src/bag3d/common/utils/files.py

@@ -55,7 +55,7 @@ def geoflow_crop_dir(root_dir: os.PathLike) -> Path:
 def bag3d_export_dir(root_dir: os.PathLike, version: str) -> Path:
     """Create the 3DBAG export directory if does not exist"""
     export_dir = bag3d_dir(root_dir) / f"export_{version}"
-    export_dir.mkdir(exist_ok=True)
+    export_dir.mkdir(exist_ok=True, parents=True)
     return export_dir
 
 
@@ -98,19 +98,24 @@ def get_export_tile_ids() -> Sequence[str]:
         List of tile IDs
     """
     tileids = []
-    root_dir = Path(os.getenv("BAG3D_FILESTORE", "/data"))
-    version = os.getenv("BAG3D_RELEASE_VERSION", "test_version")
+
+    env = os.getenv("DAGSTER_ENVIRONMENT", "test")
+    if env == "test":
+        root_dir = Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input"
+        version = "test_version"
+    else:
+        root_dir = Path(os.getenv("BAG3D_FILESTORE", "/data"))
+        version = os.getenv("BAG3D_RELEASE_VERSION", "test_version")
+
     export_dir = bag3d_export_dir(root_dir=root_dir, version=version)
-    if export_dir.exists():
-        path_tiles_dir = export_dir.joinpath("tiles")
-        path_quadtree_tsv = export_dir.joinpath("quadtree.tsv")
-        if path_quadtree_tsv.exists():
-            tileids = [
-                er.tile_id
-                for er in check_export_results(path_quadtree_tsv, path_tiles_dir)
-            ]
+
+    path_tiles_dir = export_dir.joinpath("tiles")
+    path_quadtree_tsv = export_dir.joinpath("quadtree.tsv")
+    if path_quadtree_tsv.exists():
+        tileids = [
+            er.tile_id for er in check_export_results(path_quadtree_tsv, path_tiles_dir)
+        ]
     else:
-        raise FileNotFoundError(f"""Export directory {export_dir} does not exist.
-                                You need to set the BAG3D_EXPORT_DIR variable.""")
+        raise FileNotFoundError(f"File not found: {path_quadtree_tsv}")
 
     return tileids