Skip to content

Commit 2de97f4

Browse files
GinaStavropoulouGinaStavropoulou
authored
Merge pull request #113 from 3DBAG/101-test-data-fixing-for-reconstruction-input
101 test data fixing for reconstruction input
2 parents 7e057bc + 8dc51e4 commit 2de97f4

File tree

19 files changed

+62
-92
lines changed

19 files changed

+62
-92
lines changed

‎.github/workflows/lint-and-formatting.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@ jobs:
1616
- uses: actions/setup-python@v5
1717
with:
1818
python-version: "3.11"
19+
- name: Intall uv
20+
run: curl -LsSf https://astral.sh/uv/install.sh | sh
1921
- name: Install dependencies
20-
run: pip install ruff
22+
run: uv pip install ruff --system
2123
- name: Lint
2224
run: ruff check
2325
- name: Format

‎docker/pipeline/bag3d-core.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM 3dgi/3dbag-pipeline-tools:2024.12.03 AS develop
1+
FROM 3dgi/3dbag-pipeline-tools:2024.12.05 AS develop
22
ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
33

44
LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"

‎docker/pipeline/bag3d-floors-estimation.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM 3dgi/3dbag-pipeline-tools:2024.12.03 AS develop
1+
FROM 3dgi/3dbag-pipeline-tools:2024.12.05 AS develop
22
ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
33

44
LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"

‎docker/pipeline/bag3d-party-walls.dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM 3dgi/3dbag-pipeline-tools:2024.12.03 AS develop
1+
FROM 3dgi/3dbag-pipeline-tools:2024.12.05 AS develop
22
ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
33

44
LABEL org.opencontainers.image.authors="Balázs Dukai <[email protected]>"

‎docker/tools/Dockerfile‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -100,13 +100,13 @@ FROM roofer AS geoflow-old
100100
ARG JOBS=2
101101
ARG BAG3D_PIPELINE_LOCATION=/opt/3dbag-pipeline
102102

103-
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.03 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof
104-
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.03 /export /export
105-
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.03 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib
106-
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.03 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so
107-
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.03 /usr/local/lib/LASlib /export/usr/local/lib/
108-
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.03 /usr/local/lib/gdalplugins /export/usr/local/lib/
109-
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.03 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins
103+
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/bin/geof $BAG3D_PIPELINE_LOCATION/tools/bin/geof
104+
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /export /export
105+
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/src/FileGDB/lib/. /usr/local/src/FileGDB/lib
106+
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/libfgdbunixrtl.so /usr/local/lib/libfgdbunixrtl.so
107+
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/LASlib /export/usr/local/lib/
108+
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/gdalplugins /export/usr/local/lib/
109+
COPY --link --from=3dgi/geoflow-bundle-builder:2024.12.04 /usr/local/lib/geoflow-plugins/. $BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins
110110
ENV GF_PLUGIN_FOLDER=$BAG3D_PIPELINE_LOCATION/tools/share/geoflow-bundle/plugins
111111

112112
RUN echo /export/usr/local/lib >> /etc/ld.so.conf.d/geoflow.conf; \

‎makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ source:
88
download:
99
rm -rf $(BAG3D_TEST_DATA)
1010
mkdir -p $(BAG3D_TEST_DATA)
11-
cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v5.zip ; unzip -q test_data_v5.zip ; rm test_data_v5.zip
11+
cd $(BAG3D_TEST_DATA) ; curl -O https://data.3dbag.nl/testdata/pipeline/test_data_v6.zip ; unzip -q test_data_v6.zip ; rm test_data_v6.zip
1212

1313
docker_volume_create:
1414
docker rm -f bag3d-dev-temp-container > /dev/null 2>&1 || true

‎packages/common/src/bag3d/common/resources/__init__.py‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,21 +40,14 @@
4040
)
4141

4242

43-
file_store = FileStoreResource(
44-
data_dir=os.getenv("BAG3D_FILESTORE"), dir_id=os.getenv("BAG3D_RELEASE_VERSION")
45-
)
46-
file_store_fastssd = FileStoreResource(
47-
data_dir=os.getenv("BAG3D_FILESTORE"),
48-
dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
49-
)
43+
file_store = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE"))
44+
file_store_fastssd = FileStoreResource(data_dir=os.getenv("BAG3D_FILESTORE"))
5045

5146
file_store_test = FileStoreResource(
52-
data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input"),
53-
dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
47+
data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "reconstruction_input")
5448
)
5549
file_store_fastssd_test = FileStoreResource(
56-
data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "integration_core"),
57-
dir_id=os.getenv("BAG3D_RELEASE_VERSION"),
50+
data_dir=str(Path(os.getenv("BAG3D_FILESTORE")) / "integration_core")
5851
)
5952

6053
# Configure for gilfoyle

‎packages/common/src/bag3d/common/resources/files.py‎

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -93,40 +93,23 @@ def mkdir_temp(temp_dir_id: str = None) -> Path:
9393

9494
class FileStoreResource(ConfigurableResource):
9595
"""Location of the data files that are generated in the pipeline.
96-
Either local directory or a docker volume.
97-
If neither `data_dir` nor `docker_volume` is given, a local
98-
temporary directory is created.
99-
If both `data_dir` and `dir_id` are input then a new folder is created within
100-
the `data_dir`, with the name "release_<dir_id>"
96+
data_dir: The directory where the files are stored.
97+
If None, the resource is initialized with a temporary directory.
10198
10299
TODO: make the directory functions in .core (bag3d_export_dir etc) members of this
103100
"""
104101

105102
data_dir: Optional[str] = None
106-
docker_volume_id: Optional[str] = None
107-
dir_id: Optional[str] = None
108103

109104
def __init__(
110105
self,
111106
data_dir: Optional[Union[Path, str]] = None,
112-
docker_volume_id: Optional[str] = None,
113-
dir_id: Optional[str] = None,
114107
):
115-
super().__init__(
116-
data_dir=str(data_dir) if data_dir else None,
117-
docker_volume_id=docker_volume_id,
118-
dir_id=dir_id,
119-
)
108+
super().__init__(data_dir=str(data_dir) if data_dir else None)
120109

121110
@property
122111
def file_store(self) -> FileStore:
123-
if self.data_dir and self.dir_id:
124-
return FileStore(data_dir=self.data_dir, dir_id=self.dir_id)
125-
elif self.data_dir:
112+
if self.data_dir:
126113
return FileStore(data_dir=self.data_dir)
127-
elif self.docker_volume_id:
128-
return FileStore(docker_volume_id=self.docker_volume_id)
129-
elif self.dir_id:
130-
return FileStore(dir_id=self.dir_id)
131114
else:
132115
return FileStore()

‎packages/common/tests/test_requests.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,12 @@ def test_download_extra(context):
6060
_ = download_extract(
6161
dataset="top10nl",
6262
url_api="https://api.pdok.nl/brt/top10nl/download/v1_0",
63-
featuretypes=context.op_config["featuretypes"],
63+
featuretypes=context.op_execution_context.op_execution_context.op_config[
64+
"featuretypes"
65+
],
6466
data_format="gml",
65-
geofilter=context.op_config.get("geofilter"),
67+
geofilter=context.op_execution_context.op_execution_context.op_config.get(
68+
"geofilter"
69+
),
6670
download_dir=context.resources.file_store.file_store.data_dir,
6771
)

‎packages/common/tests/test_resources.py‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -80,22 +80,6 @@ def test_file_store_init_temp():
8080
assert not path.exists()
8181

8282

83-
def test_file_store_init_temp_with_id():
84-
"""Can we create a local temporary directory with input id
85-
with the correct permissions?"""
86-
res = resources.files.FileStoreResource(dir_id="myID").file_store
87-
path = Path(res.data_dir)
88-
assert path.exists()
89-
assert str(res.data_dir)[-4:] == "myID"
90-
with (res.data_dir / "file.txt").open("w") as fo:
91-
fo.write("test")
92-
with (res.data_dir / "file.txt").open("r") as fo:
93-
assert fo.read() == "test"
94-
res.rm(force=True)
95-
assert not res.data_dir
96-
assert not path.exists()
97-
98-
9983
def test_file_store_init_data_dir(tmp_path):
10084
"""Can we use an existing directory?"""
10185
res = resources.files.FileStoreResource(data_dir=tmp_path).file_store

0 commit comments

Comments
 (0)