Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions src/backend/app/gcp/gcp_routes.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import os
import tempfile
import uuid
from typing import Annotated, List

Expand Down Expand Up @@ -57,12 +59,18 @@ async def save_gcp_file(
The file is stored at `projects/{project_id}/gcp.txt` in S3
and will be automatically included when final processing is triggered.
"""
gcp_file_path = f"/tmp/{uuid.uuid4()}"
with open(gcp_file_path, "wb") as f:
f.write(await gcp_file.read())
# Write the upload to a temp file and clean it up after the S3 put.
# Using delete=False + manual unlink is cross-platform safe because
# add_file_to_bucket needs to reopen the file by path.
with tempfile.NamedTemporaryFile(delete=False) as tmp:
tmp.write(await gcp_file.read())
gcp_file_path = tmp.name

s3_path = f"projects/{project.id}/gcp.txt"
add_file_to_bucket(settings.S3_BUCKET_NAME, gcp_file_path, s3_path)
try:
s3_path = f"projects/{project.id}/gcp.txt"
add_file_to_bucket(settings.S3_BUCKET_NAME, gcp_file_path, s3_path)
finally:
os.unlink(gcp_file_path)
log.info(f"GCP file saved for project {project.id} by user {user_data.id}")

return {"message": "GCP file saved successfully", "project_id": str(project.id)}
31 changes: 8 additions & 23 deletions src/backend/app/jaxa/upload_dem.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import io
import asyncio
import os
import shutil
import tempfile
from pathlib import Path

from loguru import logger as log
Expand Down Expand Up @@ -187,9 +189,10 @@ async def download_and_upload_dem(
f"with coordinates: {coordinates_str}"
)

# Use project-specific directory to avoid conflicts in k8s
project_dir = Path(f"/tmp/tif_processing/{project_id}")
project_dir.mkdir(parents=True, exist_ok=True)
# Use a project-scoped temp directory so concurrent jobs don't collide
# in k8s. tempfile picks the system temp root (TMPDIR / TEMP / /tmp)
# and the directory is cleaned up automatically on success or failure.
project_dir = Path(tempfile.mkdtemp(prefix=f"dtm-dem-{project_id}-"))
tif_file_path = str(project_dir / "merged.tif")

try:
Expand All @@ -208,24 +211,6 @@ async def download_and_upload_dem(
log.error(
f"DEM download job failed for project ({project_id}): {e}", exc_info=True
)

# Clean up on failure
if os.path.exists(tif_file_path):
os.remove(tif_file_path)
log.info(f"Cleaned up partial file: {tif_file_path}")

if project_dir.exists():
# Try to clean up any remaining files
for file in project_dir.glob("*"):
try:
file.unlink()
except Exception as cleanup_error:
log.warning(f"Could not clean up {file}: {cleanup_error}")

# Remove directory if empty
try:
project_dir.rmdir()
except Exception:
pass

raise
finally:
shutil.rmtree(project_dir, ignore_errors=True)
17 changes: 13 additions & 4 deletions src/backend/app/projects/classification_routes.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import os
import shutil
import tempfile
from datetime import datetime
from typing import Annotated, Literal, Optional
from uuid import UUID
Expand Down Expand Up @@ -924,12 +926,19 @@ async def download_reflight_plan(
)

flightplan_config = get_flightplan_output_config(flight_drone_type)
file_path = f"/tmp/reflight_{task_id}{flightplan_config['suffix']}"
work_dir = tempfile.mkdtemp(prefix="dtm-reflight-")
file_path = os.path.join(
work_dir, f"reflight_{task_id}{flightplan_config['suffix']}"
)

with open(file_path, "wb") as f:
f.write(result["kmz_bytes"])
try:
with open(file_path, "wb") as f:
f.write(result["kmz_bytes"])
except Exception:
shutil.rmtree(work_dir, ignore_errors=True)
raise

background_tasks.add_task(os.remove, file_path)
background_tasks.add_task(shutil.rmtree, work_dir, ignore_errors=True)

return build_flightplan_download_response(
file_path,
Expand Down
102 changes: 50 additions & 52 deletions src/backend/app/projects/project_logic.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import asyncio
import json
import os
import shutil
import tempfile
import uuid
from datetime import datetime, timezone
from io import BytesIO
Expand Down Expand Up @@ -675,20 +675,23 @@ async def process_task_metrics(db, tasks_data, project):
}

if project.is_terrain_follow:
dem_path = f"/tmp/{uuid.uuid4()}/dem.tif"
points = create_waypoint(**waypoint_params)
try:
get_file_from_bucket(
settings.S3_BUCKET_NAME,
f"projects/{project.id}/dem.tif",
dem_path,
)
outfile_with_elevation = "/tmp/output_file_with_elevation.geojson"
add_elevation_from_dem(dem_path, points, outfile_with_elevation)
with open(outfile_with_elevation) as inpointsfile:
points_with_elevation = inpointsfile.read()
except Exception:
points_with_elevation = points
with tempfile.TemporaryDirectory(prefix="dtm-dem-") as dem_work_dir:
dem_path = os.path.join(dem_work_dir, "dem.tif")
try:
get_file_from_bucket(
settings.S3_BUCKET_NAME,
f"projects/{project.id}/dem.tif",
dem_path,
)
outfile_with_elevation = os.path.join(
dem_work_dir, "output_file_with_elevation.geojson"
)
add_elevation_from_dem(dem_path, points, outfile_with_elevation)
with open(outfile_with_elevation, "r") as inpointsfile:
points_with_elevation = inpointsfile.read()
except Exception:
points_with_elevation = points

if (
isinstance(points_with_elevation, dict)
Expand Down Expand Up @@ -1393,50 +1396,45 @@ async def process_waypoints_and_waylines(
count_data = {"waypoints": 0, "waylines": 0}

if is_terrain_follow and dem:
temp_dir = f"/tmp/{uuid.uuid4()}"
dem_path = os.path.join(temp_dir, "dem.tif")

try:
os.makedirs(temp_dir, exist_ok=True)
# Read DEM content into memory and write to the file
file_content = await dem.read()
with open(dem_path, "wb") as file:
file.write(file_content)

# Process waypoints with terrain-follow elevation
waypoint_params["mode"] = FlightMode.WAYPOINTS
points = create_waypoint(**waypoint_params)

# Add elevation data to waypoints
outfile_with_elevation = os.path.join(
temp_dir, "output_file_with_elevation.geojson"
)
add_elevation_from_dem(dem_path, points, outfile_with_elevation)
# TemporaryDirectory handles cleanup automatically, even on exceptions.
with tempfile.TemporaryDirectory(prefix="dtm-waypoints-") as temp_dir:
dem_path = os.path.join(temp_dir, "dem.tif")

# Read the updated waypoints with elevation
with open(outfile_with_elevation) as inpointsfile:
points_with_elevation = inpointsfile.read()
count_data["waypoints"] = len(
json.loads(points_with_elevation)["features"]
try:
# Read DEM content into memory and write to the file
file_content = await dem.read()
with open(dem_path, "wb") as file:
file.write(file_content)

# Process waypoints with terrain-follow elevation
waypoint_params["mode"] = FlightMode.WAYPOINTS
points = create_waypoint(**waypoint_params)

# Add elevation data to waypoints
outfile_with_elevation = os.path.join(
temp_dir, "output_file_with_elevation.geojson"
)
add_elevation_from_dem(dem_path, points, outfile_with_elevation)

# Generate waylines from waypoints with elevation
wayline_placemarks = create_placemarks(
geojson.loads(points_with_elevation), parameters
)
# Read the updated waypoints with elevation
with open(outfile_with_elevation, "r") as inpointsfile:
points_with_elevation = inpointsfile.read()
count_data["waypoints"] = len(
json.loads(points_with_elevation)["features"]
)

placemarks = terrain_following_waylines.waypoints2waylines(
wayline_placemarks, 5
)
count_data["waylines"] = len(placemarks["features"])
# Generate waylines from waypoints with elevation
wayline_placemarks = create_placemarks(
geojson.loads(points_with_elevation), parameters
)

except Exception as e:
log.error(f"Error processing DEM: {e}")
placemarks = terrain_following_waylines.waypoints2waylines(
wayline_placemarks, 5
)
count_data["waylines"] = len(placemarks["features"])

finally:
# Cleanup temporary files and directory
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
except Exception as e:
log.error(f"Error processing DEM: {e}")
return count_data

else:
Expand Down
13 changes: 12 additions & 1 deletion src/backend/app/waypoints/flightplan_output.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
from typing import Optional

from fastapi import HTTPException
from fastapi.responses import FileResponse
from starlette.background import BackgroundTask

from drone_flightplan.drone_type import DRONE_PARAMS, DroneType

Expand Down Expand Up @@ -44,11 +47,19 @@ def build_flightplan_download_response(
outpath: str,
drone_type: DroneType,
filename_stem: str,
cleanup: Optional[BackgroundTask] = None,
):
"""Wrap a generated flightplan file in the correct download response."""
"""Wrap a generated flightplan file in the correct download response.

If ``cleanup`` is supplied, it runs after the response body has been
streamed to the client. Callers that generate the file in a
``tempfile.TemporaryDirectory`` should pass a BackgroundTask that
removes the directory so it does not leak on disk.
"""
config = get_flightplan_output_config(drone_type)
return FileResponse(
outpath,
media_type=config["media_type"],
filename=f"{filename_stem}{config['suffix']}",
background=cleanup,
)
Loading
Loading