From 5f23ef3b764fcc4ebdf394c652b318b53cd8b18f Mon Sep 17 00:00:00 2001 From: Chad Burt Date: Tue, 8 Oct 2024 14:10:59 -0700 Subject: [PATCH] Add netcdf support --- .../uploads/ProjectBackgroundJobContext.tsx | 2 +- .../src/formats/netcdf.ts | 32 +++++++++++++++++++ .../src/handleUpload.ts | 4 ++- .../src/processRasterUpload.ts | 20 ++++++++++-- .../src/rasterInfoForBands.ts | 3 +- 5 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 packages/spatial-uploads-handler/src/formats/netcdf.ts diff --git a/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx b/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx index 71e4acd4..70cacf28 100644 --- a/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx +++ b/packages/client/src/admin/uploads/ProjectBackgroundJobContext.tsx @@ -432,7 +432,7 @@ export default function DataUploadDropzone({

{t( - "SeaSketch currently supports vector data in GeoJSON, Shapefile (zipped), GeoTiff, and FlatGeobuf formats." + "SeaSketch currently supports vector data in GeoJSON, Shapefile (zipped), GeoTiff, NetCDF and FlatGeobuf formats." )}

{Boolean(state.droppedFiles) && !state.error && ( diff --git a/packages/spatial-uploads-handler/src/formats/netcdf.ts b/packages/spatial-uploads-handler/src/formats/netcdf.ts new file mode 100644 index 00000000..2f95454b --- /dev/null +++ b/packages/spatial-uploads-handler/src/formats/netcdf.ts @@ -0,0 +1,32 @@ +import gdal from "gdal-async"; +import { Logger } from "../logger"; + +export async function getLayerIdentifiers(path: string, logger: Logger) { + const ids: string[] = []; + const data = await logger.exec( + ["gdalinfo", ["-json", path]], + "Problem getting layer identifiers" + ); + const info = JSON.parse(data); + const metadata = info.metadata || {}; + const subdatasets = metadata["SUBDATASETS"] || {}; + for (const key in subdatasets) { + if (/_NAME$/.test(key)) { + ids.push(subdatasets[key]); + } + } + console.log("ids", ids); + return ids; +} + +export async function convertToGeoTiff( + layerId: string, + outputPath: string, + logger: Logger +) { + await logger.exec( + ["gdal_translate", ["-co", "COMPRESS=DEFLATE", layerId, outputPath]], + "Problem converting to GeoTIFF" + ); + return outputPath; +} diff --git a/packages/spatial-uploads-handler/src/handleUpload.ts b/packages/spatial-uploads-handler/src/handleUpload.ts index 854e5ac9..15294c36 100644 --- a/packages/spatial-uploads-handler/src/handleUpload.ts +++ b/packages/spatial-uploads-handler/src/handleUpload.ts @@ -17,6 +17,7 @@ import { processRasterUpload } from "./processRasterUpload"; import { notifySlackChannel } from "./notifySlackChannel"; import { getObject, putObject } from "./remotes"; import { Logger } from "./logger"; +import { getLayerIdentifiers } from "./formats/netcdf"; export { SpatialUploadsHandlerRequest }; @@ -150,6 +151,7 @@ export default async function handleUpload( const s3LogPath = `s3://${process.env.BUCKET}/${jobId}.log.txt`; let { name, ext, base } = path.parse(objectKey); + name = sanitize(name); const originalName = name; name = `${jobId}`; @@ -170,7 +172,7 @@ export default async function handleUpload( // After the environment is set up, we can start processing the file depending // on its type try { - if (isTif) { + if (isTif || ext === ".nc") { stats = await processRasterUpload({ logger, path: workingFilePath, diff --git a/packages/spatial-uploads-handler/src/processRasterUpload.ts b/packages/spatial-uploads-handler/src/processRasterUpload.ts index c4aee1f8..cdc21139 100644 --- a/packages/spatial-uploads-handler/src/processRasterUpload.ts +++ b/packages/spatial-uploads-handler/src/processRasterUpload.ts @@ -9,6 +9,7 @@ import { rasterInfoForBands } from "./rasterInfoForBands"; import { Logger } from "./logger"; import gdal from "gdal-async"; import bbox from "@turf/bbox"; +import { convertToGeoTiff, getLayerIdentifiers } from "./formats/netcdf"; export async function processRasterUpload(options: { logger: Logger; @@ -41,6 +42,19 @@ export async function processRasterUpload(options: { const { ext, isCorrectProjection } = await validateInput(path, logger); + if (ext === ".nc") { + const layerIdentifiers = await getLayerIdentifiers(path, logger); + if (layerIdentifiers.length > 0) { + path = await convertToGeoTiff( + layerIdentifiers[0], + pathJoin(workingDirectory, jobId + ".tif"), + logger + ); + } else { + throw new Error("No layers found in NetCDF file"); + } + } + await updateProgress("running", "analyzing"); // Get raster stats const stats = await rasterInfoForBands(path); @@ -155,12 +169,12 @@ async function validateInput(path: string, logger: Logger) { // Use rasterio to see if it is a supported file format const isTif = ext === ".tif" || ext === ".tiff"; - if (!isTif) { - throw new Error("Only GeoTIFF files are supported"); + if (!isTif && ext !== ".nc") { + throw new Error("Only GeoTIFF and NetCDF files are supported"); } const ds = await gdal.openAsync(path); - if (ds.driver.description !== "GTiff") { + if (ds.driver.description !== "GTiff" && ds.driver.description !== "netCDF") { throw new Error(`Unrecognized raster driver "${ds.driver.description}"`); } diff --git a/packages/spatial-uploads-handler/src/rasterInfoForBands.ts b/packages/spatial-uploads-handler/src/rasterInfoForBands.ts index a15bef7b..d050e56c 100644 --- a/packages/spatial-uploads-handler/src/rasterInfoForBands.ts +++ b/packages/spatial-uploads-handler/src/rasterInfoForBands.ts @@ -169,7 +169,8 @@ export async function rasterInfoForBands( ) { // find a scaling factor that will represent the range of data values with // the full range of the encoding scheme. - if (range < 16_777_216) { + // This is useful for float values which may just be 0-1 + if (range < 500) { scale = 1; // stretch values to fit full encoding scheme // Use factors of 10, e.g. 10, 100, 1000, etc.