From 86fdc80fd2c9db6e60b6420e182a0e960d492d04 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 13 Jul 2023 10:30:55 -0400 Subject: [PATCH 1/2] add check size to builder to fail faster --- .../ft2/coverage/writer/CFGridCoverageWriter.java | 12 ++++++------ .../java/ucar/nc2/write/NetcdfFormatWriter.java | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CFGridCoverageWriter.java b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CFGridCoverageWriter.java index 1e9063e8ce..049fb4ee65 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CFGridCoverageWriter.java +++ b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/writer/CFGridCoverageWriter.java @@ -155,14 +155,14 @@ private Result writeFile(CoverageCollection gdsOrg, List gridNames, Subs } addCFAnnotations(subsetDataset, rootGroup, shouldAddLatLon2D); + // test if its too large + long totalSizeOfVars = writer.calcSize(); + if (maxBytes > 0 && totalSizeOfVars > maxBytes) { + return Result.create(totalSizeOfVars, false, TOO_LARGE_MESSAGE); + } + // Actually create file and write variable data to it. try (NetcdfFormatWriter ncwriter = writer.build()) { - // test if its too large - long totalSizeOfVars = ncwriter.calcSize(); - if (maxBytes > 0 && totalSizeOfVars > maxBytes) { - return Result.create(totalSizeOfVars, false, TOO_LARGE_MESSAGE); - } - writeCoordinateData(subsetDataset, ncwriter); writeCoverageData(gdsOrg, subsetParams, subsetDataset, ncwriter); diff --git a/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java b/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java index d766ea185b..88631f7897 100644 --- a/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java +++ b/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java @@ -245,6 +245,21 @@ public Structure.Builder addStructure(String shortName, String dimString) { rootGroup.addVariable(vb); return vb; } + public long calcSize() { + return calcSize(this.rootGroup); + } + + // Note that we have enough info to try to estimate effects of compression, if its a Netcdf4 file. + private long calcSize(Group.Builder group) { + long totalSizeOfVars = 0; + for (Variable.Builder var : this.rootGroup.vbuilders) { + totalSizeOfVars += Dimensions.getSize(var.getDimensions()) * var.getElementSize(); + } + for (Group.Builder nested : group.gbuilders) { + totalSizeOfVars += calcSize(nested); + } + return totalSizeOfVars; + } /** Once this is called, do not use the Builder again. */ public NetcdfFormatWriter build() throws IOException { From 9507546604bf89040ad931f69ba36ff40fdb6598 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 13 Jul 2023 10:39:06 -0400 Subject: [PATCH 2/2] spotless --- cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java b/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java index 88631f7897..3d968eceb5 100644 --- a/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java +++ b/cdm/core/src/main/java/ucar/nc2/write/NetcdfFormatWriter.java @@ -245,6 +245,7 @@ public Structure.Builder addStructure(String shortName, String dimString) { rootGroup.addVariable(vb); return vb; } + public long calcSize() { return calcSize(this.rootGroup); }