From 7e0e1ca148a9d0e6aa9dbb3457970fb11fe62906 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 27 Jan 2024 11:10:13 +0100 Subject: [PATCH 01/16] cdm-radial: sigmet Support for all data types instead of just 5 Universal handling for data types instead of each one separately Support for data types spanning two bytes instead of one is experimental at this point as I didn't have such a file to test Most new data types are returned as raw values meaning no normalization like done for total power, reflectivity, velocity, width, differential reflectivity Added comments and small fixes Ultimately all these changes add support for IDEAM Colombia files Addresses the github issue Unidata/netcdf-java#1292 --- .../main/java/ucar/nc2/iosp/sigmet/Ray.java | 56 +++- .../iosp/sigmet/SigmetIOServiceProvider.java | 256 +++++++++++---- .../nc2/iosp/sigmet/SigmetVolumeScan.java | 292 +++++++++++------- 3 files changed, 418 insertions(+), 186 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index f6e557231b..d06afe8511 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -201,18 +201,27 @@ public String toString() { public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) throws IOException { int REC_SIZE = 6144; raf.seek(offset); - byte[] data = new byte[bins]; - float[] dd = new float[bins]; - byte d; + Number[] dd = new Number[bins]; int nb = 0; short a00; + + short dty = getDataType(); + String var_name = SigmetVolumeScan.data_name[dty]; + // support for 2 byte data types is experimental + boolean twoBytes = var_name.endsWith("_2"); + // raf.readFully(data); if (dataRead > 0) { raf.seek(offset); for (int i = 0; i < dataRead; i++) { - d = raf.readByte(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), d); + if (!twoBytes) { + byte d = raf.readByte(); + dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + } else { + short d = raf.readShort(); + dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + } nb++; } } @@ -240,10 +249,17 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th } raf.seek(cur_len); for (int i = 0; i < dataRead1; i++) { - d = raf.readByte(); - dd[nb] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), d); + if (!twoBytes) { + byte d = raf.readByte(); + dd[nb] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + cur_len = cur_len + 1; + } else { + short d = raf.readShort(); + dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + cur_len = cur_len + 2; + } nb = nb + 1; - cur_len = cur_len + 1; + if (nb % REC_SIZE == 0) { pos = i + 1; break; @@ -257,7 +273,10 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th int num_zero = a00 * 2; int dataRead1 = num_zero; for (int k = 0; k < dataRead1; k++) { - dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), (byte) 0); + if (!twoBytes) + dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (byte) 0); + else + dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (short) 0); } nb = nb + dataRead1; if (cur_len % REC_SIZE == 0) { @@ -267,11 +286,20 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th } } // ------ end of while for num_bins--------------------------------- - for (int gateIdx : gateRange) { - if (gateIdx >= bins) - ii.setFloatNext(Float.NaN); - else - ii.setFloatNext(dd[gateIdx]); + if (!twoBytes) { + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setFloatNext(Float.NaN); + else + ii.setFloatNext(dd[gateIdx].floatValue()); + } + } else { + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setDoubleNext(Double.NaN); + else + ii.setDoubleNext(dd[gateIdx].doubleValue()); + } } } // end of readData diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index 39599c325d..09f6d9d62c 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -5,25 +5,9 @@ package ucar.nc2.iosp.sigmet; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.ma2.Range; -import ucar.ma2.Array; -import ucar.ma2.ArrayFloat; -import ucar.ma2.ArrayInt; -import ucar.ma2.DataType; -import ucar.ma2.Index; -import ucar.ma2.IndexIterator; -import ucar.ma2.InvalidRangeException; -import ucar.ma2.Section; +import ucar.ma2.*; import ucar.nc2.Attribute; import ucar.nc2.Dimension; import ucar.nc2.Variable; @@ -35,6 +19,15 @@ import ucar.nc2.iosp.LayoutRegular; import ucar.unidata.io.RandomAccessFile; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + /** * Implementation of the ServerProvider pattern provides input/output * of the SIGMET-IRIS dataset. IOSP are managed by the NetcdfFile class. @@ -78,12 +71,16 @@ public class SigmetIOServiceProvider extends AbstractIOServiceProvider { private static Logger logger = LoggerFactory.getLogger(SigmetIOServiceProvider.class); + + // meaning unprocessed + public static String RAW_VARIABLE_PREFIX = "raw_"; + private ArrayList varList; private int[] tsu_sec; private int[] sweep_bins; private String date0; - public static java.util.Map recHdr = new java.util.HashMap<>(); + public static Map recHdr = new java.util.HashMap<>(); private SigmetVolumeScan volScan; public String getFileTypeDescription() { @@ -101,7 +98,7 @@ public String getFileTypeId() { /** * Check if this is a valid SIGMET-IRIS file for this IOServiceProvider. */ - public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) { + public boolean isValidFile(RandomAccessFile raf) { try { raf.order(RandomAccessFile.LITTLE_ENDIAN); // The first struct in the file is the product_hdr, which will have the @@ -124,11 +121,11 @@ public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) { /** * Open existing file, and populate ncfile with it. */ - public void open(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - ucar.nc2.util.CancelTask cancelTask) throws java.io.IOException { + public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, + ucar.nc2.util.CancelTask cancelTask) throws IOException { super.open(raf, ncfile, cancelTask); // java.util.Map recHdr=new java.util.HashMap(); - java.util.Map hdrNames = new java.util.HashMap<>(); + Map hdrNames = new java.util.HashMap<>(); volScan = new SigmetVolumeScan(raf, ncfile, varList); this.varList = init(raf, ncfile, hdrNames); @@ -141,8 +138,8 @@ public void open(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfil * Read some global data from SIGMET file. The SIGMET file consists of records with * fixed length=6144 bytes. */ - public static java.util.Map readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) { - java.util.Map recHdr1 = new java.util.HashMap<>(); + public static Map readRecordsHdr(RandomAccessFile raf) { + Map recHdr1 = new java.util.HashMap<>(); try { int nparams = 0; // -- Read from of the 1st record -- 12+320+120 @@ -164,17 +161,51 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando short num_rays = raf.readShort(); // 6340 raf.skipBytes(2); int radar_alt = raf.readInt(); // 6344 - raf.seek(6648); - int time_beg = raf.readInt(); + // end of ingest_config would be 6636 + + // next is 2612 Bytes + raf.seek(6648); // + task_configuration 12 + // inner 120 Bytes + int time_beg = raf.readInt(); // Start time (seconds within a day) raf.seek(6652); - int time_end = raf.readInt(); - raf.seek(6772); + int time_end = raf.readInt(); // Stop time (seconds within a day) + // end inner would be 6768 + + // next inner is 320 Bytes. + raf.seek(6772); // + 2 Major mode + 2 DSP type + + // 4 - 24 - Current Data type mask + // Mask word 0 int data_mask = raf.readInt(); for (int j = 0; j < 32; j++) { nparams += ((data_mask >> j) & (0x1)); } + raf.readInt(); // Extended header type + // Mask word 1 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 2 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 3 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 4 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + raf.seek(6912); short multiprf = raf.readShort(); + // end inner would be 7088 + raf.seek(7408); int range_first = raf.readInt(); // cm 7408 int range_last = raf.readInt(); // cm 7412 @@ -186,6 +217,9 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando int step = raf.readInt(); // cm 7424 raf.seek(7574); short number_sweeps = raf.readShort(); // 7574 + // end of task_configuration would be 9248 + + raf.seek(12312); int base_time = raf.readInt(); // 3d rec raf.skipBytes(2); @@ -218,8 +252,8 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando /** * Read StationName strings */ - public java.util.Map readStnNames(ucar.unidata.io.RandomAccessFile raf) { - java.util.Map hdrNames = new java.util.HashMap<>(); + public Map readStnNames(RandomAccessFile raf) { + Map hdrNames = new java.util.HashMap<>(); try { raf.seek(6288); String stnName = raf.readString(16); @@ -241,12 +275,12 @@ public java.util.Map readStnNames(ucar.unidata.io.RandomAccessFi * @param hdrNames java.util.Map with values for "StationName.." Attributes * @return ArrayList of Variables of ncfile */ - public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - java.util.Map hdrNames) { + public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, + Map hdrNames) { // prepare attribute values - String[] data_name = {" ", "TotalPower", "Reflectivity", "Velocity", "Width", "Differential_Reflectivity"}; - String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; - int[] type = {1, 2, 3, 4, 5}; + String[] data_name = SigmetVolumeScan.data_name; + String[] unit = SigmetVolumeScan.data_unit; + String def_datafile = "SIGMET-IRIS"; String tim = ""; int ngates = 0; @@ -305,22 +339,37 @@ public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.N ArrayList varList = new ArrayList<>(); + short[] dataTypes = volScan.getDataTypes(); + Variable[][] v = new Variable[nparams][number_sweeps]; String var_name; for (int j = 0; j < nparams; j++) { - int tp = type[j]; - var_name = data_name[tp]; + short dty = dataTypes[j]; + String var_name_original = data_name[dty]; + + // see also calcData - a lot of data types are raw values + if (dty == 6 || dty > 11) + // this is make it obvious to the user + var_name_original = RAW_VARIABLE_PREFIX + var_name_original; + + var_name = var_name_original; + // support for 2 byte data types is experimental + boolean twoBytes = var_name.endsWith("_2"); + for (int jj = 0; jj < number_sweeps; jj++) { if (number_sweeps > 1) { - var_name = data_name[tp] + "_sweep_" + (jj + 1); + var_name = var_name_original + "_sweep_" + (jj + 1); } v[j][jj] = new Variable(ncfile, null, null, var_name); - v[j][jj].setDataType(DataType.FLOAT); + if (twoBytes) + v[j][jj].setDataType(DataType.DOUBLE); + else + v[j][jj].setDataType(DataType.FLOAT); dims2.add(radial); dims2.add(gateR[jj]); v[j][jj].setDimensions(dims2); v[j][jj].addAttribute(new Attribute(CDM.LONG_NAME, var_name)); - v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[tp])); + v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[dty])); String coordinates = "time elevationR azimuthR distanceR"; v[j][jj].addAttribute(new Attribute(_Coordinate.Axes, coordinates)); v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, -999.99f)); @@ -485,7 +534,7 @@ public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.N * @param recHdr java.util.Map with values for Attributes */ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[] yr, short[] m, short[] dda, - ArrayList varList, java.util.Map recHdr) { + ArrayList varList, Map recHdr) { // prepare attribute values String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; @@ -559,9 +608,16 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ distArr[i].setFloat(distIndex[i].set(ii), (range_first + ii * stp)); } } - List rgp = volScan.getTotalPowerGroups(); - if (rgp.isEmpty()) - rgp = volScan.getReflectivityGroups(); + + // used for lists of distance, time, azimuth, elevation + // probably there is a better way to do this, but I'm keeping the old logic + List rgp = volScan.getGroup("TotalPower"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("TotalPower_2"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("Reflectivity"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("Reflectivity_2"); List[] sgp = new ArrayList[number_sweeps]; for (int i = 0; i < number_sweeps; i++) { sgp[i] = (List) rgp.get((short) i); @@ -586,11 +642,12 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ timeArr[i] = (ArrayInt.D1) Array.factory(DataType.INT, time[i].getShape()); timeIndex[i] = timeArr[i].getIndex(); List rlist = sgp[i]; + int num_rays_actual = Math.min(num_rays, rlist.size()); - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); } } @@ -612,11 +669,12 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ azimArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, azimuthR[i].getShape()); azimIndex[i] = azimArr[i].getIndex(); List rlist = sgp[i]; + int num_rays_actual = Math.min(num_rays, rlist.size()); - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); } } @@ -638,11 +696,12 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ elevArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, elevationR[i].getShape()); elevIndex[i] = elevArr[i].getIndex(); List rlist = sgp[i]; + int num_rays_actual = Math.min(num_rays, rlist.size()); - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); } } @@ -661,7 +720,9 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ for (int i = 0; i < number_sweeps; i++) { List rlist = sgp[i]; - for (int jj = 0; jj < num_rays; jj++) { + int num_rays_actual = Math.min(num_rays, rlist.size()); + + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } ngates = rtemp[0].getBins(); @@ -689,7 +750,7 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ * of ucar.ma2.Range which define the requested data subset. * @return Array of data which will be read from Variable through this call. */ - public Array readData1(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { + public Array readData1(Variable v2, Section section) throws IOException, InvalidRangeException { // doData(raf, ncfile, varList); int[] sh = section.getShape(); Array temp = Array.factory(v2.getDataType(), sh); @@ -715,17 +776,16 @@ public Array readData(Variable v2, Section section) throws IOException { List> groups; String shortName = v2.getShortName(); - if (shortName.startsWith("Reflectivity")) - groups = volScan.getReflectivityGroups(); - else if (shortName.startsWith("Velocity")) - groups = volScan.getVelocityGroups(); - else if (shortName.startsWith("TotalPower")) - groups = volScan.getTotalPowerGroups(); - else if (shortName.startsWith("Width")) - groups = volScan.getWidthGroups(); - else if (shortName.startsWith("DiffReflectivity")) - groups = volScan.getDifferentialReflectivityGroups(); - else + String groupName = shortName; + + int posSweep = groupName.indexOf("_sweep_"); + if (posSweep > 0) + groupName = groupName.substring(0, posSweep); + if (groupName.startsWith(RAW_VARIABLE_PREFIX)) + groupName = groupName.substring(RAW_VARIABLE_PREFIX.length()); + + groups = volScan.getGroup(groupName); + if (groups == null) throw new IllegalStateException("Illegal variable name = " + shortName); if (section.getRank() == 2) { @@ -759,7 +819,7 @@ private void readOneScan(List mapScan, Range radialRange, Range gateRange, } private void readOneRadial(Ray r, Range gateRange, IndexIterator ii) throws IOException { - if (r == null) { + if (r == null || r.offset == -999) { for (int i = 0; i < gateRange.length(); i++) ii.setFloatNext(Float.NaN); return; @@ -811,8 +871,8 @@ public Array readFloatData(LayoutRegular index, Variable v2) throws IOException * @param channel WritableByteChannel object - channel that can write bytes. * @return the number of bytes written, possibly zero. */ - public long readToByteChannel11(ucar.nc2.Variable v2, Section section, WritableByteChannel channel) - throws java.io.IOException { + public long readToByteChannel11(Variable v2, Section section, WritableByteChannel channel) + throws IOException { Array data = readData(v2, section); float[] ftdata = new float[(int) data.getSize()]; byte[] bytedata = new byte[(int) data.getSize() * 4]; @@ -946,7 +1006,8 @@ static float calcData(Map recHdr, short dty, byte data) { float vNyq = recHdr.get("vNyq").floatValue(); double temp = -999.99; switch (dty) { - default: // dty=1,2 -total_power, reflectivity (dBZ) + case 1:// dty=1,2 -total_power, reflectivity (dBZ) + case 2: if (data != 0) { temp = (((int) data & 0xFF) - 64) * 0.5; } @@ -967,12 +1028,73 @@ static float calcData(Map recHdr, short dty, byte data) { temp = ((((int) data & 0xFF) - 128) / 16.0); } break; + default: + // TODO implement for more SigmetVolumeScan.data_name (only 1 byte) + // using only the raw value + temp = (int) data & 0xFF; + + // doing no rounding in this case as below for the other cases + BigDecimal bd = new BigDecimal(temp); + return bd.floatValue(); + //logger.warn("calcData: unimplemented 1 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); } BigDecimal bd = new BigDecimal(temp); + // this should be reviewed, not sure why would you want a loss of precision here? BigDecimal result = bd.setScale(2, RoundingMode.HALF_DOWN); return result.floatValue(); } + /** + * Calculate data values from raw ingest data + * + * @param recHdr java.util.Map object with values for calculation + * @param dty type of data ( "Total_Power", "Reflectivity", "Velocity", + * "Width", "Differential_Reflectivity") + * @param data 2-byte input value + * @return double value + */ + static double calcData(Map recHdr, short dty, short data) { + short[] coef = {1, 2, 3, 4}; // MultiPRF modes + short multiprf = recHdr.get("multiprf").shortValue(); + float vNyq = recHdr.get("vNyq").floatValue(); + double temp = -999.99; + + switch (dty) { + case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) + case 8: + if (data != 0) { + temp = (data - 64) * 0.5; + } + break; + case 9: // dty=9 - mean velocity 2 (m/sec) + if (data != 0) { + temp = ((data - 128) / 127.0) * vNyq * coef[multiprf]; + } + break; + case 10: // dty=10 - spectrum width 2 (m/sec) + if (data != 0) { + double v = ((data - 128) / 127.0) * vNyq * coef[multiprf]; + temp = (data / 256.0) * v; + } + break; + case 11: // dty=11 - differential reflectivity 2 (dB) + if (data != 0) { + temp = ((data - 128) / 16.0); + } + break; + default: + // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) + // using only the raw value + temp = data; + //logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); + break; + } + + BigDecimal bd = new BigDecimal(temp); + // full precision + return bd.doubleValue(); + } + /** * Calculate time as hh:mm:ss * diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 679c970eb5..7da489a57c 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import ucar.nc2.Variable; import ucar.unidata.io.RandomAccessFile; + import java.util.*; /** @@ -20,13 +21,126 @@ */ public class SigmetVolumeScan { private static Logger logger = LoggerFactory.getLogger(SigmetVolumeScan.class); - String[] data_name = {" ", "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity"}; - private List> differentialReflectivityGroups; - private List> reflectivityGroups; - private List> totalPowerGroups; - private List> velocityGroups; - private List> widthGroups; - private List> timeGroups; + /** + * See IRIS-Programming-Guide-M211318EN: 4.9 Constants + * empty or undefined types are set to their index number to avoid duplicates + * if suffix is "_2" it means a two byte value instead of just one byte + * (2 bytes are experimental at this point - didn't have a file to test) + */ + public static final String[] data_name = { + "ExtendedHeaders", // ? bytes + "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity", "[6]", "CorrectedReflectivity", // 1 byte + "TotalPower_2", "Reflectivity_2", "Velocity_2", "Width_2", "DifferentialReflectivity_2", // 2 bytes + "RainfallRate_2" /* 2 bytes */, "KDPDifferentialPhase" /* 1 byte */, "KDPDifferentialPhase_2" /* 2 bytes */, + "PhiDPDifferentialPhase" /* 1 byte */, "CorrectedVelocity" /* 1 byte */, "SQI" /* 1 byte */, + "RhoHV" /* 1 byte */, "RhoHV_2" /* 2 bytes */, + "CorrectedReflectivity_2" /* 2 bytes */, "CorrectedVelocity_2" /* 2 bytes */, "SQI_2" /* 2 bytes */, + "PhiDPDifferentialPhase_2" /* 2 bytes */, + "LDRH" /* 1 byte */, "LDRH_2" /* 2 bytes */, "LDRV" /* 1 byte */, "LDRV_2" /* 2 bytes */, + "[29]", "[30]", "[31]", + "Height" /* 1 byte */, "LinearLiquid_2" /* 2 bytes */, "RawData" /* ? */, + "WindShear" /* 1 byte */, "Divergence_2" /* 2 bytes */, "FloatedLiquid_2" /* 2 bytes */, + "UserType" /* 1 byte */, "UnspecifiedData" /* 1 byte */, "Deformation_2" /* 2 bytes */, + "VerticalVelocity_2" /* 2 bytes */, "HorizontalVelocity_2" /* 2 bytes */, + "HorizontalWindDirection_2" /* 2 bytes */, "AxisOfDilatation_2" /* 2 bytes */, "TimeInSeconds_2" /* 2 bytes */, + "RHOH" /* 1 byte */, "RHOH_2" /* 2 bytes */, "RHOV" /* 1 byte */, "RHOV_2" /* 2 bytes */, + "PHIH" /* 1 byte */, "PHIH_2" /* 2 bytes */, "PHIV" /* 1 byte */, "PHIV_2" /* 2 bytes */, + "UserType_2" /* 2 bytes */, "HydrometeorClass" /* 1 byte */, "HydrometeorClass_2" /* 2 bytes */, + "CorrectedDifferentialReflectivity" /* 1 byte */, "CorrectedDifferentialReflectivity_2" /* 2 bytes */, + // 16 empty + "[59]", "[60]", "[61]", "[62]", "[63]", "[64]", "[65]", "[66]", "[67]", "[68]", "[69]", "[70]", "[71]", "[72]", "[73]", "[74]", + "PolarimetricMeteoIndex" /* 1 byte */, "PolarimetricMeteoIndex_2" /* 2 bytes */, + "LOG8" /* 1 byte */, "LOG16_2" /* 2 bytes */, "CSP8" /* 1 byte */, "CSP16_2" /* 2 bytes */, + "CCOR8" /* 1 byte */, "CCOR16_2" /* 2 bytes */, "AH8" /* 1 byte */, "AH16_2" /* 2 bytes */, + "AV8" /* 1 byte */, "AV16_2" /* 2 bytes */, "AZDR8" /* 1 byte */, "AZDR16_2" /* 2 bytes */, + }; + + /* + Some units are unknown, some were correlated and assumed by ChatGPT. + Not all might be correct! + Just extending the initial list here from previous version of SigmetIOServiceProvider + // TODO fill and double check + */ + public static final String[] data_unit = { + "?", // DB_XHDR: Extended Headers + "dBm", // DB_DBT: Total Power (1 byte) + "dBZ", // DB_DBZ: Reflectivity (1 byte) + "m/s", // DB_VEL: Velocity (1 byte) + "m/s", // DB_WIDTH: Width (1 byte) + "dB", // DB_ZDR: Differential Reflectivity (1 byte) + "?", // empty + "dBZ", // DB_DBZC: Corrected Reflectivity (1 byte) + "dBm", // DB_DBT2: Total Power (2 byte) + "dBZ", // DB_DBZ2: Reflectivity (2 byte) + "m/s", // DB_VEL2: Velocity (2 byte) + "m/s", // DB_WIDTH2: Width (2 byte) + "dB", // DB_ZDR2: Differential Reflectivity (2 byte) + "mm/hr", // DB_RAINRATE2: Rainfall Rate (2 byte) + "°/km", // DB_KDP: KDP (Differential Phase) (1 byte) + "°/km", // DB_KDP2: KDP (Differential Phase) (2 byte) + "°", // DB_PHIDP: PhiDP (Differential Phase) (1 byte) + "m/s", // DB_VELC: Corrected Velocity (1 byte) + "?", // DB_SQI: SQI (Signal Quality Index) (1 byte) + "?", // DB_RHOHV: RhoHV (1 byte) + "?", // DB_RHOHV2: RhoHV (2 byte) + "dBZ", // DB_DBZC2: Corrected Reflectivity (2 byte) + "m/s", // DB_VELC2: Corrected Velocity (2 byte) + "?", // DB_SQI2: SQI (Signal Quality Index) (2 byte) + "°", // DB_PHIDP2: PhiDP (Differential Phase) (2 byte) + "?", // DB_LDRH: LDR xmt H rcv V (1 byte) + "?", // DB_LDRH2: LDR xmt H rcv V (2 byte) + "?", // DB_LDRV: LDR xmt V rcv H (1 byte) + "?", // DB_LDRV2: LDR xmt V rcv H (2 byte) + // 3 empty + "?", "?", "?", + "1/10 km", // DB_HEIGHT: Height (1/10 km) (1 byte) + ".001mm", // DB_VIL2: Linear liquid (.001mm) (2 byte) + "?", // DB_RAW: Unknown unit or unitless (Raw Data) + "m/s", // DB_SHEAR: Shear (Velocity difference) + "?", // DB_DIVERGE2: Divergence (2 byte) + "mm", // DB_FLIQUID2: Liquid equivalent (2 byte) + "?", // DB_USER: User-defined (unit depends on definition) + "?", // DB_OTHER: Other data type (unit depends on definition) + "1/s", // DB_DEFORM2: Deformation (2 byte) + "m/s", // DB_VVEL2: Vertical Velocity (2 byte) + "m/s", // DB_HVEL2: Horizontal Velocity (2 byte) + "°", // DB_HDIR2: Horizontal Direction (2 byte) + "1/s", // DB_AXDIL2: Axis of Dilation (2 byte) + "s", // DB_TIME2: Time (2 byte) + "?", // DB_RHOH: RhoH (1 byte) + "?", // DB_RHOH2: RhoH (2 byte) + "?", // DB_RHOV: RhoV (1 byte) + "?", // DB_RHOV2: RhoV (2 byte) + "°", // DB_PHIH: PhiH (1 byte) + "°", // DB_PHIH2: PhiH (2 byte) + "°", // DB_PHIV: PhiV (1 byte) + "°", // DB_PHIV2: PhiV (2 byte) + "?", // DB_USER2: User-defined (2 byte) + "?", // DB_HCLASS: Hydrometeor Classification (1 byte) + "?", // DB_HCLASS2: Hydrometeor Classification (2 byte) + "dB", // DB_ZDRC: Corrected Differential Reflectivity (1 byte) + "dB", // DB_ZDRC2: Corrected Differential Reflectivity (2 byte) + // 16 empty + "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", + "?", // PolarimetricMeteoIndex (1 byte) + "?", // PolarimetricMeteoIndex2 (2 bytes) + "?", // LOG8 (1 byte) + "?", // LOG16 (2 bytes) + "?", // CSP8 (1 byte) + "?", // CSP16 (2 bytes) + "?", // CCOR8 (1 byte) + "?", // CCOR16 (2 bytes) + "?", // AH8 (1 byte) + "?", // AH16 (2 bytes) + "?", // AV8 (1 byte) + "?", // AV16 (2 bytes) + "?", // AZDR8 (1 byte) + "?", // AZDR16 (2 bytes) + }; + + private HashMap>> allGroups = new HashMap<>(); + + private short[] data_type; private int[] num_gates; public int[] base_time; public short[] year; @@ -34,13 +148,7 @@ public class SigmetVolumeScan { public short[] day; public Ray firstRay; public Ray lastRay; - public ucar.unidata.io.RandomAccessFile raf; - public boolean hasReflectivity; - public boolean hasVelocity; - public boolean hasWidth; - public boolean hasTotalPower; - public boolean hasDifferentialReflectivity; - public boolean hasTime; + public RandomAccessFile raf; /** * Read all the values from SIGMET-IRIS file which are necessary to fill in the ncfile. @@ -49,7 +157,7 @@ public class SigmetVolumeScan { * @param ncfile an empty NetcdfFile object which will be filled. * @param varList ArrayList of Variables of ncfile */ - SigmetVolumeScan(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) + SigmetVolumeScan(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) throws java.io.IOException { int REC_SIZE = 6144; int len = 12288; // ---- Read from the 3d record----------- 6144*2=12288 @@ -68,7 +176,7 @@ public class SigmetVolumeScan { raf.order(RandomAccessFile.LITTLE_ENDIAN); int fileLength = (int) raf.length(); - java.util.Map recHdr = SigmetIOServiceProvider.readRecordsHdr(raf); + Map recHdr = SigmetIOServiceProvider.readRecordsHdr(raf); int nparams = recHdr.get("nparams").intValue(); short number_sweeps = recHdr.get("number_sweeps").shortValue(); @@ -84,10 +192,11 @@ public class SigmetVolumeScan { short[] num_sweep = new short[nparams]; short[] num_rays_swp = new short[nparams]; short[] indx_1ray = new short[nparams]; + short[] num_rays_exp = new short[nparams]; short[] num_rays_act = new short[nparams]; short[] angl_swp = new short[nparams]; short[] bin_len = new short[nparams]; - short[] data_type = new short[nparams]; + data_type = new short[nparams]; // float[] dd = new float[bins]; num_gates = new int[number_sweeps]; base_time = new int[nparams * number_sweeps]; @@ -97,15 +206,10 @@ public class SigmetVolumeScan { // Array of Ray objects is 2D. Number of columns=number of rays // Number of raws = number of types of data if number_sweeps=1, // or number of raws = number_sweeps - List totalPower = new ArrayList<>(); - List velocity = new ArrayList<>(); - List reflectivity = new ArrayList<>(); - List width = new ArrayList<>(); - List diffReflectivity = new ArrayList<>(); - List time = new ArrayList<>(); + HashMap> all = new HashMap<>(); + int irays = (int) num_rays; Ray ray = null; - int two = 0; // init array float[] val = new float[bins]; @@ -142,7 +246,19 @@ public class SigmetVolumeScan { beg = 0; for (int i = 0; i < nparams; i++) { - int idh_len = cur_len + 12 + i * 76; + int idh_len = cur_len + 12 + i * 76; // + 12 == skipping over + + /* debug structure_header + raf.seek(idh_len-12); + // Structure identifier + short si = raf.readShort(); + raf.readShort(); // format version + int nob = raf.readInt(); + if (si != (short)24) + throw new IllegalStateException("not a Ingest_header"); + raf.readShort(); // reserved + raf.readShort(); // flags + */ raf.seek(idh_len); @@ -156,10 +272,12 @@ public class SigmetVolumeScan { num_sweep[i] = raf.readShort(); // idh_len+12 num_rays_swp[i] = raf.readShort(); // idh_len+14 indx_1ray[i] = raf.readShort(); // idh_len+16 - raf.skipBytes(2); + num_rays_exp[i] = raf.readShort(); + beg += num_rays_exp[i]; // before num_rays_act[i] was used but it does seem not work num_rays_act[i] = raf.readShort(); - beg += num_rays_act[i]; // idh_len+20 + //beg += num_rays_act[i]; // idh_len+20 angl_swp[i] = raf.readShort(); // idh_len+22 + // TODO maybe use in stead of variable twoBytes? bin_len[i] = raf.readShort(); // idh_len+24 data_type[i] = raf.readShort(); // idh_len+26 } @@ -220,6 +338,8 @@ public class SigmetVolumeScan { } String var_name = data_name[dty]; + // support for 2 byte data types is experimental + boolean twoBytes = var_name.endsWith("_2"); // --- read ray_header (size=12 bytes=6 words)--------------------------------------- if (read_ray_hdr) { @@ -333,6 +453,10 @@ public class SigmetVolumeScan { // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); cur_len++; nb++; + if (twoBytes) { + cur_len++; + //i++;? + } if (cur_len % REC_SIZE == 0) { pos = i + 1; @@ -400,6 +524,10 @@ public class SigmetVolumeScan { // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); cur_len = cur_len + 1; nb = nb + 1; + if (twoBytes) { + cur_len++; + //ii++;? + } if (cur_len % REC_SIZE == 0) { pos = ii + 1; @@ -422,6 +550,7 @@ public class SigmetVolumeScan { // } nb = nb + num_zero; + // TODO extra handling for twoBytes here, too? if (cur_len % REC_SIZE == 0) { beg_rec = true; @@ -450,21 +579,15 @@ public class SigmetVolumeScan { ray = new Ray(range_first, step, az, elev, num_bins, time_start_sw, rayoffset, datalen, rayoffset1, nsweep, var_name, dty); rays_count++; - two++; + if ((nsweep == number_sweeps) & (rays_count % beg == 0)) { - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found1 {}", var_name); + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); } + varL.add(ray); break; } @@ -476,40 +599,29 @@ public class SigmetVolumeScan { data_read = 0; nb = 0; len = cur_len; - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found2 {}", var_name); + + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); } + varL.add(ray); + continue; } } - // "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity" if (firstRay == null) firstRay = ray; - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found3 {}", var_name); + // using a universal structure that works for all data types + List additionalL = all.get(var_name.trim()); + if (additionalL == null) { + additionalL = new ArrayList<>(); + all.put(var_name.trim(), additionalL); } + additionalL.add(ray); pos = 0; data_read = 0; @@ -530,30 +642,11 @@ public class SigmetVolumeScan { } // ------------end of outer while --------------- lastRay = ray; - if (!reflectivity.isEmpty()) { - reflectivityGroups = sortScans("reflectivity", reflectivity, 1000); - hasReflectivity = true; - } - if (!velocity.isEmpty()) { - velocityGroups = sortScans("velocity", velocity, 1000); - hasVelocity = true; - } - if (!totalPower.isEmpty()) { - totalPowerGroups = sortScans("totalPower", totalPower, 1000); - hasTotalPower = true; - } - if (!width.isEmpty()) { - widthGroups = sortScans("width", width, 1000); - hasWidth = true; - } - if (!diffReflectivity.isEmpty()) { - differentialReflectivityGroups = sortScans("diffReflectivity", diffReflectivity, 1000); - hasDifferentialReflectivity = true; - } - - if (!time.isEmpty()) { - timeGroups = sortScans("diffReflectivity", diffReflectivity, 1000); - hasTime = true; + // using a universal structure that works for all data types + for (String var_name : all.keySet()) { + List additionalL = all.get(var_name); + if (!additionalL.isEmpty()) + allGroups.put(var_name, sortScans(var_name, additionalL, 1000)); } // --------- fill all of values in the ncfile ------ @@ -624,24 +717,13 @@ public int compare(List group1, List group2) { } } - public List> getTotalPowerGroups() { - return totalPowerGroups; - } - - public List> getVelocityGroups() { - return velocityGroups; - } - - public List> getWidthGroups() { - return widthGroups; - } - - public List> getReflectivityGroups() { - return reflectivityGroups; + // using a universal structure that works for all data types + public List> getGroup(String name) { + return allGroups.get(name); } - public List> getDifferentialReflectivityGroups() { - return differentialReflectivityGroups; + public short[] getDataTypes() { + return data_type; } public int[] getNumberGates() { From ec7cab2b777071ffa267bffe64749c27f63bd1d2 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 27 Jan 2024 11:30:04 +0100 Subject: [PATCH 02/16] cdm-radial: sigmet Code Style... :/ --- .../iosp/sigmet/SigmetIOServiceProvider.java | 16 +- .../nc2/iosp/sigmet/SigmetVolumeScan.java | 233 +++++++++--------- 2 files changed, 121 insertions(+), 128 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index 09f6d9d62c..66a0a906a8 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -121,8 +121,8 @@ public boolean isValidFile(RandomAccessFile raf) { /** * Open existing file, and populate ncfile with it. */ - public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - ucar.nc2.util.CancelTask cancelTask) throws IOException { + public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ucar.nc2.util.CancelTask cancelTask) + throws IOException { super.open(raf, ncfile, cancelTask); // java.util.Map recHdr=new java.util.HashMap(); Map hdrNames = new java.util.HashMap<>(); @@ -275,8 +275,7 @@ public Map readStnNames(RandomAccessFile raf) { * @param hdrNames java.util.Map with values for "StationName.." Attributes * @return ArrayList of Variables of ncfile */ - public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - Map hdrNames) { + public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, Map hdrNames) { // prepare attribute values String[] data_name = SigmetVolumeScan.data_name; String[] unit = SigmetVolumeScan.data_unit; @@ -782,7 +781,7 @@ public Array readData(Variable v2, Section section) throws IOException { if (posSweep > 0) groupName = groupName.substring(0, posSweep); if (groupName.startsWith(RAW_VARIABLE_PREFIX)) - groupName = groupName.substring(RAW_VARIABLE_PREFIX.length()); + groupName = groupName.substring(RAW_VARIABLE_PREFIX.length()); groups = volScan.getGroup(groupName); if (groups == null) @@ -871,8 +870,7 @@ public Array readFloatData(LayoutRegular index, Variable v2) throws IOException * @param channel WritableByteChannel object - channel that can write bytes. * @return the number of bytes written, possibly zero. */ - public long readToByteChannel11(Variable v2, Section section, WritableByteChannel channel) - throws IOException { + public long readToByteChannel11(Variable v2, Section section, WritableByteChannel channel) throws IOException { Array data = readData(v2, section); float[] ftdata = new float[(int) data.getSize()]; byte[] bytedata = new byte[(int) data.getSize() * 4]; @@ -1036,7 +1034,7 @@ static float calcData(Map recHdr, short dty, byte data) { // doing no rounding in this case as below for the other cases BigDecimal bd = new BigDecimal(temp); return bd.floatValue(); - //logger.warn("calcData: unimplemented 1 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); + // logger.warn("calcData: unimplemented 1 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); } BigDecimal bd = new BigDecimal(temp); // this should be reviewed, not sure why would you want a loss of precision here? @@ -1086,7 +1084,7 @@ static double calcData(Map recHdr, short dty, short data) { // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) // using only the raw value temp = data; - //logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); + // logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); break; } diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 7da489a57c..17d86de109 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -27,115 +27,109 @@ public class SigmetVolumeScan { * if suffix is "_2" it means a two byte value instead of just one byte * (2 bytes are experimental at this point - didn't have a file to test) */ - public static final String[] data_name = { - "ExtendedHeaders", // ? bytes - "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity", "[6]", "CorrectedReflectivity", // 1 byte - "TotalPower_2", "Reflectivity_2", "Velocity_2", "Width_2", "DifferentialReflectivity_2", // 2 bytes - "RainfallRate_2" /* 2 bytes */, "KDPDifferentialPhase" /* 1 byte */, "KDPDifferentialPhase_2" /* 2 bytes */, - "PhiDPDifferentialPhase" /* 1 byte */, "CorrectedVelocity" /* 1 byte */, "SQI" /* 1 byte */, - "RhoHV" /* 1 byte */, "RhoHV_2" /* 2 bytes */, - "CorrectedReflectivity_2" /* 2 bytes */, "CorrectedVelocity_2" /* 2 bytes */, "SQI_2" /* 2 bytes */, - "PhiDPDifferentialPhase_2" /* 2 bytes */, - "LDRH" /* 1 byte */, "LDRH_2" /* 2 bytes */, "LDRV" /* 1 byte */, "LDRV_2" /* 2 bytes */, - "[29]", "[30]", "[31]", - "Height" /* 1 byte */, "LinearLiquid_2" /* 2 bytes */, "RawData" /* ? */, - "WindShear" /* 1 byte */, "Divergence_2" /* 2 bytes */, "FloatedLiquid_2" /* 2 bytes */, - "UserType" /* 1 byte */, "UnspecifiedData" /* 1 byte */, "Deformation_2" /* 2 bytes */, - "VerticalVelocity_2" /* 2 bytes */, "HorizontalVelocity_2" /* 2 bytes */, - "HorizontalWindDirection_2" /* 2 bytes */, "AxisOfDilatation_2" /* 2 bytes */, "TimeInSeconds_2" /* 2 bytes */, - "RHOH" /* 1 byte */, "RHOH_2" /* 2 bytes */, "RHOV" /* 1 byte */, "RHOV_2" /* 2 bytes */, - "PHIH" /* 1 byte */, "PHIH_2" /* 2 bytes */, "PHIV" /* 1 byte */, "PHIV_2" /* 2 bytes */, - "UserType_2" /* 2 bytes */, "HydrometeorClass" /* 1 byte */, "HydrometeorClass_2" /* 2 bytes */, - "CorrectedDifferentialReflectivity" /* 1 byte */, "CorrectedDifferentialReflectivity_2" /* 2 bytes */, - // 16 empty - "[59]", "[60]", "[61]", "[62]", "[63]", "[64]", "[65]", "[66]", "[67]", "[68]", "[69]", "[70]", "[71]", "[72]", "[73]", "[74]", - "PolarimetricMeteoIndex" /* 1 byte */, "PolarimetricMeteoIndex_2" /* 2 bytes */, - "LOG8" /* 1 byte */, "LOG16_2" /* 2 bytes */, "CSP8" /* 1 byte */, "CSP16_2" /* 2 bytes */, - "CCOR8" /* 1 byte */, "CCOR16_2" /* 2 bytes */, "AH8" /* 1 byte */, "AH16_2" /* 2 bytes */, - "AV8" /* 1 byte */, "AV16_2" /* 2 bytes */, "AZDR8" /* 1 byte */, "AZDR16_2" /* 2 bytes */, - }; + public static final String[] data_name = {"ExtendedHeaders", // ? bytes + "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity", "[6]", "CorrectedReflectivity", // 1 + // byte + "TotalPower_2", "Reflectivity_2", "Velocity_2", "Width_2", "DifferentialReflectivity_2", // 2 bytes + "RainfallRate_2" /* 2 bytes */, "KDPDifferentialPhase" /* 1 byte */, "KDPDifferentialPhase_2" /* 2 bytes */, + "PhiDPDifferentialPhase" /* 1 byte */, "CorrectedVelocity" /* 1 byte */, "SQI" /* 1 byte */, "RhoHV" /* 1 byte */, + "RhoHV_2" /* 2 bytes */, "CorrectedReflectivity_2" /* 2 bytes */, "CorrectedVelocity_2" /* 2 bytes */, + "SQI_2" /* 2 bytes */, "PhiDPDifferentialPhase_2" /* 2 bytes */, "LDRH" /* 1 byte */, "LDRH_2" /* 2 bytes */, + "LDRV" /* 1 byte */, "LDRV_2" /* 2 bytes */, "[29]", "[30]", "[31]", "Height" /* 1 byte */, + "LinearLiquid_2" /* 2 bytes */, "RawData" /* ? */, "WindShear" /* 1 byte */, "Divergence_2" /* 2 bytes */, + "FloatedLiquid_2" /* 2 bytes */, "UserType" /* 1 byte */, "UnspecifiedData" /* 1 byte */, + "Deformation_2" /* 2 bytes */, "VerticalVelocity_2" /* 2 bytes */, "HorizontalVelocity_2" /* 2 bytes */, + "HorizontalWindDirection_2" /* 2 bytes */, "AxisOfDilatation_2" /* 2 bytes */, "TimeInSeconds_2" /* 2 bytes */, + "RHOH" /* 1 byte */, "RHOH_2" /* 2 bytes */, "RHOV" /* 1 byte */, "RHOV_2" /* 2 bytes */, "PHIH" /* 1 byte */, + "PHIH_2" /* 2 bytes */, "PHIV" /* 1 byte */, "PHIV_2" /* 2 bytes */, "UserType_2" /* 2 bytes */, + "HydrometeorClass" /* 1 byte */, "HydrometeorClass_2" /* 2 bytes */, + "CorrectedDifferentialReflectivity" /* 1 byte */, "CorrectedDifferentialReflectivity_2" /* 2 bytes */, + // 16 empty + "[59]", "[60]", "[61]", "[62]", "[63]", "[64]", "[65]", "[66]", "[67]", "[68]", "[69]", "[70]", "[71]", "[72]", + "[73]", "[74]", "PolarimetricMeteoIndex" /* 1 byte */, "PolarimetricMeteoIndex_2" /* 2 bytes */, + "LOG8" /* 1 byte */, "LOG16_2" /* 2 bytes */, "CSP8" /* 1 byte */, "CSP16_2" /* 2 bytes */, "CCOR8" /* 1 byte */, + "CCOR16_2" /* 2 bytes */, "AH8" /* 1 byte */, "AH16_2" /* 2 bytes */, "AV8" /* 1 byte */, "AV16_2" /* 2 bytes */, + "AZDR8" /* 1 byte */, "AZDR16_2" /* 2 bytes */,}; /* - Some units are unknown, some were correlated and assumed by ChatGPT. - Not all might be correct! - Just extending the initial list here from previous version of SigmetIOServiceProvider - // TODO fill and double check + * Some units are unknown, some were correlated and assumed by ChatGPT. + * Not all might be correct! + * Just extending the initial list here from previous version of SigmetIOServiceProvider + * // TODO fill and double check */ - public static final String[] data_unit = { - "?", // DB_XHDR: Extended Headers - "dBm", // DB_DBT: Total Power (1 byte) - "dBZ", // DB_DBZ: Reflectivity (1 byte) - "m/s", // DB_VEL: Velocity (1 byte) - "m/s", // DB_WIDTH: Width (1 byte) - "dB", // DB_ZDR: Differential Reflectivity (1 byte) - "?", // empty - "dBZ", // DB_DBZC: Corrected Reflectivity (1 byte) - "dBm", // DB_DBT2: Total Power (2 byte) - "dBZ", // DB_DBZ2: Reflectivity (2 byte) - "m/s", // DB_VEL2: Velocity (2 byte) - "m/s", // DB_WIDTH2: Width (2 byte) - "dB", // DB_ZDR2: Differential Reflectivity (2 byte) - "mm/hr", // DB_RAINRATE2: Rainfall Rate (2 byte) - "°/km", // DB_KDP: KDP (Differential Phase) (1 byte) - "°/km", // DB_KDP2: KDP (Differential Phase) (2 byte) - "°", // DB_PHIDP: PhiDP (Differential Phase) (1 byte) - "m/s", // DB_VELC: Corrected Velocity (1 byte) - "?", // DB_SQI: SQI (Signal Quality Index) (1 byte) - "?", // DB_RHOHV: RhoHV (1 byte) - "?", // DB_RHOHV2: RhoHV (2 byte) - "dBZ", // DB_DBZC2: Corrected Reflectivity (2 byte) - "m/s", // DB_VELC2: Corrected Velocity (2 byte) - "?", // DB_SQI2: SQI (Signal Quality Index) (2 byte) - "°", // DB_PHIDP2: PhiDP (Differential Phase) (2 byte) - "?", // DB_LDRH: LDR xmt H rcv V (1 byte) - "?", // DB_LDRH2: LDR xmt H rcv V (2 byte) - "?", // DB_LDRV: LDR xmt V rcv H (1 byte) - "?", // DB_LDRV2: LDR xmt V rcv H (2 byte) - // 3 empty - "?", "?", "?", - "1/10 km", // DB_HEIGHT: Height (1/10 km) (1 byte) - ".001mm", // DB_VIL2: Linear liquid (.001mm) (2 byte) - "?", // DB_RAW: Unknown unit or unitless (Raw Data) - "m/s", // DB_SHEAR: Shear (Velocity difference) - "?", // DB_DIVERGE2: Divergence (2 byte) - "mm", // DB_FLIQUID2: Liquid equivalent (2 byte) - "?", // DB_USER: User-defined (unit depends on definition) - "?", // DB_OTHER: Other data type (unit depends on definition) - "1/s", // DB_DEFORM2: Deformation (2 byte) - "m/s", // DB_VVEL2: Vertical Velocity (2 byte) - "m/s", // DB_HVEL2: Horizontal Velocity (2 byte) - "°", // DB_HDIR2: Horizontal Direction (2 byte) - "1/s", // DB_AXDIL2: Axis of Dilation (2 byte) - "s", // DB_TIME2: Time (2 byte) - "?", // DB_RHOH: RhoH (1 byte) - "?", // DB_RHOH2: RhoH (2 byte) - "?", // DB_RHOV: RhoV (1 byte) - "?", // DB_RHOV2: RhoV (2 byte) - "°", // DB_PHIH: PhiH (1 byte) - "°", // DB_PHIH2: PhiH (2 byte) - "°", // DB_PHIV: PhiV (1 byte) - "°", // DB_PHIV2: PhiV (2 byte) - "?", // DB_USER2: User-defined (2 byte) - "?", // DB_HCLASS: Hydrometeor Classification (1 byte) - "?", // DB_HCLASS2: Hydrometeor Classification (2 byte) - "dB", // DB_ZDRC: Corrected Differential Reflectivity (1 byte) - "dB", // DB_ZDRC2: Corrected Differential Reflectivity (2 byte) - // 16 empty - "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", - "?", // PolarimetricMeteoIndex (1 byte) - "?", // PolarimetricMeteoIndex2 (2 bytes) - "?", // LOG8 (1 byte) - "?", // LOG16 (2 bytes) - "?", // CSP8 (1 byte) - "?", // CSP16 (2 bytes) - "?", // CCOR8 (1 byte) - "?", // CCOR16 (2 bytes) - "?", // AH8 (1 byte) - "?", // AH16 (2 bytes) - "?", // AV8 (1 byte) - "?", // AV16 (2 bytes) - "?", // AZDR8 (1 byte) - "?", // AZDR16 (2 bytes) + public static final String[] data_unit = {"?", // DB_XHDR: Extended Headers + "dBm", // DB_DBT: Total Power (1 byte) + "dBZ", // DB_DBZ: Reflectivity (1 byte) + "m/s", // DB_VEL: Velocity (1 byte) + "m/s", // DB_WIDTH: Width (1 byte) + "dB", // DB_ZDR: Differential Reflectivity (1 byte) + "?", // empty + "dBZ", // DB_DBZC: Corrected Reflectivity (1 byte) + "dBm", // DB_DBT2: Total Power (2 byte) + "dBZ", // DB_DBZ2: Reflectivity (2 byte) + "m/s", // DB_VEL2: Velocity (2 byte) + "m/s", // DB_WIDTH2: Width (2 byte) + "dB", // DB_ZDR2: Differential Reflectivity (2 byte) + "mm/hr", // DB_RAINRATE2: Rainfall Rate (2 byte) + "°/km", // DB_KDP: KDP (Differential Phase) (1 byte) + "°/km", // DB_KDP2: KDP (Differential Phase) (2 byte) + "°", // DB_PHIDP: PhiDP (Differential Phase) (1 byte) + "m/s", // DB_VELC: Corrected Velocity (1 byte) + "?", // DB_SQI: SQI (Signal Quality Index) (1 byte) + "?", // DB_RHOHV: RhoHV (1 byte) + "?", // DB_RHOHV2: RhoHV (2 byte) + "dBZ", // DB_DBZC2: Corrected Reflectivity (2 byte) + "m/s", // DB_VELC2: Corrected Velocity (2 byte) + "?", // DB_SQI2: SQI (Signal Quality Index) (2 byte) + "°", // DB_PHIDP2: PhiDP (Differential Phase) (2 byte) + "?", // DB_LDRH: LDR xmt H rcv V (1 byte) + "?", // DB_LDRH2: LDR xmt H rcv V (2 byte) + "?", // DB_LDRV: LDR xmt V rcv H (1 byte) + "?", // DB_LDRV2: LDR xmt V rcv H (2 byte) + // 3 empty + "?", "?", "?", "1/10 km", // DB_HEIGHT: Height (1/10 km) (1 byte) + ".001mm", // DB_VIL2: Linear liquid (.001mm) (2 byte) + "?", // DB_RAW: Unknown unit or unitless (Raw Data) + "m/s", // DB_SHEAR: Shear (Velocity difference) + "?", // DB_DIVERGE2: Divergence (2 byte) + "mm", // DB_FLIQUID2: Liquid equivalent (2 byte) + "?", // DB_USER: User-defined (unit depends on definition) + "?", // DB_OTHER: Other data type (unit depends on definition) + "1/s", // DB_DEFORM2: Deformation (2 byte) + "m/s", // DB_VVEL2: Vertical Velocity (2 byte) + "m/s", // DB_HVEL2: Horizontal Velocity (2 byte) + "°", // DB_HDIR2: Horizontal Direction (2 byte) + "1/s", // DB_AXDIL2: Axis of Dilation (2 byte) + "s", // DB_TIME2: Time (2 byte) + "?", // DB_RHOH: RhoH (1 byte) + "?", // DB_RHOH2: RhoH (2 byte) + "?", // DB_RHOV: RhoV (1 byte) + "?", // DB_RHOV2: RhoV (2 byte) + "°", // DB_PHIH: PhiH (1 byte) + "°", // DB_PHIH2: PhiH (2 byte) + "°", // DB_PHIV: PhiV (1 byte) + "°", // DB_PHIV2: PhiV (2 byte) + "?", // DB_USER2: User-defined (2 byte) + "?", // DB_HCLASS: Hydrometeor Classification (1 byte) + "?", // DB_HCLASS2: Hydrometeor Classification (2 byte) + "dB", // DB_ZDRC: Corrected Differential Reflectivity (1 byte) + "dB", // DB_ZDRC2: Corrected Differential Reflectivity (2 byte) + // 16 empty + "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", // PolarimetricMeteoIndex (1 + // byte) + "?", // PolarimetricMeteoIndex2 (2 bytes) + "?", // LOG8 (1 byte) + "?", // LOG16 (2 bytes) + "?", // CSP8 (1 byte) + "?", // CSP16 (2 bytes) + "?", // CCOR8 (1 byte) + "?", // CCOR16 (2 bytes) + "?", // AH8 (1 byte) + "?", // AH16 (2 bytes) + "?", // AV8 (1 byte) + "?", // AV16 (2 bytes) + "?", // AZDR8 (1 byte) + "?", // AZDR16 (2 bytes) }; private HashMap>> allGroups = new HashMap<>(); @@ -248,16 +242,17 @@ public class SigmetVolumeScan { for (int i = 0; i < nparams; i++) { int idh_len = cur_len + 12 + i * 76; // + 12 == skipping over - /* debug structure_header - raf.seek(idh_len-12); - // Structure identifier - short si = raf.readShort(); - raf.readShort(); // format version - int nob = raf.readInt(); - if (si != (short)24) - throw new IllegalStateException("not a Ingest_header"); - raf.readShort(); // reserved - raf.readShort(); // flags + /* + * debug structure_header + * raf.seek(idh_len-12); + * // Structure identifier + * short si = raf.readShort(); + * raf.readShort(); // format version + * int nob = raf.readInt(); + * if (si != (short)24) + * throw new IllegalStateException("not a Ingest_header"); + * raf.readShort(); // reserved + * raf.readShort(); // flags */ raf.seek(idh_len); @@ -275,7 +270,7 @@ public class SigmetVolumeScan { num_rays_exp[i] = raf.readShort(); beg += num_rays_exp[i]; // before num_rays_act[i] was used but it does seem not work num_rays_act[i] = raf.readShort(); - //beg += num_rays_act[i]; // idh_len+20 + // beg += num_rays_act[i]; // idh_len+20 angl_swp[i] = raf.readShort(); // idh_len+22 // TODO maybe use in stead of variable twoBytes? bin_len[i] = raf.readShort(); // idh_len+24 @@ -455,7 +450,7 @@ public class SigmetVolumeScan { nb++; if (twoBytes) { cur_len++; - //i++;? + // i++;? } if (cur_len % REC_SIZE == 0) { @@ -526,7 +521,7 @@ public class SigmetVolumeScan { nb = nb + 1; if (twoBytes) { cur_len++; - //ii++;? + // ii++;? } if (cur_len % REC_SIZE == 0) { From 0c07e0b2ffe40274676c1d1d5a430517a8363bba Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Tue, 30 Jan 2024 12:08:06 +0100 Subject: [PATCH 03/16] cdm-radial: sigmet Another fix necessary to support IDEAM Colombia files In this case not all sweeps of num_sweeps have data, so adapting for this and handling it gracefully. Happens for the radars San_Andres and Corozal. --- .../iosp/sigmet/SigmetIOServiceProvider.java | 92 ++++++++++++------- 1 file changed, 61 insertions(+), 31 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index 66a0a906a8..aa3c31c4a2 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -617,11 +617,23 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ rgp = volScan.getGroup("Reflectivity"); if (rgp == null || rgp.isEmpty()) rgp = volScan.getGroup("Reflectivity_2"); + List[] sgp = new ArrayList[number_sweeps]; - for (int i = 0; i < number_sweeps; i++) { - sgp[i] = (List) rgp.get((short) i); - } + for (int sweepMinus1 = 0; sweepMinus1 < number_sweeps; sweepMinus1++) { + List found = null; + // in case some rays are missing/empty this is a safer way to find them + for (int i = 0; i < rgp.size() && found == null; i++) { + List rlist = (List) rgp.get(i); + if (rlist.size() > 0) { + Ray r = rlist.get(0); + if (r.getNsweep() == sweepMinus1 + 1) { + found = rlist; + } + } + } + sgp[sweepMinus1] = found; + } Variable[] time = new Variable[number_sweeps]; ArrayInt.D1[] timeArr = new ArrayInt.D1[number_sweeps]; @@ -641,13 +653,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ timeArr[i] = (ArrayInt.D1) Array.factory(DataType.INT, time[i].getShape()); timeIndex[i] = timeArr[i].getIndex(); List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays_actual; jj++) { - timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + timeArr[i].setInt(timeIndex[i].set(jj), -999); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); + } } } @@ -668,13 +685,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ azimArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, azimuthR[i].getShape()); azimIndex[i] = azimArr[i].getIndex(); List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays_actual; jj++) { - azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + azimArr[i].setFloat(azimIndex[i].set(jj), -999.99f); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); + } } } @@ -695,13 +717,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ elevArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, elevationR[i].getShape()); elevIndex[i] = elevArr[i].getIndex(); List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays_actual; jj++) { - elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + elevArr[i].setFloat(elevIndex[i].set(jj), -999.99f); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); + } } } @@ -719,13 +746,16 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ for (int i = 0; i < number_sweeps; i++) { List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - ngates = rtemp[0].getBins(); - gatesArr.setInt(gatesIndex.set(i), ngates); + if (rlist == null) { + gatesArr.setInt(gatesIndex.set(i), -999); + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + ngates = rtemp[0].getBins(); + gatesArr.setInt(gatesIndex.set(i), ngates); + } } for (int i = 0; i < number_sweeps; i++) { From e3b35f71bf7a84fde5b1fe86ec377fbbd57b1e10 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Fri, 2 Feb 2024 15:19:44 +0100 Subject: [PATCH 04/16] cdm-radial: sigmet Data types with more than 1 byte fully supported A few small fixes --- .../main/java/ucar/nc2/iosp/sigmet/Ray.java | 189 ++++++++++++------ .../iosp/sigmet/SigmetIOServiceProvider.java | 147 ++++++++++---- .../nc2/iosp/sigmet/SigmetVolumeScan.java | 159 ++++++++++----- 3 files changed, 354 insertions(+), 141 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index d06afe8511..29f789abef 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -8,7 +8,9 @@ import ucar.ma2.IndexIterator; import ucar.ma2.Range; import ucar.unidata.io.RandomAccessFile; + import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Formatter; /** @@ -16,20 +18,23 @@ * @since Apr 7, 2010 */ public class Ray { - private short bins; + + private short bins, bins_actual; int dataRead; int offset; int offset1; private float range, step, az, elev; - private short time; + // int because UINT2 data type (Unsigned 16-bit integer) + private int time; // private float[] val; String varName; int nsweep; short datatype; + int bytesPerBin; - public Ray(float range, float step, float az, float elev, short bins, short time, int offset, int dataRead, - int offset1, int nsweep, String name, short datatype) { + public Ray(float range, float step, float az, float elev, short bins, short bins_actual, int time, int offset, + int dataRead, int offset1, int nsweep, String name, short datatype, int bytesPerBin) { // this.val = new float[bins]; setRange(range); @@ -37,6 +42,7 @@ public Ray(float range, float step, float az, float elev, short bins, short time setAz(az); setElev(elev); setBins(bins); + setBinsActual(bins_actual); setTime(time); setOffset(offset); setDataRead(dataRead); @@ -45,7 +51,7 @@ public Ray(float range, float step, float az, float elev, short bins, short time setName(name); setNsweep(nsweep); setDataType(datatype); - + setBytesPerBin(bytesPerBin); } public short getDataType() { @@ -56,6 +62,14 @@ public void setDataType(short datatype) { this.datatype = datatype; } + public int getBytesPerBin() { + return bytesPerBin; + } + + public void setBytesPerBin(int bytesPerBin) { + this.bytesPerBin = bytesPerBin; + } + public float getRange() { return range; } @@ -108,11 +122,19 @@ public void setBins(short bins) { this.bins = bins; } - public short getTime() { + public short getBinsActual() { + return bins_actual; + } + + public void setBinsActual(short bins_actual) { + this.bins_actual = bins_actual; + } + + public int getTime() { return time; } - public void setTime(short time) { + public void setTime(int time) { this.time = time; } @@ -163,7 +185,8 @@ public boolean equals(Object o) { } else if (o instanceof Ray) { Ray oo = (Ray) o; - return (range == oo.range & step == oo.step & az == oo.az & elev == oo.elev & bins == oo.bins & time == oo.time); + return (range == oo.range & step == oo.step & az == oo.az & elev == oo.elev & bins == oo.bins + & bins_actual == oo.bins_actual & time == oo.time & bytesPerBin == oo.bytesPerBin); } else { return false; } @@ -171,7 +194,8 @@ public boolean equals(Object o) { public int hashCode() { return new Float(range).hashCode() + new Float(step).hashCode() + new Float(az).hashCode() - + new Float(elev).hashCode() + new Short(bins).hashCode() + new Short(time).hashCode(); + + new Float(elev).hashCode() + new Short(bins).hashCode() + new Short(bins_actual).hashCode() + + new Integer(time).hashCode() + new Integer(bytesPerBin).hashCode(); // val.hashCode(); } @@ -184,7 +208,7 @@ public String toString() { az = 360.0f + az; } - sb.format(" Az=%f Elev=%f Bins=%d Time=%d", az, elev, bins, time); + sb.format(" Az=%f Elev=%f Bins=%d (%d) Time=%d", az, elev, bins, bins_actual, time); // for (int i=0; i 1) + Byte[] dd = new Byte[bins * bytesPerBin]; int nb = 0; short a00; short dty = getDataType(); - String var_name = SigmetVolumeScan.data_name[dty]; - // support for 2 byte data types is experimental - boolean twoBytes = var_name.endsWith("_2"); + + int posInRay_absolute = 0; // raf.readFully(data); if (dataRead > 0) { raf.seek(offset); for (int i = 0; i < dataRead; i++) { - if (!twoBytes) { - byte d = raf.readByte(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - } else { - short d = raf.readShort(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - } - nb++; + dd[i] = raf.readByte(); + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; } } raf.seek(offset1); int cur_len = offset1; + // this only works for 1 additional record, not for more. Only a theoretical possibility for now. + // (relevant only for data types with a lot of bytes per bin) while (nb < (int) bins) { // --- Check if the code=1 ("1" means an end of a ray) a00 = raf.readShort(); cur_len = cur_len + 2; + // end of ray if (a00 == (short) 1) { - for (int uk = 0; uk < (int) bins; uk++) { - dd[uk] = -999.99f; - } + // don't need to do anything as the rest of dd is already null break; } if (a00 < 0) { // -- This is data int nwords = a00 & 0x7fff; int dataRead1 = nwords * 2; - int pos = 0; - if (cur_len % REC_SIZE == 0) { + boolean breakOuterLoop = false; + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { break; } raf.seek(cur_len); for (int i = 0; i < dataRead1; i++) { - if (!twoBytes) { - byte d = raf.readByte(); - dd[nb] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - cur_len = cur_len + 1; - } else { - short d = raf.readShort(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - cur_len = cur_len + 2; - } - nb = nb + 1; + dd[posInRay_absolute] = raf.readByte(); + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; + + cur_len++; - if (nb % REC_SIZE == 0) { - pos = i + 1; + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { + breakOuterLoop = true; break; } } - - if (pos > 0) { + if (breakOuterLoop) { break; } + } else if (a00 > 0 & a00 != 1) { int num_zero = a00 * 2; int dataRead1 = num_zero; for (int k = 0; k < dataRead1; k++) { - if (!twoBytes) - dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (byte) 0); - else - dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (short) 0); + dd[posInRay_absolute] = 0; + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; } - nb = nb + dataRead1; - if (cur_len % REC_SIZE == 0) { + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { break; } - } + } // ------ end of while for num_bins--------------------------------- - if (!twoBytes) { + // only supporting float, double, byte and byte[]/Object for now + Class dtyClass = SigmetIOServiceProvider.calcDataClass(dty, bytesPerBin); + + if (dtyClass == Float.TYPE) { for (int gateIdx : gateRange) { if (gateIdx >= bins) ii.setFloatNext(Float.NaN); - else - ii.setFloatNext(dd[gateIdx].floatValue()); + else if (gateIdx >= bins_actual) + ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); + else + ii.setFloatNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, ddx)); + } } - } else { + + } else if (dtyClass == Double.TYPE) { for (int gateIdx : gateRange) { if (gateIdx >= bins) ii.setDoubleNext(Double.NaN); - else - ii.setDoubleNext(dd[gateIdx].doubleValue()); + else if (gateIdx >= bins_actual) + ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); + else { + int ddx2 = dd[2]; + int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); + ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); + } + } + } + + } else if (dtyClass == Byte.TYPE) { + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else if (gateIdx >= bins_actual) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else + ii.setByteNext(ddx); + } } - } + } else { // byte[]/Object + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else if (gateIdx >= bins_actual) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else { + byte[] b = new byte[bytesPerBin]; + for (int i = 0; i < bytesPerBin; i++) { + ddx = dd[offset + i]; + b[i] = ddx == null ? SigmetVolumeScan.MISSING_VALUE_BYTE : ddx; + } + ii.setObjectNext(ByteBuffer.wrap(b)); + } + } + } + } } // end of readData } // class Ray end------------------------------------------ diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index aa3c31c4a2..b3a07ca8ce 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -352,18 +352,33 @@ public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile var_name_original = RAW_VARIABLE_PREFIX + var_name_original; var_name = var_name_original; - // support for 2 byte data types is experimental - boolean twoBytes = var_name.endsWith("_2"); + int bytesPerBin = 1; + List> allRays = volScan.getGroup(data_name[dty]); + if (allRays.size() > 0 && allRays.get(0).size() > 0) + bytesPerBin = allRays.get(0).get(0).getBytesPerBin(); + + Class dtyClass = SigmetIOServiceProvider.calcDataClass(dty, bytesPerBin); for (int jj = 0; jj < number_sweeps; jj++) { if (number_sweeps > 1) { var_name = var_name_original + "_sweep_" + (jj + 1); } v[j][jj] = new Variable(ncfile, null, null, var_name); - if (twoBytes) - v[j][jj].setDataType(DataType.DOUBLE); - else + if (dtyClass == Float.TYPE) { v[j][jj].setDataType(DataType.FLOAT); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_FLOAT)); + } else if (dtyClass == Double.TYPE) { + v[j][jj].setDataType(DataType.DOUBLE); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_DOUBLE)); + } else if (dtyClass == Byte.TYPE) { + v[j][jj].setDataType(DataType.BYTE); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_BYTE)); + } else { // byte[]/Object + v[j][jj].setDataType(DataType.OPAQUE); + Attribute a = new Attribute(CDM.MISSING_VALUE, DataType.OPAQUE); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY)); + } + dims2.add(radial); dims2.add(gateR[jj]); v[j][jj].setDimensions(dims2); @@ -371,7 +386,6 @@ public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[dty])); String coordinates = "time elevationR azimuthR distanceR"; v[j][jj].addAttribute(new Attribute(_Coordinate.Axes, coordinates)); - v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, -999.99f)); ncfile.addVariable(null, v[j][jj]); varList.add(v[j][jj]); dims2.clear(); @@ -536,7 +550,6 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ ArrayList varList, Map recHdr) { // prepare attribute values - String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; String def_datafile = "SIGMET-IRIS"; Short header_length = 80; Short ray_header_length = 6; @@ -837,23 +850,49 @@ public Array readData(Variable v2, Section section) throws IOException { private void readOneScan(List mapScan, Range radialRange, Range gateRange, IndexIterator ii) throws IOException { int siz = mapScan.size(); + short dataType = 1; // TotalPower as fallback + int bytesPerBin = 1; + for (int radialIdx : radialRange) { if (radialIdx >= siz) - readOneRadial(null, gateRange, ii); + readOneRadialMissing(dataType, bytesPerBin, gateRange, ii); else { Ray r = mapScan.get(radialIdx); + dataType = r.datatype; + bytesPerBin = r.bytesPerBin; readOneRadial(r, gateRange, ii); } } } private void readOneRadial(Ray r, Range gateRange, IndexIterator ii) throws IOException { - if (r == null || r.offset == -999) { - for (int i = 0; i < gateRange.length(); i++) + r.readData(volScan.raf, gateRange, ii); + } + + private void readOneRadialMissing(short datatype, int bytesPerBin, Range gateRange, IndexIterator ii) + throws IOException { + Class dtyClass = calcDataClass(datatype, bytesPerBin); + + if (dtyClass == Float.TYPE) { + for (int i = 0; i < gateRange.length(); i++) { ii.setFloatNext(Float.NaN); - return; + } + + } else if (dtyClass == Double.TYPE) { + for (int i = 0; i < gateRange.length(); i++) { + ii.setDoubleNext(Double.NaN); + } + + } else if (dtyClass == Byte.TYPE) { + for (int i = 0; i < gateRange.length(); i++) { + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + } + + } else { // byte[]/Object + for (int i = 0; i < gateRange.length(); i++) { + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + } } - r.readData(volScan.raf, gateRange, ii); } /** @@ -1019,12 +1058,44 @@ static float calcAz(short az0, short az1) { return result.floatValue(); } + /** + * Calculate Object class for data type + * + * @see Ray + * + * @param dty type of data (@see SigmetVolumeScan.data_name) + * @return float value with precision of two decimal + */ + static Class calcDataClass(short dty, int bytes) { + switch (dty) { + case 1:// dty=1,2 -total_power, reflectivity (dBZ) + case 2: + case 3: // dty=3 - mean velocity (m/sec) + case 4: // dty=4 - spectrum width (m/sec) + case 5: // dty=5 - differential reflectivity (dB) + return Float.TYPE; + + case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) + case 8: + case 9: // dty=9 - mean velocity 2 (m/sec) + case 10: // dty=10 - spectrum width 2 (m/sec) + case 11: // dty=11 - differential reflectivity 2 (dB) + return Double.TYPE; + + default: + // TODO implement for more SigmetVolumeScan.data_name (see chapter 4.4) + if (bytes == 1) + return Byte.TYPE; + + return byte[].class; + } + } + /** * Calculate data values from raw ingest data * * @param recHdr java.util.Map object with values for calculation - * @param dty type of data ( "Total_Power", "Reflectivity", "Velocity", - * "Width", "Differential_Reflectivity") + * @param dty type of data (@see SigmetVolumeScan.data_name) * @param data 1-byte input value * @return float value with precision of two decimal */ @@ -1032,32 +1103,38 @@ static float calcData(Map recHdr, short dty, byte data) { short[] coef = {1, 2, 3, 4}; // MultiPRF modes short multiprf = recHdr.get("multiprf").shortValue(); float vNyq = recHdr.get("vNyq").floatValue(); - double temp = -999.99; + double temp = SigmetVolumeScan.MISSING_VALUE_FLOAT; + + // see chapter 4.4 switch (dty) { case 1:// dty=1,2 -total_power, reflectivity (dBZ) case 2: - if (data != 0) { + if (data != 0 && data != (byte) 255) { temp = (((int) data & 0xFF) - 64) * 0.5; } break; case 3: // dty=3 - mean velocity (m/sec) if (data != 0) { + // seems incorrect according to "4.4.44 1-byte Velocity Format (DB_VEL)" + // TODO needs more research temp = ((((int) data & 0xFF) - 128) / 127.0) * vNyq * coef[multiprf]; } break; case 4: // dty=4 - spectrum width (m/sec) - if (data != 0) { + if (data != 0 && data != (byte) 255) { + // seems incorrect according to "4.4.48 1-byte Width Format (DB_WIDTH)" + // TODO needs more research double v = ((((int) data & 0xFF) - 128) / 127.0) * vNyq * coef[multiprf]; temp = (((int) data & 0xFF) / 256.0) * v; } break; case 5: // dty=5 - differential reflectivity (dB) - if (data != 0) { + if (data != 0 && data != (byte) 255) { temp = ((((int) data & 0xFF) - 128) / 16.0); } break; default: - // TODO implement for more SigmetVolumeScan.data_name (only 1 byte) + // TODO implement for more SigmetVolumeScan.data_name (only 1 byte) (see chapter 4.4) // using only the raw value temp = (int) data & 0xFF; @@ -1076,42 +1153,38 @@ static float calcData(Map recHdr, short dty, byte data) { * Calculate data values from raw ingest data * * @param recHdr java.util.Map object with values for calculation - * @param dty type of data ( "Total_Power", "Reflectivity", "Velocity", - * "Width", "Differential_Reflectivity") - * @param data 2-byte input value + * @param dty type of data (@see SigmetVolumeScan.data_name) + * @param data 2-byte input value (unsigned short) * @return double value */ - static double calcData(Map recHdr, short dty, short data) { - short[] coef = {1, 2, 3, 4}; // MultiPRF modes - short multiprf = recHdr.get("multiprf").shortValue(); - float vNyq = recHdr.get("vNyq").floatValue(); - double temp = -999.99; + static double calcData(Map recHdr, short dty, int data) { + double temp = SigmetVolumeScan.MISSING_VALUE_DOUBLE; + // see chapter 4.4 switch (dty) { case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) case 8: - if (data != 0) { - temp = (data - 64) * 0.5; + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; } break; case 9: // dty=9 - mean velocity 2 (m/sec) - if (data != 0) { - temp = ((data - 128) / 127.0) * vNyq * coef[multiprf]; + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; } break; case 10: // dty=10 - spectrum width 2 (m/sec) - if (data != 0) { - double v = ((data - 128) / 127.0) * vNyq * coef[multiprf]; - temp = (data / 256.0) * v; + if (data != 0 && data != 65535) { + temp = data / 100d; } break; case 11: // dty=11 - differential reflectivity 2 (dB) - if (data != 0) { - temp = ((data - 128) / 16.0); + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; } break; default: - // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) + // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) (see chapter 4.4) // using only the raw value temp = data; // logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 17d86de109..8437b1e629 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -10,9 +10,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.DataType; import ucar.nc2.Variable; import ucar.unidata.io.RandomAccessFile; +import java.nio.ByteBuffer; import java.util.*; /** @@ -132,6 +135,16 @@ public class SigmetVolumeScan { "?", // AZDR16 (2 bytes) }; + public static final int REC_SIZE = 6144; + + public static final float MISSING_VALUE_FLOAT = -999.99f; + public static final double MISSING_VALUE_DOUBLE = -999.99; + public static final byte MISSING_VALUE_BYTE = 0; + + public static final ByteBuffer MISSING_VALUE_BYTE_ARRAY_BB = ByteBuffer.wrap(new byte[] {MISSING_VALUE_BYTE}); + public static final Array MISSING_VALUE_BYTE_ARRAY = + Array.factoryConstant(DataType.OPAQUE, new int[] {1}, new ByteBuffer[] {MISSING_VALUE_BYTE_ARRAY_BB}); + private HashMap>> allGroups = new HashMap<>(); private short[] data_type; @@ -153,12 +166,14 @@ public class SigmetVolumeScan { */ SigmetVolumeScan(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) throws java.io.IOException { - int REC_SIZE = 6144; int len = 12288; // ---- Read from the 3d record----------- 6144*2=12288 short nrec = 0, nsweep = 1, nray = 0, byteoff = 0; - int nwords, end_words, data_read = 0, num_zero, rays_count = 0, nb = 0, pos = 0, pos_ray_hdr = 0, t = 0; + int nwords, end_words, data_read = 0, num_zero, rays_count = 0, nb = 0, posInRay_relative = 0, + posInRay_absolute = 0, pos_ray_hdr = 0, t = 0; short a0, a00, dty; - short beg_az = 0, beg_elev = 0, end_az = 0, end_elev = 0, num_bins = 0, time_start_sw = 0; + short beg_az = 0, beg_elev = 0, end_az = 0, end_elev = 0, num_bins = 0; + // int because UINT2 data type (Unsigned 16-bit integer) + int time_start_sw = 0; float az, elev, d = 0.0f, step; // byte data = 0; boolean beg_rec = true, end_rec = true, read_ray_hdr = true, begin = true; @@ -270,10 +285,8 @@ public class SigmetVolumeScan { num_rays_exp[i] = raf.readShort(); beg += num_rays_exp[i]; // before num_rays_act[i] was used but it does seem not work num_rays_act[i] = raf.readShort(); - // beg += num_rays_act[i]; // idh_len+20 angl_swp[i] = raf.readShort(); // idh_len+22 - // TODO maybe use in stead of variable twoBytes? - bin_len[i] = raf.readShort(); // idh_len+24 + bin_len[i] = raf.readShort(); // idh_len+24 (Number of bits per bin for this data type) data_type[i] = raf.readShort(); // idh_len+26 } @@ -294,7 +307,8 @@ public class SigmetVolumeScan { end_rec = true; rays_count++; read_ray_hdr = true; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; len = cur_len; @@ -308,7 +322,7 @@ public class SigmetVolumeScan { } nwords = a0 & 0x7fff; - end_words = nwords - 6; + end_words = nwords - 6; // because of raw_prod_bhdr 12-byte structure data_read = end_words * 2; end_rec = false; @@ -325,19 +339,27 @@ public class SigmetVolumeScan { // ---Define output data files for each data_type (= nparams)/sweep --------- dty = data_type[0]; + int bitsToRead = bin_len[0]; if (nparams > 1) { kk = rays_count % nparams; dty = data_type[kk]; + bitsToRead = bin_len[kk]; + } else if (number_sweeps > 1) { } + if (bitsToRead <= 0 || bitsToRead % 8 != 0) + throw new IllegalStateException("Not compatible! Number of bits per bin for this data type = " + bitsToRead); + int bytesToRead = bitsToRead / 8; + String var_name = data_name[dty]; - // support for 2 byte data types is experimental - boolean twoBytes = var_name.endsWith("_2"); // --- read ray_header (size=12 bytes=6 words)--------------------------------------- if (read_ray_hdr) { + if (ray != null) + throw new IllegalStateException("ray != null"); + if (pos_ray_hdr < 2) { raf.seek(cur_len); beg_az = raf.readShort(); @@ -414,7 +436,7 @@ public class SigmetVolumeScan { if (pos_ray_hdr < 12) { raf.seek(cur_len); - time_start_sw = raf.readShort(); + time_start_sw = raf.readUnsignedShort(); cur_len = cur_len + 2; len = cur_len; } @@ -433,9 +455,9 @@ public class SigmetVolumeScan { continue; } - if (pos > 0) { - data_read = data_read - pos; - pos = 0; + if (posInRay_relative > 0) { + data_read = data_read - posInRay_relative; + posInRay_relative = 0; } if (data_read > 0) { @@ -447,14 +469,12 @@ public class SigmetVolumeScan { // data = raf.readByte(); // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); cur_len++; - nb++; - if (twoBytes) { - cur_len++; - // i++;? - } + posInRay_absolute++; + if (posInRay_absolute % bytesToRead == 0) + nb++; if (cur_len % REC_SIZE == 0) { - pos = i + 1; + posInRay_relative = i + 1; beg_rec = true; read_ray_hdr = false; len = cur_len; @@ -463,13 +483,13 @@ public class SigmetVolumeScan { } } raf.seek(cur_len); - if (pos > 0) { + if (posInRay_relative > 0) { continue; } } if (cur_len % REC_SIZE == 0) { - pos = 0; + posInRay_relative = 0; beg_rec = true; read_ray_hdr = false; data_read = 0; @@ -487,11 +507,8 @@ public class SigmetVolumeScan { // --- Check if the code=1 ("1" means an end of a ray) if (a00 == (short) 1) { - // for (int uk = 0; uk < (int) num_bins; uk++) { - // dd[uk] = -999.99f; - // } - ray = new Ray(-999.99f, -999.99f, -999.99f, -999.99f, num_bins, (short) (-99), -999, 0, -999, nsweep, - var_name, dty); + ray = new Ray(range_first, step, az, elev, num_bins, (short) nb, time_start_sw, rayoffset, datalen, + rayoffset1, nsweep, var_name, dty, bytesToRead); rays_count++; beg_rec = false; end_rec = true; @@ -504,7 +521,7 @@ public class SigmetVolumeScan { data_read = nwords * 2; if (cur_len % REC_SIZE == 0) { - pos = 0; + posInRay_relative = 0; beg_rec = true; end_rec = false; read_ray_hdr = false; @@ -517,15 +534,13 @@ public class SigmetVolumeScan { for (int ii = 0; ii < data_read; ii++) { // data = raf.readByte(); // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); - cur_len = cur_len + 1; - nb = nb + 1; - if (twoBytes) { - cur_len++; - // ii++;? - } + cur_len++; + posInRay_absolute++; + if (posInRay_absolute % bytesToRead == 0) + nb++; if (cur_len % REC_SIZE == 0) { - pos = ii + 1; + posInRay_relative = ii + 1; beg_rec = true; end_rec = false; read_ray_hdr = false; @@ -534,7 +549,7 @@ public class SigmetVolumeScan { } } raf.seek(cur_len); - if (pos > 0) { + if (posInRay_relative > 0) { break; } } else if (a00 > 0 & a00 != 1) { @@ -544,14 +559,19 @@ public class SigmetVolumeScan { // dd[nb + k] = SigmetIOServiceProvider.calcData(recHdr, dty, (byte) 0); // } - nb = nb + num_zero; - // TODO extra handling for twoBytes here, too? + int nb_before = posInRay_absolute / bytesToRead; + // sanity check + if (nb_before != nb) + throw new IllegalStateException("nb_before != nb"); + posInRay_absolute += num_zero; + int nb_after = posInRay_absolute / bytesToRead; + nb = nb + (nb_after - nb_before); if (cur_len % REC_SIZE == 0) { beg_rec = true; end_rec = false; read_ray_hdr = false; - pos = 0; + posInRay_relative = 0; data_read = 0; break; @@ -562,20 +582,52 @@ public class SigmetVolumeScan { if (cur_len % REC_SIZE == 0) { len = cur_len; + // will get lost otherwise + if (ray != null) { + beg_rec = true; + end_rec = true; + read_ray_hdr = true; + posInRay_relative = 0; + posInRay_absolute = 0; + data_read = 0; + nb = 0; + len = cur_len; + + if (ray == null) + throw new IllegalStateException("ray == null"); + + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); + } + varL.add(ray); + lastRay = ray; + ray = null; + } + continue; } raf.seek(cur_len); if (nb == (int) num_bins) { - a00 = raf.readShort(); + a00 = raf.readShort(); // should be 1 == end of ray + if (a00 != 1) + // should not be able to get here + logger.warn("nb == num_bins but a00 != 1 - something seems wrong"); cur_len = cur_len + 2; end_rec = true; - ray = new Ray(range_first, step, az, elev, num_bins, time_start_sw, rayoffset, datalen, rayoffset1, nsweep, - var_name, dty); + ray = new Ray(range_first, step, az, elev, num_bins, (short) nb, time_start_sw, rayoffset, datalen, rayoffset1, + nsweep, var_name, dty, bytesToRead); rays_count++; + // last ray of last sweep -> end of file if ((nsweep == number_sweeps) & (rays_count % beg == 0)) { + if (ray == null) + throw new IllegalStateException("ray == null"); + // using a universal structure that works for all data types List varL = all.get(var_name.trim()); if (varL == null) { @@ -583,6 +635,8 @@ public class SigmetVolumeScan { all.put(var_name.trim(), varL); } varL.add(ray); + lastRay = ray; + ray = null; break; } @@ -590,11 +644,15 @@ public class SigmetVolumeScan { beg_rec = true; end_rec = true; read_ray_hdr = true; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; len = cur_len; + if (ray == null) + throw new IllegalStateException("ray == null"); + // using a universal structure that works for all data types List varL = all.get(var_name.trim()); if (varL == null) { @@ -602,6 +660,8 @@ public class SigmetVolumeScan { all.put(var_name.trim(), varL); } varL.add(ray); + lastRay = ray; + ray = null; continue; } @@ -610,6 +670,9 @@ public class SigmetVolumeScan { if (firstRay == null) firstRay = ray; + if (ray == null) + throw new IllegalStateException("ray == null"); + // using a universal structure that works for all data types List additionalL = all.get(var_name.trim()); if (additionalL == null) { @@ -617,8 +680,11 @@ public class SigmetVolumeScan { all.put(var_name.trim(), additionalL); } additionalL.add(ray); + lastRay = ray; + ray = null; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; read_ray_hdr = true; @@ -635,7 +701,6 @@ public class SigmetVolumeScan { len = cur_len; } // ------------end of outer while --------------- - lastRay = ray; // using a universal structure that works for all data types for (String var_name : all.keySet()) { @@ -737,7 +802,7 @@ public int[] getStartSweep() { */ void checkSort(Ray[] r) { int j = 0, n = 0, n1, n2; - short time1, time2; + int time1, time2; int[] k1 = new int[300]; int[] k2 = new int[300]; // define the groups of rays with the same "time". For ex.: @@ -753,7 +818,7 @@ void checkSort(Ray[] r) { k1[j] = i + 1; } } - if (k2[j] < r.length - 1) { + if (k2[j] < r.length - 1 && j > 0) { k1[j] = k2[j - 1] + 1; k2[j] = r.length - 1; } From e3a19e10ca8d76704df33f181f2af18db4f523ef Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 3 Feb 2024 11:08:04 +0100 Subject: [PATCH 05/16] cdm-radial: sigmet Fix for 2 byte data types (was able to find an example file finally) --- cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index 29f789abef..099c2488a0 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -332,7 +332,7 @@ else if (gateIdx >= bins_actual) if (ddx == null) ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); else { - int ddx2 = dd[2]; + int ddx2 = dd[offset+1]; int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); } From 2d8921617ab63d16d304c2f0f1dbac0de1ba7913 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 3 Feb 2024 11:13:24 +0100 Subject: [PATCH 06/16] code style... :/ --- cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index 099c2488a0..41eb584fca 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -332,7 +332,7 @@ else if (gateIdx >= bins_actual) if (ddx == null) ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); else { - int ddx2 = dd[offset+1]; + int ddx2 = dd[offset + 1]; int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); } From d597a4eb3f027b32cfa7629f73e7d49ac90144eb Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sun, 4 Feb 2024 12:25:30 +0100 Subject: [PATCH 07/16] cdm-radial: sigmet Fix for reading data from ray for only the number of bins available --- .../src/main/java/ucar/nc2/iosp/sigmet/Ray.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index 41eb584fca..be6185b9e0 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -268,7 +268,7 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th break; } raf.seek(cur_len); - for (int i = 0; i < dataRead1; i++) { + for (int i = 0; i < dataRead1 && nb < bins; i++) { dd[posInRay_absolute] = raf.readByte(); posInRay_absolute++; if (posInRay_absolute % bytesPerBin == 0) @@ -288,14 +288,15 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th } else if (a00 > 0 & a00 != 1) { int num_zero = a00 * 2; int dataRead1 = num_zero; - for (int k = 0; k < dataRead1; k++) { + for (int k = 0; k < dataRead1 && nb < bins; k++) { dd[posInRay_absolute] = 0; posInRay_absolute++; if (posInRay_absolute % bytesPerBin == 0) nb++; - } - if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { - break; + + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { + break; + } } } From 330e2725d6fb9369873b461b635f1c475e3dff34 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 27 Jan 2024 11:10:13 +0100 Subject: [PATCH 08/16] cdm-radial: sigmet Support for all data types instead of just 5 Universal handling for data types instead of each one separately Support for data types spanning two bytes instead of one is experimental at this point as I didn't have such a file to test Most new data types are returned as raw values meaning no normalization like done for total power, reflectivity, velocity, width, differential reflectivity Added comments and small fixes Ultimately all these changes add support for IDEAM Colombia files Addresses the github issue Unidata/netcdf-java#1292 --- .../main/java/ucar/nc2/iosp/sigmet/Ray.java | 56 +++- .../iosp/sigmet/SigmetIOServiceProvider.java | 256 +++++++++++---- .../nc2/iosp/sigmet/SigmetVolumeScan.java | 292 +++++++++++------- 3 files changed, 418 insertions(+), 186 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index f6e557231b..d06afe8511 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -201,18 +201,27 @@ public String toString() { public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) throws IOException { int REC_SIZE = 6144; raf.seek(offset); - byte[] data = new byte[bins]; - float[] dd = new float[bins]; - byte d; + Number[] dd = new Number[bins]; int nb = 0; short a00; + + short dty = getDataType(); + String var_name = SigmetVolumeScan.data_name[dty]; + // support for 2 byte data types is experimental + boolean twoBytes = var_name.endsWith("_2"); + // raf.readFully(data); if (dataRead > 0) { raf.seek(offset); for (int i = 0; i < dataRead; i++) { - d = raf.readByte(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), d); + if (!twoBytes) { + byte d = raf.readByte(); + dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + } else { + short d = raf.readShort(); + dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + } nb++; } } @@ -240,10 +249,17 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th } raf.seek(cur_len); for (int i = 0; i < dataRead1; i++) { - d = raf.readByte(); - dd[nb] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), d); + if (!twoBytes) { + byte d = raf.readByte(); + dd[nb] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + cur_len = cur_len + 1; + } else { + short d = raf.readShort(); + dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); + cur_len = cur_len + 2; + } nb = nb + 1; - cur_len = cur_len + 1; + if (nb % REC_SIZE == 0) { pos = i + 1; break; @@ -257,7 +273,10 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th int num_zero = a00 * 2; int dataRead1 = num_zero; for (int k = 0; k < dataRead1; k++) { - dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), (byte) 0); + if (!twoBytes) + dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (byte) 0); + else + dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (short) 0); } nb = nb + dataRead1; if (cur_len % REC_SIZE == 0) { @@ -267,11 +286,20 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th } } // ------ end of while for num_bins--------------------------------- - for (int gateIdx : gateRange) { - if (gateIdx >= bins) - ii.setFloatNext(Float.NaN); - else - ii.setFloatNext(dd[gateIdx]); + if (!twoBytes) { + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setFloatNext(Float.NaN); + else + ii.setFloatNext(dd[gateIdx].floatValue()); + } + } else { + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setDoubleNext(Double.NaN); + else + ii.setDoubleNext(dd[gateIdx].doubleValue()); + } } } // end of readData diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index 39599c325d..09f6d9d62c 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -5,25 +5,9 @@ package ucar.nc2.iosp.sigmet; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.ma2.Range; -import ucar.ma2.Array; -import ucar.ma2.ArrayFloat; -import ucar.ma2.ArrayInt; -import ucar.ma2.DataType; -import ucar.ma2.Index; -import ucar.ma2.IndexIterator; -import ucar.ma2.InvalidRangeException; -import ucar.ma2.Section; +import ucar.ma2.*; import ucar.nc2.Attribute; import ucar.nc2.Dimension; import ucar.nc2.Variable; @@ -35,6 +19,15 @@ import ucar.nc2.iosp.LayoutRegular; import ucar.unidata.io.RandomAccessFile; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + /** * Implementation of the ServerProvider pattern provides input/output * of the SIGMET-IRIS dataset. IOSP are managed by the NetcdfFile class. @@ -78,12 +71,16 @@ public class SigmetIOServiceProvider extends AbstractIOServiceProvider { private static Logger logger = LoggerFactory.getLogger(SigmetIOServiceProvider.class); + + // meaning unprocessed + public static String RAW_VARIABLE_PREFIX = "raw_"; + private ArrayList varList; private int[] tsu_sec; private int[] sweep_bins; private String date0; - public static java.util.Map recHdr = new java.util.HashMap<>(); + public static Map recHdr = new java.util.HashMap<>(); private SigmetVolumeScan volScan; public String getFileTypeDescription() { @@ -101,7 +98,7 @@ public String getFileTypeId() { /** * Check if this is a valid SIGMET-IRIS file for this IOServiceProvider. */ - public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) { + public boolean isValidFile(RandomAccessFile raf) { try { raf.order(RandomAccessFile.LITTLE_ENDIAN); // The first struct in the file is the product_hdr, which will have the @@ -124,11 +121,11 @@ public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) { /** * Open existing file, and populate ncfile with it. */ - public void open(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - ucar.nc2.util.CancelTask cancelTask) throws java.io.IOException { + public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, + ucar.nc2.util.CancelTask cancelTask) throws IOException { super.open(raf, ncfile, cancelTask); // java.util.Map recHdr=new java.util.HashMap(); - java.util.Map hdrNames = new java.util.HashMap<>(); + Map hdrNames = new java.util.HashMap<>(); volScan = new SigmetVolumeScan(raf, ncfile, varList); this.varList = init(raf, ncfile, hdrNames); @@ -141,8 +138,8 @@ public void open(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfil * Read some global data from SIGMET file. The SIGMET file consists of records with * fixed length=6144 bytes. */ - public static java.util.Map readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) { - java.util.Map recHdr1 = new java.util.HashMap<>(); + public static Map readRecordsHdr(RandomAccessFile raf) { + Map recHdr1 = new java.util.HashMap<>(); try { int nparams = 0; // -- Read from of the 1st record -- 12+320+120 @@ -164,17 +161,51 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando short num_rays = raf.readShort(); // 6340 raf.skipBytes(2); int radar_alt = raf.readInt(); // 6344 - raf.seek(6648); - int time_beg = raf.readInt(); + // end of ingest_config would be 6636 + + // next is 2612 Bytes + raf.seek(6648); // + task_configuration 12 + // inner 120 Bytes + int time_beg = raf.readInt(); // Start time (seconds within a day) raf.seek(6652); - int time_end = raf.readInt(); - raf.seek(6772); + int time_end = raf.readInt(); // Stop time (seconds within a day) + // end inner would be 6768 + + // next inner is 320 Bytes. + raf.seek(6772); // + 2 Major mode + 2 DSP type + + // 4 - 24 - Current Data type mask + // Mask word 0 int data_mask = raf.readInt(); for (int j = 0; j < 32; j++) { nparams += ((data_mask >> j) & (0x1)); } + raf.readInt(); // Extended header type + // Mask word 1 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 2 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 3 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 4 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + raf.seek(6912); short multiprf = raf.readShort(); + // end inner would be 7088 + raf.seek(7408); int range_first = raf.readInt(); // cm 7408 int range_last = raf.readInt(); // cm 7412 @@ -186,6 +217,9 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando int step = raf.readInt(); // cm 7424 raf.seek(7574); short number_sweeps = raf.readShort(); // 7574 + // end of task_configuration would be 9248 + + raf.seek(12312); int base_time = raf.readInt(); // 3d rec raf.skipBytes(2); @@ -218,8 +252,8 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando /** * Read StationName strings */ - public java.util.Map readStnNames(ucar.unidata.io.RandomAccessFile raf) { - java.util.Map hdrNames = new java.util.HashMap<>(); + public Map readStnNames(RandomAccessFile raf) { + Map hdrNames = new java.util.HashMap<>(); try { raf.seek(6288); String stnName = raf.readString(16); @@ -241,12 +275,12 @@ public java.util.Map readStnNames(ucar.unidata.io.RandomAccessFi * @param hdrNames java.util.Map with values for "StationName.." Attributes * @return ArrayList of Variables of ncfile */ - public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - java.util.Map hdrNames) { + public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, + Map hdrNames) { // prepare attribute values - String[] data_name = {" ", "TotalPower", "Reflectivity", "Velocity", "Width", "Differential_Reflectivity"}; - String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; - int[] type = {1, 2, 3, 4, 5}; + String[] data_name = SigmetVolumeScan.data_name; + String[] unit = SigmetVolumeScan.data_unit; + String def_datafile = "SIGMET-IRIS"; String tim = ""; int ngates = 0; @@ -305,22 +339,37 @@ public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.N ArrayList varList = new ArrayList<>(); + short[] dataTypes = volScan.getDataTypes(); + Variable[][] v = new Variable[nparams][number_sweeps]; String var_name; for (int j = 0; j < nparams; j++) { - int tp = type[j]; - var_name = data_name[tp]; + short dty = dataTypes[j]; + String var_name_original = data_name[dty]; + + // see also calcData - a lot of data types are raw values + if (dty == 6 || dty > 11) + // this is make it obvious to the user + var_name_original = RAW_VARIABLE_PREFIX + var_name_original; + + var_name = var_name_original; + // support for 2 byte data types is experimental + boolean twoBytes = var_name.endsWith("_2"); + for (int jj = 0; jj < number_sweeps; jj++) { if (number_sweeps > 1) { - var_name = data_name[tp] + "_sweep_" + (jj + 1); + var_name = var_name_original + "_sweep_" + (jj + 1); } v[j][jj] = new Variable(ncfile, null, null, var_name); - v[j][jj].setDataType(DataType.FLOAT); + if (twoBytes) + v[j][jj].setDataType(DataType.DOUBLE); + else + v[j][jj].setDataType(DataType.FLOAT); dims2.add(radial); dims2.add(gateR[jj]); v[j][jj].setDimensions(dims2); v[j][jj].addAttribute(new Attribute(CDM.LONG_NAME, var_name)); - v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[tp])); + v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[dty])); String coordinates = "time elevationR azimuthR distanceR"; v[j][jj].addAttribute(new Attribute(_Coordinate.Axes, coordinates)); v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, -999.99f)); @@ -485,7 +534,7 @@ public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.N * @param recHdr java.util.Map with values for Attributes */ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[] yr, short[] m, short[] dda, - ArrayList varList, java.util.Map recHdr) { + ArrayList varList, Map recHdr) { // prepare attribute values String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; @@ -559,9 +608,16 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ distArr[i].setFloat(distIndex[i].set(ii), (range_first + ii * stp)); } } - List rgp = volScan.getTotalPowerGroups(); - if (rgp.isEmpty()) - rgp = volScan.getReflectivityGroups(); + + // used for lists of distance, time, azimuth, elevation + // probably there is a better way to do this, but I'm keeping the old logic + List rgp = volScan.getGroup("TotalPower"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("TotalPower_2"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("Reflectivity"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("Reflectivity_2"); List[] sgp = new ArrayList[number_sweeps]; for (int i = 0; i < number_sweeps; i++) { sgp[i] = (List) rgp.get((short) i); @@ -586,11 +642,12 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ timeArr[i] = (ArrayInt.D1) Array.factory(DataType.INT, time[i].getShape()); timeIndex[i] = timeArr[i].getIndex(); List rlist = sgp[i]; + int num_rays_actual = Math.min(num_rays, rlist.size()); - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); } } @@ -612,11 +669,12 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ azimArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, azimuthR[i].getShape()); azimIndex[i] = azimArr[i].getIndex(); List rlist = sgp[i]; + int num_rays_actual = Math.min(num_rays, rlist.size()); - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); } } @@ -638,11 +696,12 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ elevArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, elevationR[i].getShape()); elevIndex[i] = elevArr[i].getIndex(); List rlist = sgp[i]; + int num_rays_actual = Math.min(num_rays, rlist.size()); - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { + for (int jj = 0; jj < num_rays_actual; jj++) { elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); } } @@ -661,7 +720,9 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ for (int i = 0; i < number_sweeps; i++) { List rlist = sgp[i]; - for (int jj = 0; jj < num_rays; jj++) { + int num_rays_actual = Math.min(num_rays, rlist.size()); + + for (int jj = 0; jj < num_rays_actual; jj++) { rtemp[jj] = (Ray) rlist.get(jj); } // ray[i][jj]; } ngates = rtemp[0].getBins(); @@ -689,7 +750,7 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ * of ucar.ma2.Range which define the requested data subset. * @return Array of data which will be read from Variable through this call. */ - public Array readData1(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { + public Array readData1(Variable v2, Section section) throws IOException, InvalidRangeException { // doData(raf, ncfile, varList); int[] sh = section.getShape(); Array temp = Array.factory(v2.getDataType(), sh); @@ -715,17 +776,16 @@ public Array readData(Variable v2, Section section) throws IOException { List> groups; String shortName = v2.getShortName(); - if (shortName.startsWith("Reflectivity")) - groups = volScan.getReflectivityGroups(); - else if (shortName.startsWith("Velocity")) - groups = volScan.getVelocityGroups(); - else if (shortName.startsWith("TotalPower")) - groups = volScan.getTotalPowerGroups(); - else if (shortName.startsWith("Width")) - groups = volScan.getWidthGroups(); - else if (shortName.startsWith("DiffReflectivity")) - groups = volScan.getDifferentialReflectivityGroups(); - else + String groupName = shortName; + + int posSweep = groupName.indexOf("_sweep_"); + if (posSweep > 0) + groupName = groupName.substring(0, posSweep); + if (groupName.startsWith(RAW_VARIABLE_PREFIX)) + groupName = groupName.substring(RAW_VARIABLE_PREFIX.length()); + + groups = volScan.getGroup(groupName); + if (groups == null) throw new IllegalStateException("Illegal variable name = " + shortName); if (section.getRank() == 2) { @@ -759,7 +819,7 @@ private void readOneScan(List mapScan, Range radialRange, Range gateRange, } private void readOneRadial(Ray r, Range gateRange, IndexIterator ii) throws IOException { - if (r == null) { + if (r == null || r.offset == -999) { for (int i = 0; i < gateRange.length(); i++) ii.setFloatNext(Float.NaN); return; @@ -811,8 +871,8 @@ public Array readFloatData(LayoutRegular index, Variable v2) throws IOException * @param channel WritableByteChannel object - channel that can write bytes. * @return the number of bytes written, possibly zero. */ - public long readToByteChannel11(ucar.nc2.Variable v2, Section section, WritableByteChannel channel) - throws java.io.IOException { + public long readToByteChannel11(Variable v2, Section section, WritableByteChannel channel) + throws IOException { Array data = readData(v2, section); float[] ftdata = new float[(int) data.getSize()]; byte[] bytedata = new byte[(int) data.getSize() * 4]; @@ -946,7 +1006,8 @@ static float calcData(Map recHdr, short dty, byte data) { float vNyq = recHdr.get("vNyq").floatValue(); double temp = -999.99; switch (dty) { - default: // dty=1,2 -total_power, reflectivity (dBZ) + case 1:// dty=1,2 -total_power, reflectivity (dBZ) + case 2: if (data != 0) { temp = (((int) data & 0xFF) - 64) * 0.5; } @@ -967,12 +1028,73 @@ static float calcData(Map recHdr, short dty, byte data) { temp = ((((int) data & 0xFF) - 128) / 16.0); } break; + default: + // TODO implement for more SigmetVolumeScan.data_name (only 1 byte) + // using only the raw value + temp = (int) data & 0xFF; + + // doing no rounding in this case as below for the other cases + BigDecimal bd = new BigDecimal(temp); + return bd.floatValue(); + //logger.warn("calcData: unimplemented 1 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); } BigDecimal bd = new BigDecimal(temp); + // this should be reviewed, not sure why would you want a loss of precision here? BigDecimal result = bd.setScale(2, RoundingMode.HALF_DOWN); return result.floatValue(); } + /** + * Calculate data values from raw ingest data + * + * @param recHdr java.util.Map object with values for calculation + * @param dty type of data ( "Total_Power", "Reflectivity", "Velocity", + * "Width", "Differential_Reflectivity") + * @param data 2-byte input value + * @return double value + */ + static double calcData(Map recHdr, short dty, short data) { + short[] coef = {1, 2, 3, 4}; // MultiPRF modes + short multiprf = recHdr.get("multiprf").shortValue(); + float vNyq = recHdr.get("vNyq").floatValue(); + double temp = -999.99; + + switch (dty) { + case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) + case 8: + if (data != 0) { + temp = (data - 64) * 0.5; + } + break; + case 9: // dty=9 - mean velocity 2 (m/sec) + if (data != 0) { + temp = ((data - 128) / 127.0) * vNyq * coef[multiprf]; + } + break; + case 10: // dty=10 - spectrum width 2 (m/sec) + if (data != 0) { + double v = ((data - 128) / 127.0) * vNyq * coef[multiprf]; + temp = (data / 256.0) * v; + } + break; + case 11: // dty=11 - differential reflectivity 2 (dB) + if (data != 0) { + temp = ((data - 128) / 16.0); + } + break; + default: + // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) + // using only the raw value + temp = data; + //logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); + break; + } + + BigDecimal bd = new BigDecimal(temp); + // full precision + return bd.doubleValue(); + } + /** * Calculate time as hh:mm:ss * diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 679c970eb5..7da489a57c 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import ucar.nc2.Variable; import ucar.unidata.io.RandomAccessFile; + import java.util.*; /** @@ -20,13 +21,126 @@ */ public class SigmetVolumeScan { private static Logger logger = LoggerFactory.getLogger(SigmetVolumeScan.class); - String[] data_name = {" ", "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity"}; - private List> differentialReflectivityGroups; - private List> reflectivityGroups; - private List> totalPowerGroups; - private List> velocityGroups; - private List> widthGroups; - private List> timeGroups; + /** + * See IRIS-Programming-Guide-M211318EN: 4.9 Constants + * empty or undefined types are set to their index number to avoid duplicates + * if suffix is "_2" it means a two byte value instead of just one byte + * (2 bytes are experimental at this point - didn't have a file to test) + */ + public static final String[] data_name = { + "ExtendedHeaders", // ? bytes + "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity", "[6]", "CorrectedReflectivity", // 1 byte + "TotalPower_2", "Reflectivity_2", "Velocity_2", "Width_2", "DifferentialReflectivity_2", // 2 bytes + "RainfallRate_2" /* 2 bytes */, "KDPDifferentialPhase" /* 1 byte */, "KDPDifferentialPhase_2" /* 2 bytes */, + "PhiDPDifferentialPhase" /* 1 byte */, "CorrectedVelocity" /* 1 byte */, "SQI" /* 1 byte */, + "RhoHV" /* 1 byte */, "RhoHV_2" /* 2 bytes */, + "CorrectedReflectivity_2" /* 2 bytes */, "CorrectedVelocity_2" /* 2 bytes */, "SQI_2" /* 2 bytes */, + "PhiDPDifferentialPhase_2" /* 2 bytes */, + "LDRH" /* 1 byte */, "LDRH_2" /* 2 bytes */, "LDRV" /* 1 byte */, "LDRV_2" /* 2 bytes */, + "[29]", "[30]", "[31]", + "Height" /* 1 byte */, "LinearLiquid_2" /* 2 bytes */, "RawData" /* ? */, + "WindShear" /* 1 byte */, "Divergence_2" /* 2 bytes */, "FloatedLiquid_2" /* 2 bytes */, + "UserType" /* 1 byte */, "UnspecifiedData" /* 1 byte */, "Deformation_2" /* 2 bytes */, + "VerticalVelocity_2" /* 2 bytes */, "HorizontalVelocity_2" /* 2 bytes */, + "HorizontalWindDirection_2" /* 2 bytes */, "AxisOfDilatation_2" /* 2 bytes */, "TimeInSeconds_2" /* 2 bytes */, + "RHOH" /* 1 byte */, "RHOH_2" /* 2 bytes */, "RHOV" /* 1 byte */, "RHOV_2" /* 2 bytes */, + "PHIH" /* 1 byte */, "PHIH_2" /* 2 bytes */, "PHIV" /* 1 byte */, "PHIV_2" /* 2 bytes */, + "UserType_2" /* 2 bytes */, "HydrometeorClass" /* 1 byte */, "HydrometeorClass_2" /* 2 bytes */, + "CorrectedDifferentialReflectivity" /* 1 byte */, "CorrectedDifferentialReflectivity_2" /* 2 bytes */, + // 16 empty + "[59]", "[60]", "[61]", "[62]", "[63]", "[64]", "[65]", "[66]", "[67]", "[68]", "[69]", "[70]", "[71]", "[72]", "[73]", "[74]", + "PolarimetricMeteoIndex" /* 1 byte */, "PolarimetricMeteoIndex_2" /* 2 bytes */, + "LOG8" /* 1 byte */, "LOG16_2" /* 2 bytes */, "CSP8" /* 1 byte */, "CSP16_2" /* 2 bytes */, + "CCOR8" /* 1 byte */, "CCOR16_2" /* 2 bytes */, "AH8" /* 1 byte */, "AH16_2" /* 2 bytes */, + "AV8" /* 1 byte */, "AV16_2" /* 2 bytes */, "AZDR8" /* 1 byte */, "AZDR16_2" /* 2 bytes */, + }; + + /* + Some units are unknown, some were correlated and assumed by ChatGPT. + Not all might be correct! + Just extending the initial list here from previous version of SigmetIOServiceProvider + // TODO fill and double check + */ + public static final String[] data_unit = { + "?", // DB_XHDR: Extended Headers + "dBm", // DB_DBT: Total Power (1 byte) + "dBZ", // DB_DBZ: Reflectivity (1 byte) + "m/s", // DB_VEL: Velocity (1 byte) + "m/s", // DB_WIDTH: Width (1 byte) + "dB", // DB_ZDR: Differential Reflectivity (1 byte) + "?", // empty + "dBZ", // DB_DBZC: Corrected Reflectivity (1 byte) + "dBm", // DB_DBT2: Total Power (2 byte) + "dBZ", // DB_DBZ2: Reflectivity (2 byte) + "m/s", // DB_VEL2: Velocity (2 byte) + "m/s", // DB_WIDTH2: Width (2 byte) + "dB", // DB_ZDR2: Differential Reflectivity (2 byte) + "mm/hr", // DB_RAINRATE2: Rainfall Rate (2 byte) + "°/km", // DB_KDP: KDP (Differential Phase) (1 byte) + "°/km", // DB_KDP2: KDP (Differential Phase) (2 byte) + "°", // DB_PHIDP: PhiDP (Differential Phase) (1 byte) + "m/s", // DB_VELC: Corrected Velocity (1 byte) + "?", // DB_SQI: SQI (Signal Quality Index) (1 byte) + "?", // DB_RHOHV: RhoHV (1 byte) + "?", // DB_RHOHV2: RhoHV (2 byte) + "dBZ", // DB_DBZC2: Corrected Reflectivity (2 byte) + "m/s", // DB_VELC2: Corrected Velocity (2 byte) + "?", // DB_SQI2: SQI (Signal Quality Index) (2 byte) + "°", // DB_PHIDP2: PhiDP (Differential Phase) (2 byte) + "?", // DB_LDRH: LDR xmt H rcv V (1 byte) + "?", // DB_LDRH2: LDR xmt H rcv V (2 byte) + "?", // DB_LDRV: LDR xmt V rcv H (1 byte) + "?", // DB_LDRV2: LDR xmt V rcv H (2 byte) + // 3 empty + "?", "?", "?", + "1/10 km", // DB_HEIGHT: Height (1/10 km) (1 byte) + ".001mm", // DB_VIL2: Linear liquid (.001mm) (2 byte) + "?", // DB_RAW: Unknown unit or unitless (Raw Data) + "m/s", // DB_SHEAR: Shear (Velocity difference) + "?", // DB_DIVERGE2: Divergence (2 byte) + "mm", // DB_FLIQUID2: Liquid equivalent (2 byte) + "?", // DB_USER: User-defined (unit depends on definition) + "?", // DB_OTHER: Other data type (unit depends on definition) + "1/s", // DB_DEFORM2: Deformation (2 byte) + "m/s", // DB_VVEL2: Vertical Velocity (2 byte) + "m/s", // DB_HVEL2: Horizontal Velocity (2 byte) + "°", // DB_HDIR2: Horizontal Direction (2 byte) + "1/s", // DB_AXDIL2: Axis of Dilation (2 byte) + "s", // DB_TIME2: Time (2 byte) + "?", // DB_RHOH: RhoH (1 byte) + "?", // DB_RHOH2: RhoH (2 byte) + "?", // DB_RHOV: RhoV (1 byte) + "?", // DB_RHOV2: RhoV (2 byte) + "°", // DB_PHIH: PhiH (1 byte) + "°", // DB_PHIH2: PhiH (2 byte) + "°", // DB_PHIV: PhiV (1 byte) + "°", // DB_PHIV2: PhiV (2 byte) + "?", // DB_USER2: User-defined (2 byte) + "?", // DB_HCLASS: Hydrometeor Classification (1 byte) + "?", // DB_HCLASS2: Hydrometeor Classification (2 byte) + "dB", // DB_ZDRC: Corrected Differential Reflectivity (1 byte) + "dB", // DB_ZDRC2: Corrected Differential Reflectivity (2 byte) + // 16 empty + "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", + "?", // PolarimetricMeteoIndex (1 byte) + "?", // PolarimetricMeteoIndex2 (2 bytes) + "?", // LOG8 (1 byte) + "?", // LOG16 (2 bytes) + "?", // CSP8 (1 byte) + "?", // CSP16 (2 bytes) + "?", // CCOR8 (1 byte) + "?", // CCOR16 (2 bytes) + "?", // AH8 (1 byte) + "?", // AH16 (2 bytes) + "?", // AV8 (1 byte) + "?", // AV16 (2 bytes) + "?", // AZDR8 (1 byte) + "?", // AZDR16 (2 bytes) + }; + + private HashMap>> allGroups = new HashMap<>(); + + private short[] data_type; private int[] num_gates; public int[] base_time; public short[] year; @@ -34,13 +148,7 @@ public class SigmetVolumeScan { public short[] day; public Ray firstRay; public Ray lastRay; - public ucar.unidata.io.RandomAccessFile raf; - public boolean hasReflectivity; - public boolean hasVelocity; - public boolean hasWidth; - public boolean hasTotalPower; - public boolean hasDifferentialReflectivity; - public boolean hasTime; + public RandomAccessFile raf; /** * Read all the values from SIGMET-IRIS file which are necessary to fill in the ncfile. @@ -49,7 +157,7 @@ public class SigmetVolumeScan { * @param ncfile an empty NetcdfFile object which will be filled. * @param varList ArrayList of Variables of ncfile */ - SigmetVolumeScan(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) + SigmetVolumeScan(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) throws java.io.IOException { int REC_SIZE = 6144; int len = 12288; // ---- Read from the 3d record----------- 6144*2=12288 @@ -68,7 +176,7 @@ public class SigmetVolumeScan { raf.order(RandomAccessFile.LITTLE_ENDIAN); int fileLength = (int) raf.length(); - java.util.Map recHdr = SigmetIOServiceProvider.readRecordsHdr(raf); + Map recHdr = SigmetIOServiceProvider.readRecordsHdr(raf); int nparams = recHdr.get("nparams").intValue(); short number_sweeps = recHdr.get("number_sweeps").shortValue(); @@ -84,10 +192,11 @@ public class SigmetVolumeScan { short[] num_sweep = new short[nparams]; short[] num_rays_swp = new short[nparams]; short[] indx_1ray = new short[nparams]; + short[] num_rays_exp = new short[nparams]; short[] num_rays_act = new short[nparams]; short[] angl_swp = new short[nparams]; short[] bin_len = new short[nparams]; - short[] data_type = new short[nparams]; + data_type = new short[nparams]; // float[] dd = new float[bins]; num_gates = new int[number_sweeps]; base_time = new int[nparams * number_sweeps]; @@ -97,15 +206,10 @@ public class SigmetVolumeScan { // Array of Ray objects is 2D. Number of columns=number of rays // Number of raws = number of types of data if number_sweeps=1, // or number of raws = number_sweeps - List totalPower = new ArrayList<>(); - List velocity = new ArrayList<>(); - List reflectivity = new ArrayList<>(); - List width = new ArrayList<>(); - List diffReflectivity = new ArrayList<>(); - List time = new ArrayList<>(); + HashMap> all = new HashMap<>(); + int irays = (int) num_rays; Ray ray = null; - int two = 0; // init array float[] val = new float[bins]; @@ -142,7 +246,19 @@ public class SigmetVolumeScan { beg = 0; for (int i = 0; i < nparams; i++) { - int idh_len = cur_len + 12 + i * 76; + int idh_len = cur_len + 12 + i * 76; // + 12 == skipping over + + /* debug structure_header + raf.seek(idh_len-12); + // Structure identifier + short si = raf.readShort(); + raf.readShort(); // format version + int nob = raf.readInt(); + if (si != (short)24) + throw new IllegalStateException("not a Ingest_header"); + raf.readShort(); // reserved + raf.readShort(); // flags + */ raf.seek(idh_len); @@ -156,10 +272,12 @@ public class SigmetVolumeScan { num_sweep[i] = raf.readShort(); // idh_len+12 num_rays_swp[i] = raf.readShort(); // idh_len+14 indx_1ray[i] = raf.readShort(); // idh_len+16 - raf.skipBytes(2); + num_rays_exp[i] = raf.readShort(); + beg += num_rays_exp[i]; // before num_rays_act[i] was used but it does seem not work num_rays_act[i] = raf.readShort(); - beg += num_rays_act[i]; // idh_len+20 + //beg += num_rays_act[i]; // idh_len+20 angl_swp[i] = raf.readShort(); // idh_len+22 + // TODO maybe use in stead of variable twoBytes? bin_len[i] = raf.readShort(); // idh_len+24 data_type[i] = raf.readShort(); // idh_len+26 } @@ -220,6 +338,8 @@ public class SigmetVolumeScan { } String var_name = data_name[dty]; + // support for 2 byte data types is experimental + boolean twoBytes = var_name.endsWith("_2"); // --- read ray_header (size=12 bytes=6 words)--------------------------------------- if (read_ray_hdr) { @@ -333,6 +453,10 @@ public class SigmetVolumeScan { // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); cur_len++; nb++; + if (twoBytes) { + cur_len++; + //i++;? + } if (cur_len % REC_SIZE == 0) { pos = i + 1; @@ -400,6 +524,10 @@ public class SigmetVolumeScan { // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); cur_len = cur_len + 1; nb = nb + 1; + if (twoBytes) { + cur_len++; + //ii++;? + } if (cur_len % REC_SIZE == 0) { pos = ii + 1; @@ -422,6 +550,7 @@ public class SigmetVolumeScan { // } nb = nb + num_zero; + // TODO extra handling for twoBytes here, too? if (cur_len % REC_SIZE == 0) { beg_rec = true; @@ -450,21 +579,15 @@ public class SigmetVolumeScan { ray = new Ray(range_first, step, az, elev, num_bins, time_start_sw, rayoffset, datalen, rayoffset1, nsweep, var_name, dty); rays_count++; - two++; + if ((nsweep == number_sweeps) & (rays_count % beg == 0)) { - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found1 {}", var_name); + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); } + varL.add(ray); break; } @@ -476,40 +599,29 @@ public class SigmetVolumeScan { data_read = 0; nb = 0; len = cur_len; - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found2 {}", var_name); + + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); } + varL.add(ray); + continue; } } - // "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity" if (firstRay == null) firstRay = ray; - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found3 {}", var_name); + // using a universal structure that works for all data types + List additionalL = all.get(var_name.trim()); + if (additionalL == null) { + additionalL = new ArrayList<>(); + all.put(var_name.trim(), additionalL); } + additionalL.add(ray); pos = 0; data_read = 0; @@ -530,30 +642,11 @@ public class SigmetVolumeScan { } // ------------end of outer while --------------- lastRay = ray; - if (!reflectivity.isEmpty()) { - reflectivityGroups = sortScans("reflectivity", reflectivity, 1000); - hasReflectivity = true; - } - if (!velocity.isEmpty()) { - velocityGroups = sortScans("velocity", velocity, 1000); - hasVelocity = true; - } - if (!totalPower.isEmpty()) { - totalPowerGroups = sortScans("totalPower", totalPower, 1000); - hasTotalPower = true; - } - if (!width.isEmpty()) { - widthGroups = sortScans("width", width, 1000); - hasWidth = true; - } - if (!diffReflectivity.isEmpty()) { - differentialReflectivityGroups = sortScans("diffReflectivity", diffReflectivity, 1000); - hasDifferentialReflectivity = true; - } - - if (!time.isEmpty()) { - timeGroups = sortScans("diffReflectivity", diffReflectivity, 1000); - hasTime = true; + // using a universal structure that works for all data types + for (String var_name : all.keySet()) { + List additionalL = all.get(var_name); + if (!additionalL.isEmpty()) + allGroups.put(var_name, sortScans(var_name, additionalL, 1000)); } // --------- fill all of values in the ncfile ------ @@ -624,24 +717,13 @@ public int compare(List group1, List group2) { } } - public List> getTotalPowerGroups() { - return totalPowerGroups; - } - - public List> getVelocityGroups() { - return velocityGroups; - } - - public List> getWidthGroups() { - return widthGroups; - } - - public List> getReflectivityGroups() { - return reflectivityGroups; + // using a universal structure that works for all data types + public List> getGroup(String name) { + return allGroups.get(name); } - public List> getDifferentialReflectivityGroups() { - return differentialReflectivityGroups; + public short[] getDataTypes() { + return data_type; } public int[] getNumberGates() { From 97ca4ed87bfb0090f333a5b3d657a084bf327641 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 27 Jan 2024 11:30:04 +0100 Subject: [PATCH 09/16] cdm-radial: sigmet Code Style... :/ --- .../iosp/sigmet/SigmetIOServiceProvider.java | 16 +- .../nc2/iosp/sigmet/SigmetVolumeScan.java | 233 +++++++++--------- 2 files changed, 121 insertions(+), 128 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index 09f6d9d62c..66a0a906a8 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -121,8 +121,8 @@ public boolean isValidFile(RandomAccessFile raf) { /** * Open existing file, and populate ncfile with it. */ - public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - ucar.nc2.util.CancelTask cancelTask) throws IOException { + public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ucar.nc2.util.CancelTask cancelTask) + throws IOException { super.open(raf, ncfile, cancelTask); // java.util.Map recHdr=new java.util.HashMap(); Map hdrNames = new java.util.HashMap<>(); @@ -275,8 +275,7 @@ public Map readStnNames(RandomAccessFile raf) { * @param hdrNames java.util.Map with values for "StationName.." Attributes * @return ArrayList of Variables of ncfile */ - public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - Map hdrNames) { + public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, Map hdrNames) { // prepare attribute values String[] data_name = SigmetVolumeScan.data_name; String[] unit = SigmetVolumeScan.data_unit; @@ -782,7 +781,7 @@ public Array readData(Variable v2, Section section) throws IOException { if (posSweep > 0) groupName = groupName.substring(0, posSweep); if (groupName.startsWith(RAW_VARIABLE_PREFIX)) - groupName = groupName.substring(RAW_VARIABLE_PREFIX.length()); + groupName = groupName.substring(RAW_VARIABLE_PREFIX.length()); groups = volScan.getGroup(groupName); if (groups == null) @@ -871,8 +870,7 @@ public Array readFloatData(LayoutRegular index, Variable v2) throws IOException * @param channel WritableByteChannel object - channel that can write bytes. * @return the number of bytes written, possibly zero. */ - public long readToByteChannel11(Variable v2, Section section, WritableByteChannel channel) - throws IOException { + public long readToByteChannel11(Variable v2, Section section, WritableByteChannel channel) throws IOException { Array data = readData(v2, section); float[] ftdata = new float[(int) data.getSize()]; byte[] bytedata = new byte[(int) data.getSize() * 4]; @@ -1036,7 +1034,7 @@ static float calcData(Map recHdr, short dty, byte data) { // doing no rounding in this case as below for the other cases BigDecimal bd = new BigDecimal(temp); return bd.floatValue(); - //logger.warn("calcData: unimplemented 1 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); + // logger.warn("calcData: unimplemented 1 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); } BigDecimal bd = new BigDecimal(temp); // this should be reviewed, not sure why would you want a loss of precision here? @@ -1086,7 +1084,7 @@ static double calcData(Map recHdr, short dty, short data) { // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) // using only the raw value temp = data; - //logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); + // logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); break; } diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 7da489a57c..17d86de109 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -27,115 +27,109 @@ public class SigmetVolumeScan { * if suffix is "_2" it means a two byte value instead of just one byte * (2 bytes are experimental at this point - didn't have a file to test) */ - public static final String[] data_name = { - "ExtendedHeaders", // ? bytes - "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity", "[6]", "CorrectedReflectivity", // 1 byte - "TotalPower_2", "Reflectivity_2", "Velocity_2", "Width_2", "DifferentialReflectivity_2", // 2 bytes - "RainfallRate_2" /* 2 bytes */, "KDPDifferentialPhase" /* 1 byte */, "KDPDifferentialPhase_2" /* 2 bytes */, - "PhiDPDifferentialPhase" /* 1 byte */, "CorrectedVelocity" /* 1 byte */, "SQI" /* 1 byte */, - "RhoHV" /* 1 byte */, "RhoHV_2" /* 2 bytes */, - "CorrectedReflectivity_2" /* 2 bytes */, "CorrectedVelocity_2" /* 2 bytes */, "SQI_2" /* 2 bytes */, - "PhiDPDifferentialPhase_2" /* 2 bytes */, - "LDRH" /* 1 byte */, "LDRH_2" /* 2 bytes */, "LDRV" /* 1 byte */, "LDRV_2" /* 2 bytes */, - "[29]", "[30]", "[31]", - "Height" /* 1 byte */, "LinearLiquid_2" /* 2 bytes */, "RawData" /* ? */, - "WindShear" /* 1 byte */, "Divergence_2" /* 2 bytes */, "FloatedLiquid_2" /* 2 bytes */, - "UserType" /* 1 byte */, "UnspecifiedData" /* 1 byte */, "Deformation_2" /* 2 bytes */, - "VerticalVelocity_2" /* 2 bytes */, "HorizontalVelocity_2" /* 2 bytes */, - "HorizontalWindDirection_2" /* 2 bytes */, "AxisOfDilatation_2" /* 2 bytes */, "TimeInSeconds_2" /* 2 bytes */, - "RHOH" /* 1 byte */, "RHOH_2" /* 2 bytes */, "RHOV" /* 1 byte */, "RHOV_2" /* 2 bytes */, - "PHIH" /* 1 byte */, "PHIH_2" /* 2 bytes */, "PHIV" /* 1 byte */, "PHIV_2" /* 2 bytes */, - "UserType_2" /* 2 bytes */, "HydrometeorClass" /* 1 byte */, "HydrometeorClass_2" /* 2 bytes */, - "CorrectedDifferentialReflectivity" /* 1 byte */, "CorrectedDifferentialReflectivity_2" /* 2 bytes */, - // 16 empty - "[59]", "[60]", "[61]", "[62]", "[63]", "[64]", "[65]", "[66]", "[67]", "[68]", "[69]", "[70]", "[71]", "[72]", "[73]", "[74]", - "PolarimetricMeteoIndex" /* 1 byte */, "PolarimetricMeteoIndex_2" /* 2 bytes */, - "LOG8" /* 1 byte */, "LOG16_2" /* 2 bytes */, "CSP8" /* 1 byte */, "CSP16_2" /* 2 bytes */, - "CCOR8" /* 1 byte */, "CCOR16_2" /* 2 bytes */, "AH8" /* 1 byte */, "AH16_2" /* 2 bytes */, - "AV8" /* 1 byte */, "AV16_2" /* 2 bytes */, "AZDR8" /* 1 byte */, "AZDR16_2" /* 2 bytes */, - }; + public static final String[] data_name = {"ExtendedHeaders", // ? bytes + "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity", "[6]", "CorrectedReflectivity", // 1 + // byte + "TotalPower_2", "Reflectivity_2", "Velocity_2", "Width_2", "DifferentialReflectivity_2", // 2 bytes + "RainfallRate_2" /* 2 bytes */, "KDPDifferentialPhase" /* 1 byte */, "KDPDifferentialPhase_2" /* 2 bytes */, + "PhiDPDifferentialPhase" /* 1 byte */, "CorrectedVelocity" /* 1 byte */, "SQI" /* 1 byte */, "RhoHV" /* 1 byte */, + "RhoHV_2" /* 2 bytes */, "CorrectedReflectivity_2" /* 2 bytes */, "CorrectedVelocity_2" /* 2 bytes */, + "SQI_2" /* 2 bytes */, "PhiDPDifferentialPhase_2" /* 2 bytes */, "LDRH" /* 1 byte */, "LDRH_2" /* 2 bytes */, + "LDRV" /* 1 byte */, "LDRV_2" /* 2 bytes */, "[29]", "[30]", "[31]", "Height" /* 1 byte */, + "LinearLiquid_2" /* 2 bytes */, "RawData" /* ? */, "WindShear" /* 1 byte */, "Divergence_2" /* 2 bytes */, + "FloatedLiquid_2" /* 2 bytes */, "UserType" /* 1 byte */, "UnspecifiedData" /* 1 byte */, + "Deformation_2" /* 2 bytes */, "VerticalVelocity_2" /* 2 bytes */, "HorizontalVelocity_2" /* 2 bytes */, + "HorizontalWindDirection_2" /* 2 bytes */, "AxisOfDilatation_2" /* 2 bytes */, "TimeInSeconds_2" /* 2 bytes */, + "RHOH" /* 1 byte */, "RHOH_2" /* 2 bytes */, "RHOV" /* 1 byte */, "RHOV_2" /* 2 bytes */, "PHIH" /* 1 byte */, + "PHIH_2" /* 2 bytes */, "PHIV" /* 1 byte */, "PHIV_2" /* 2 bytes */, "UserType_2" /* 2 bytes */, + "HydrometeorClass" /* 1 byte */, "HydrometeorClass_2" /* 2 bytes */, + "CorrectedDifferentialReflectivity" /* 1 byte */, "CorrectedDifferentialReflectivity_2" /* 2 bytes */, + // 16 empty + "[59]", "[60]", "[61]", "[62]", "[63]", "[64]", "[65]", "[66]", "[67]", "[68]", "[69]", "[70]", "[71]", "[72]", + "[73]", "[74]", "PolarimetricMeteoIndex" /* 1 byte */, "PolarimetricMeteoIndex_2" /* 2 bytes */, + "LOG8" /* 1 byte */, "LOG16_2" /* 2 bytes */, "CSP8" /* 1 byte */, "CSP16_2" /* 2 bytes */, "CCOR8" /* 1 byte */, + "CCOR16_2" /* 2 bytes */, "AH8" /* 1 byte */, "AH16_2" /* 2 bytes */, "AV8" /* 1 byte */, "AV16_2" /* 2 bytes */, + "AZDR8" /* 1 byte */, "AZDR16_2" /* 2 bytes */,}; /* - Some units are unknown, some were correlated and assumed by ChatGPT. - Not all might be correct! - Just extending the initial list here from previous version of SigmetIOServiceProvider - // TODO fill and double check + * Some units are unknown, some were correlated and assumed by ChatGPT. + * Not all might be correct! + * Just extending the initial list here from previous version of SigmetIOServiceProvider + * // TODO fill and double check */ - public static final String[] data_unit = { - "?", // DB_XHDR: Extended Headers - "dBm", // DB_DBT: Total Power (1 byte) - "dBZ", // DB_DBZ: Reflectivity (1 byte) - "m/s", // DB_VEL: Velocity (1 byte) - "m/s", // DB_WIDTH: Width (1 byte) - "dB", // DB_ZDR: Differential Reflectivity (1 byte) - "?", // empty - "dBZ", // DB_DBZC: Corrected Reflectivity (1 byte) - "dBm", // DB_DBT2: Total Power (2 byte) - "dBZ", // DB_DBZ2: Reflectivity (2 byte) - "m/s", // DB_VEL2: Velocity (2 byte) - "m/s", // DB_WIDTH2: Width (2 byte) - "dB", // DB_ZDR2: Differential Reflectivity (2 byte) - "mm/hr", // DB_RAINRATE2: Rainfall Rate (2 byte) - "°/km", // DB_KDP: KDP (Differential Phase) (1 byte) - "°/km", // DB_KDP2: KDP (Differential Phase) (2 byte) - "°", // DB_PHIDP: PhiDP (Differential Phase) (1 byte) - "m/s", // DB_VELC: Corrected Velocity (1 byte) - "?", // DB_SQI: SQI (Signal Quality Index) (1 byte) - "?", // DB_RHOHV: RhoHV (1 byte) - "?", // DB_RHOHV2: RhoHV (2 byte) - "dBZ", // DB_DBZC2: Corrected Reflectivity (2 byte) - "m/s", // DB_VELC2: Corrected Velocity (2 byte) - "?", // DB_SQI2: SQI (Signal Quality Index) (2 byte) - "°", // DB_PHIDP2: PhiDP (Differential Phase) (2 byte) - "?", // DB_LDRH: LDR xmt H rcv V (1 byte) - "?", // DB_LDRH2: LDR xmt H rcv V (2 byte) - "?", // DB_LDRV: LDR xmt V rcv H (1 byte) - "?", // DB_LDRV2: LDR xmt V rcv H (2 byte) - // 3 empty - "?", "?", "?", - "1/10 km", // DB_HEIGHT: Height (1/10 km) (1 byte) - ".001mm", // DB_VIL2: Linear liquid (.001mm) (2 byte) - "?", // DB_RAW: Unknown unit or unitless (Raw Data) - "m/s", // DB_SHEAR: Shear (Velocity difference) - "?", // DB_DIVERGE2: Divergence (2 byte) - "mm", // DB_FLIQUID2: Liquid equivalent (2 byte) - "?", // DB_USER: User-defined (unit depends on definition) - "?", // DB_OTHER: Other data type (unit depends on definition) - "1/s", // DB_DEFORM2: Deformation (2 byte) - "m/s", // DB_VVEL2: Vertical Velocity (2 byte) - "m/s", // DB_HVEL2: Horizontal Velocity (2 byte) - "°", // DB_HDIR2: Horizontal Direction (2 byte) - "1/s", // DB_AXDIL2: Axis of Dilation (2 byte) - "s", // DB_TIME2: Time (2 byte) - "?", // DB_RHOH: RhoH (1 byte) - "?", // DB_RHOH2: RhoH (2 byte) - "?", // DB_RHOV: RhoV (1 byte) - "?", // DB_RHOV2: RhoV (2 byte) - "°", // DB_PHIH: PhiH (1 byte) - "°", // DB_PHIH2: PhiH (2 byte) - "°", // DB_PHIV: PhiV (1 byte) - "°", // DB_PHIV2: PhiV (2 byte) - "?", // DB_USER2: User-defined (2 byte) - "?", // DB_HCLASS: Hydrometeor Classification (1 byte) - "?", // DB_HCLASS2: Hydrometeor Classification (2 byte) - "dB", // DB_ZDRC: Corrected Differential Reflectivity (1 byte) - "dB", // DB_ZDRC2: Corrected Differential Reflectivity (2 byte) - // 16 empty - "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", - "?", // PolarimetricMeteoIndex (1 byte) - "?", // PolarimetricMeteoIndex2 (2 bytes) - "?", // LOG8 (1 byte) - "?", // LOG16 (2 bytes) - "?", // CSP8 (1 byte) - "?", // CSP16 (2 bytes) - "?", // CCOR8 (1 byte) - "?", // CCOR16 (2 bytes) - "?", // AH8 (1 byte) - "?", // AH16 (2 bytes) - "?", // AV8 (1 byte) - "?", // AV16 (2 bytes) - "?", // AZDR8 (1 byte) - "?", // AZDR16 (2 bytes) + public static final String[] data_unit = {"?", // DB_XHDR: Extended Headers + "dBm", // DB_DBT: Total Power (1 byte) + "dBZ", // DB_DBZ: Reflectivity (1 byte) + "m/s", // DB_VEL: Velocity (1 byte) + "m/s", // DB_WIDTH: Width (1 byte) + "dB", // DB_ZDR: Differential Reflectivity (1 byte) + "?", // empty + "dBZ", // DB_DBZC: Corrected Reflectivity (1 byte) + "dBm", // DB_DBT2: Total Power (2 byte) + "dBZ", // DB_DBZ2: Reflectivity (2 byte) + "m/s", // DB_VEL2: Velocity (2 byte) + "m/s", // DB_WIDTH2: Width (2 byte) + "dB", // DB_ZDR2: Differential Reflectivity (2 byte) + "mm/hr", // DB_RAINRATE2: Rainfall Rate (2 byte) + "°/km", // DB_KDP: KDP (Differential Phase) (1 byte) + "°/km", // DB_KDP2: KDP (Differential Phase) (2 byte) + "°", // DB_PHIDP: PhiDP (Differential Phase) (1 byte) + "m/s", // DB_VELC: Corrected Velocity (1 byte) + "?", // DB_SQI: SQI (Signal Quality Index) (1 byte) + "?", // DB_RHOHV: RhoHV (1 byte) + "?", // DB_RHOHV2: RhoHV (2 byte) + "dBZ", // DB_DBZC2: Corrected Reflectivity (2 byte) + "m/s", // DB_VELC2: Corrected Velocity (2 byte) + "?", // DB_SQI2: SQI (Signal Quality Index) (2 byte) + "°", // DB_PHIDP2: PhiDP (Differential Phase) (2 byte) + "?", // DB_LDRH: LDR xmt H rcv V (1 byte) + "?", // DB_LDRH2: LDR xmt H rcv V (2 byte) + "?", // DB_LDRV: LDR xmt V rcv H (1 byte) + "?", // DB_LDRV2: LDR xmt V rcv H (2 byte) + // 3 empty + "?", "?", "?", "1/10 km", // DB_HEIGHT: Height (1/10 km) (1 byte) + ".001mm", // DB_VIL2: Linear liquid (.001mm) (2 byte) + "?", // DB_RAW: Unknown unit or unitless (Raw Data) + "m/s", // DB_SHEAR: Shear (Velocity difference) + "?", // DB_DIVERGE2: Divergence (2 byte) + "mm", // DB_FLIQUID2: Liquid equivalent (2 byte) + "?", // DB_USER: User-defined (unit depends on definition) + "?", // DB_OTHER: Other data type (unit depends on definition) + "1/s", // DB_DEFORM2: Deformation (2 byte) + "m/s", // DB_VVEL2: Vertical Velocity (2 byte) + "m/s", // DB_HVEL2: Horizontal Velocity (2 byte) + "°", // DB_HDIR2: Horizontal Direction (2 byte) + "1/s", // DB_AXDIL2: Axis of Dilation (2 byte) + "s", // DB_TIME2: Time (2 byte) + "?", // DB_RHOH: RhoH (1 byte) + "?", // DB_RHOH2: RhoH (2 byte) + "?", // DB_RHOV: RhoV (1 byte) + "?", // DB_RHOV2: RhoV (2 byte) + "°", // DB_PHIH: PhiH (1 byte) + "°", // DB_PHIH2: PhiH (2 byte) + "°", // DB_PHIV: PhiV (1 byte) + "°", // DB_PHIV2: PhiV (2 byte) + "?", // DB_USER2: User-defined (2 byte) + "?", // DB_HCLASS: Hydrometeor Classification (1 byte) + "?", // DB_HCLASS2: Hydrometeor Classification (2 byte) + "dB", // DB_ZDRC: Corrected Differential Reflectivity (1 byte) + "dB", // DB_ZDRC2: Corrected Differential Reflectivity (2 byte) + // 16 empty + "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", // PolarimetricMeteoIndex (1 + // byte) + "?", // PolarimetricMeteoIndex2 (2 bytes) + "?", // LOG8 (1 byte) + "?", // LOG16 (2 bytes) + "?", // CSP8 (1 byte) + "?", // CSP16 (2 bytes) + "?", // CCOR8 (1 byte) + "?", // CCOR16 (2 bytes) + "?", // AH8 (1 byte) + "?", // AH16 (2 bytes) + "?", // AV8 (1 byte) + "?", // AV16 (2 bytes) + "?", // AZDR8 (1 byte) + "?", // AZDR16 (2 bytes) }; private HashMap>> allGroups = new HashMap<>(); @@ -248,16 +242,17 @@ public class SigmetVolumeScan { for (int i = 0; i < nparams; i++) { int idh_len = cur_len + 12 + i * 76; // + 12 == skipping over - /* debug structure_header - raf.seek(idh_len-12); - // Structure identifier - short si = raf.readShort(); - raf.readShort(); // format version - int nob = raf.readInt(); - if (si != (short)24) - throw new IllegalStateException("not a Ingest_header"); - raf.readShort(); // reserved - raf.readShort(); // flags + /* + * debug structure_header + * raf.seek(idh_len-12); + * // Structure identifier + * short si = raf.readShort(); + * raf.readShort(); // format version + * int nob = raf.readInt(); + * if (si != (short)24) + * throw new IllegalStateException("not a Ingest_header"); + * raf.readShort(); // reserved + * raf.readShort(); // flags */ raf.seek(idh_len); @@ -275,7 +270,7 @@ public class SigmetVolumeScan { num_rays_exp[i] = raf.readShort(); beg += num_rays_exp[i]; // before num_rays_act[i] was used but it does seem not work num_rays_act[i] = raf.readShort(); - //beg += num_rays_act[i]; // idh_len+20 + // beg += num_rays_act[i]; // idh_len+20 angl_swp[i] = raf.readShort(); // idh_len+22 // TODO maybe use in stead of variable twoBytes? bin_len[i] = raf.readShort(); // idh_len+24 @@ -455,7 +450,7 @@ public class SigmetVolumeScan { nb++; if (twoBytes) { cur_len++; - //i++;? + // i++;? } if (cur_len % REC_SIZE == 0) { @@ -526,7 +521,7 @@ public class SigmetVolumeScan { nb = nb + 1; if (twoBytes) { cur_len++; - //ii++;? + // ii++;? } if (cur_len % REC_SIZE == 0) { From c5b66c6720815c411b1f7627f1b0033d1a31b2c0 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Tue, 30 Jan 2024 12:08:06 +0100 Subject: [PATCH 10/16] cdm-radial: sigmet Another fix necessary to support IDEAM Colombia files In this case not all sweeps of num_sweeps have data, so adapting for this and handling it gracefully. Happens for the radars San_Andres and Corozal. --- .../iosp/sigmet/SigmetIOServiceProvider.java | 92 ++++++++++++------- 1 file changed, 61 insertions(+), 31 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index 66a0a906a8..aa3c31c4a2 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -617,11 +617,23 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ rgp = volScan.getGroup("Reflectivity"); if (rgp == null || rgp.isEmpty()) rgp = volScan.getGroup("Reflectivity_2"); + List[] sgp = new ArrayList[number_sweeps]; - for (int i = 0; i < number_sweeps; i++) { - sgp[i] = (List) rgp.get((short) i); - } + for (int sweepMinus1 = 0; sweepMinus1 < number_sweeps; sweepMinus1++) { + List found = null; + // in case some rays are missing/empty this is a safer way to find them + for (int i = 0; i < rgp.size() && found == null; i++) { + List rlist = (List) rgp.get(i); + if (rlist.size() > 0) { + Ray r = rlist.get(0); + if (r.getNsweep() == sweepMinus1 + 1) { + found = rlist; + } + } + } + sgp[sweepMinus1] = found; + } Variable[] time = new Variable[number_sweeps]; ArrayInt.D1[] timeArr = new ArrayInt.D1[number_sweeps]; @@ -641,13 +653,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ timeArr[i] = (ArrayInt.D1) Array.factory(DataType.INT, time[i].getShape()); timeIndex[i] = timeArr[i].getIndex(); List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays_actual; jj++) { - timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + timeArr[i].setInt(timeIndex[i].set(jj), -999); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); + } } } @@ -668,13 +685,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ azimArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, azimuthR[i].getShape()); azimIndex[i] = azimArr[i].getIndex(); List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays_actual; jj++) { - azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + azimArr[i].setFloat(azimIndex[i].set(jj), -999.99f); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); + } } } @@ -695,13 +717,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ elevArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, elevationR[i].getShape()); elevIndex[i] = elevArr[i].getIndex(); List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays_actual; jj++) { - elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + elevArr[i].setFloat(elevIndex[i].set(jj), -999.99f); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); + } } } @@ -719,13 +746,16 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ for (int i = 0; i < number_sweeps; i++) { List rlist = sgp[i]; - int num_rays_actual = Math.min(num_rays, rlist.size()); - - for (int jj = 0; jj < num_rays_actual; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - ngates = rtemp[0].getBins(); - gatesArr.setInt(gatesIndex.set(i), ngates); + if (rlist == null) { + gatesArr.setInt(gatesIndex.set(i), -999); + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + ngates = rtemp[0].getBins(); + gatesArr.setInt(gatesIndex.set(i), ngates); + } } for (int i = 0; i < number_sweeps; i++) { From 6562aa6e28807732828d2aa0e07bb060edc4d03f Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Fri, 2 Feb 2024 15:19:44 +0100 Subject: [PATCH 11/16] cdm-radial: sigmet Data types with more than 1 byte fully supported A few small fixes --- .../main/java/ucar/nc2/iosp/sigmet/Ray.java | 189 ++++++++++++------ .../iosp/sigmet/SigmetIOServiceProvider.java | 147 ++++++++++---- .../nc2/iosp/sigmet/SigmetVolumeScan.java | 159 ++++++++++----- 3 files changed, 354 insertions(+), 141 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index d06afe8511..29f789abef 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -8,7 +8,9 @@ import ucar.ma2.IndexIterator; import ucar.ma2.Range; import ucar.unidata.io.RandomAccessFile; + import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Formatter; /** @@ -16,20 +18,23 @@ * @since Apr 7, 2010 */ public class Ray { - private short bins; + + private short bins, bins_actual; int dataRead; int offset; int offset1; private float range, step, az, elev; - private short time; + // int because UINT2 data type (Unsigned 16-bit integer) + private int time; // private float[] val; String varName; int nsweep; short datatype; + int bytesPerBin; - public Ray(float range, float step, float az, float elev, short bins, short time, int offset, int dataRead, - int offset1, int nsweep, String name, short datatype) { + public Ray(float range, float step, float az, float elev, short bins, short bins_actual, int time, int offset, + int dataRead, int offset1, int nsweep, String name, short datatype, int bytesPerBin) { // this.val = new float[bins]; setRange(range); @@ -37,6 +42,7 @@ public Ray(float range, float step, float az, float elev, short bins, short time setAz(az); setElev(elev); setBins(bins); + setBinsActual(bins_actual); setTime(time); setOffset(offset); setDataRead(dataRead); @@ -45,7 +51,7 @@ public Ray(float range, float step, float az, float elev, short bins, short time setName(name); setNsweep(nsweep); setDataType(datatype); - + setBytesPerBin(bytesPerBin); } public short getDataType() { @@ -56,6 +62,14 @@ public void setDataType(short datatype) { this.datatype = datatype; } + public int getBytesPerBin() { + return bytesPerBin; + } + + public void setBytesPerBin(int bytesPerBin) { + this.bytesPerBin = bytesPerBin; + } + public float getRange() { return range; } @@ -108,11 +122,19 @@ public void setBins(short bins) { this.bins = bins; } - public short getTime() { + public short getBinsActual() { + return bins_actual; + } + + public void setBinsActual(short bins_actual) { + this.bins_actual = bins_actual; + } + + public int getTime() { return time; } - public void setTime(short time) { + public void setTime(int time) { this.time = time; } @@ -163,7 +185,8 @@ public boolean equals(Object o) { } else if (o instanceof Ray) { Ray oo = (Ray) o; - return (range == oo.range & step == oo.step & az == oo.az & elev == oo.elev & bins == oo.bins & time == oo.time); + return (range == oo.range & step == oo.step & az == oo.az & elev == oo.elev & bins == oo.bins + & bins_actual == oo.bins_actual & time == oo.time & bytesPerBin == oo.bytesPerBin); } else { return false; } @@ -171,7 +194,8 @@ public boolean equals(Object o) { public int hashCode() { return new Float(range).hashCode() + new Float(step).hashCode() + new Float(az).hashCode() - + new Float(elev).hashCode() + new Short(bins).hashCode() + new Short(time).hashCode(); + + new Float(elev).hashCode() + new Short(bins).hashCode() + new Short(bins_actual).hashCode() + + new Integer(time).hashCode() + new Integer(bytesPerBin).hashCode(); // val.hashCode(); } @@ -184,7 +208,7 @@ public String toString() { az = 360.0f + az; } - sb.format(" Az=%f Elev=%f Bins=%d Time=%d", az, elev, bins, time); + sb.format(" Az=%f Elev=%f Bins=%d (%d) Time=%d", az, elev, bins, bins_actual, time); // for (int i=0; i 1) + Byte[] dd = new Byte[bins * bytesPerBin]; int nb = 0; short a00; short dty = getDataType(); - String var_name = SigmetVolumeScan.data_name[dty]; - // support for 2 byte data types is experimental - boolean twoBytes = var_name.endsWith("_2"); + + int posInRay_absolute = 0; // raf.readFully(data); if (dataRead > 0) { raf.seek(offset); for (int i = 0; i < dataRead; i++) { - if (!twoBytes) { - byte d = raf.readByte(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - } else { - short d = raf.readShort(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - } - nb++; + dd[i] = raf.readByte(); + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; } } raf.seek(offset1); int cur_len = offset1; + // this only works for 1 additional record, not for more. Only a theoretical possibility for now. + // (relevant only for data types with a lot of bytes per bin) while (nb < (int) bins) { // --- Check if the code=1 ("1" means an end of a ray) a00 = raf.readShort(); cur_len = cur_len + 2; + // end of ray if (a00 == (short) 1) { - for (int uk = 0; uk < (int) bins; uk++) { - dd[uk] = -999.99f; - } + // don't need to do anything as the rest of dd is already null break; } if (a00 < 0) { // -- This is data int nwords = a00 & 0x7fff; int dataRead1 = nwords * 2; - int pos = 0; - if (cur_len % REC_SIZE == 0) { + boolean breakOuterLoop = false; + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { break; } raf.seek(cur_len); for (int i = 0; i < dataRead1; i++) { - if (!twoBytes) { - byte d = raf.readByte(); - dd[nb] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - cur_len = cur_len + 1; - } else { - short d = raf.readShort(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, d); - cur_len = cur_len + 2; - } - nb = nb + 1; + dd[posInRay_absolute] = raf.readByte(); + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; + + cur_len++; - if (nb % REC_SIZE == 0) { - pos = i + 1; + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { + breakOuterLoop = true; break; } } - - if (pos > 0) { + if (breakOuterLoop) { break; } + } else if (a00 > 0 & a00 != 1) { int num_zero = a00 * 2; int dataRead1 = num_zero; for (int k = 0; k < dataRead1; k++) { - if (!twoBytes) - dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (byte) 0); - else - dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, (short) 0); + dd[posInRay_absolute] = 0; + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; } - nb = nb + dataRead1; - if (cur_len % REC_SIZE == 0) { + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { break; } - } + } // ------ end of while for num_bins--------------------------------- - if (!twoBytes) { + // only supporting float, double, byte and byte[]/Object for now + Class dtyClass = SigmetIOServiceProvider.calcDataClass(dty, bytesPerBin); + + if (dtyClass == Float.TYPE) { for (int gateIdx : gateRange) { if (gateIdx >= bins) ii.setFloatNext(Float.NaN); - else - ii.setFloatNext(dd[gateIdx].floatValue()); + else if (gateIdx >= bins_actual) + ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); + else + ii.setFloatNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, ddx)); + } } - } else { + + } else if (dtyClass == Double.TYPE) { for (int gateIdx : gateRange) { if (gateIdx >= bins) ii.setDoubleNext(Double.NaN); - else - ii.setDoubleNext(dd[gateIdx].doubleValue()); + else if (gateIdx >= bins_actual) + ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); + else { + int ddx2 = dd[2]; + int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); + ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); + } + } + } + + } else if (dtyClass == Byte.TYPE) { + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else if (gateIdx >= bins_actual) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else + ii.setByteNext(ddx); + } } - } + } else { // byte[]/Object + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else if (gateIdx >= bins_actual) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else { + byte[] b = new byte[bytesPerBin]; + for (int i = 0; i < bytesPerBin; i++) { + ddx = dd[offset + i]; + b[i] = ddx == null ? SigmetVolumeScan.MISSING_VALUE_BYTE : ddx; + } + ii.setObjectNext(ByteBuffer.wrap(b)); + } + } + } + } } // end of readData } // class Ray end------------------------------------------ diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index aa3c31c4a2..b3a07ca8ce 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -352,18 +352,33 @@ public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile var_name_original = RAW_VARIABLE_PREFIX + var_name_original; var_name = var_name_original; - // support for 2 byte data types is experimental - boolean twoBytes = var_name.endsWith("_2"); + int bytesPerBin = 1; + List> allRays = volScan.getGroup(data_name[dty]); + if (allRays.size() > 0 && allRays.get(0).size() > 0) + bytesPerBin = allRays.get(0).get(0).getBytesPerBin(); + + Class dtyClass = SigmetIOServiceProvider.calcDataClass(dty, bytesPerBin); for (int jj = 0; jj < number_sweeps; jj++) { if (number_sweeps > 1) { var_name = var_name_original + "_sweep_" + (jj + 1); } v[j][jj] = new Variable(ncfile, null, null, var_name); - if (twoBytes) - v[j][jj].setDataType(DataType.DOUBLE); - else + if (dtyClass == Float.TYPE) { v[j][jj].setDataType(DataType.FLOAT); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_FLOAT)); + } else if (dtyClass == Double.TYPE) { + v[j][jj].setDataType(DataType.DOUBLE); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_DOUBLE)); + } else if (dtyClass == Byte.TYPE) { + v[j][jj].setDataType(DataType.BYTE); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_BYTE)); + } else { // byte[]/Object + v[j][jj].setDataType(DataType.OPAQUE); + Attribute a = new Attribute(CDM.MISSING_VALUE, DataType.OPAQUE); + v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY)); + } + dims2.add(radial); dims2.add(gateR[jj]); v[j][jj].setDimensions(dims2); @@ -371,7 +386,6 @@ public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[dty])); String coordinates = "time elevationR azimuthR distanceR"; v[j][jj].addAttribute(new Attribute(_Coordinate.Axes, coordinates)); - v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, -999.99f)); ncfile.addVariable(null, v[j][jj]); varList.add(v[j][jj]); dims2.clear(); @@ -536,7 +550,6 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ ArrayList varList, Map recHdr) { // prepare attribute values - String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; String def_datafile = "SIGMET-IRIS"; Short header_length = 80; Short ray_header_length = 6; @@ -837,23 +850,49 @@ public Array readData(Variable v2, Section section) throws IOException { private void readOneScan(List mapScan, Range radialRange, Range gateRange, IndexIterator ii) throws IOException { int siz = mapScan.size(); + short dataType = 1; // TotalPower as fallback + int bytesPerBin = 1; + for (int radialIdx : radialRange) { if (radialIdx >= siz) - readOneRadial(null, gateRange, ii); + readOneRadialMissing(dataType, bytesPerBin, gateRange, ii); else { Ray r = mapScan.get(radialIdx); + dataType = r.datatype; + bytesPerBin = r.bytesPerBin; readOneRadial(r, gateRange, ii); } } } private void readOneRadial(Ray r, Range gateRange, IndexIterator ii) throws IOException { - if (r == null || r.offset == -999) { - for (int i = 0; i < gateRange.length(); i++) + r.readData(volScan.raf, gateRange, ii); + } + + private void readOneRadialMissing(short datatype, int bytesPerBin, Range gateRange, IndexIterator ii) + throws IOException { + Class dtyClass = calcDataClass(datatype, bytesPerBin); + + if (dtyClass == Float.TYPE) { + for (int i = 0; i < gateRange.length(); i++) { ii.setFloatNext(Float.NaN); - return; + } + + } else if (dtyClass == Double.TYPE) { + for (int i = 0; i < gateRange.length(); i++) { + ii.setDoubleNext(Double.NaN); + } + + } else if (dtyClass == Byte.TYPE) { + for (int i = 0; i < gateRange.length(); i++) { + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + } + + } else { // byte[]/Object + for (int i = 0; i < gateRange.length(); i++) { + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + } } - r.readData(volScan.raf, gateRange, ii); } /** @@ -1019,12 +1058,44 @@ static float calcAz(short az0, short az1) { return result.floatValue(); } + /** + * Calculate Object class for data type + * + * @see Ray + * + * @param dty type of data (@see SigmetVolumeScan.data_name) + * @return float value with precision of two decimal + */ + static Class calcDataClass(short dty, int bytes) { + switch (dty) { + case 1:// dty=1,2 -total_power, reflectivity (dBZ) + case 2: + case 3: // dty=3 - mean velocity (m/sec) + case 4: // dty=4 - spectrum width (m/sec) + case 5: // dty=5 - differential reflectivity (dB) + return Float.TYPE; + + case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) + case 8: + case 9: // dty=9 - mean velocity 2 (m/sec) + case 10: // dty=10 - spectrum width 2 (m/sec) + case 11: // dty=11 - differential reflectivity 2 (dB) + return Double.TYPE; + + default: + // TODO implement for more SigmetVolumeScan.data_name (see chapter 4.4) + if (bytes == 1) + return Byte.TYPE; + + return byte[].class; + } + } + /** * Calculate data values from raw ingest data * * @param recHdr java.util.Map object with values for calculation - * @param dty type of data ( "Total_Power", "Reflectivity", "Velocity", - * "Width", "Differential_Reflectivity") + * @param dty type of data (@see SigmetVolumeScan.data_name) * @param data 1-byte input value * @return float value with precision of two decimal */ @@ -1032,32 +1103,38 @@ static float calcData(Map recHdr, short dty, byte data) { short[] coef = {1, 2, 3, 4}; // MultiPRF modes short multiprf = recHdr.get("multiprf").shortValue(); float vNyq = recHdr.get("vNyq").floatValue(); - double temp = -999.99; + double temp = SigmetVolumeScan.MISSING_VALUE_FLOAT; + + // see chapter 4.4 switch (dty) { case 1:// dty=1,2 -total_power, reflectivity (dBZ) case 2: - if (data != 0) { + if (data != 0 && data != (byte) 255) { temp = (((int) data & 0xFF) - 64) * 0.5; } break; case 3: // dty=3 - mean velocity (m/sec) if (data != 0) { + // seems incorrect according to "4.4.44 1-byte Velocity Format (DB_VEL)" + // TODO needs more research temp = ((((int) data & 0xFF) - 128) / 127.0) * vNyq * coef[multiprf]; } break; case 4: // dty=4 - spectrum width (m/sec) - if (data != 0) { + if (data != 0 && data != (byte) 255) { + // seems incorrect according to "4.4.48 1-byte Width Format (DB_WIDTH)" + // TODO needs more research double v = ((((int) data & 0xFF) - 128) / 127.0) * vNyq * coef[multiprf]; temp = (((int) data & 0xFF) / 256.0) * v; } break; case 5: // dty=5 - differential reflectivity (dB) - if (data != 0) { + if (data != 0 && data != (byte) 255) { temp = ((((int) data & 0xFF) - 128) / 16.0); } break; default: - // TODO implement for more SigmetVolumeScan.data_name (only 1 byte) + // TODO implement for more SigmetVolumeScan.data_name (only 1 byte) (see chapter 4.4) // using only the raw value temp = (int) data & 0xFF; @@ -1076,42 +1153,38 @@ static float calcData(Map recHdr, short dty, byte data) { * Calculate data values from raw ingest data * * @param recHdr java.util.Map object with values for calculation - * @param dty type of data ( "Total_Power", "Reflectivity", "Velocity", - * "Width", "Differential_Reflectivity") - * @param data 2-byte input value + * @param dty type of data (@see SigmetVolumeScan.data_name) + * @param data 2-byte input value (unsigned short) * @return double value */ - static double calcData(Map recHdr, short dty, short data) { - short[] coef = {1, 2, 3, 4}; // MultiPRF modes - short multiprf = recHdr.get("multiprf").shortValue(); - float vNyq = recHdr.get("vNyq").floatValue(); - double temp = -999.99; + static double calcData(Map recHdr, short dty, int data) { + double temp = SigmetVolumeScan.MISSING_VALUE_DOUBLE; + // see chapter 4.4 switch (dty) { case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) case 8: - if (data != 0) { - temp = (data - 64) * 0.5; + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; } break; case 9: // dty=9 - mean velocity 2 (m/sec) - if (data != 0) { - temp = ((data - 128) / 127.0) * vNyq * coef[multiprf]; + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; } break; case 10: // dty=10 - spectrum width 2 (m/sec) - if (data != 0) { - double v = ((data - 128) / 127.0) * vNyq * coef[multiprf]; - temp = (data / 256.0) * v; + if (data != 0 && data != 65535) { + temp = data / 100d; } break; case 11: // dty=11 - differential reflectivity 2 (dB) - if (data != 0) { - temp = ((data - 128) / 16.0); + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; } break; default: - // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) + // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) (see chapter 4.4) // using only the raw value temp = data; // logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 17d86de109..8437b1e629 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -10,9 +10,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.DataType; import ucar.nc2.Variable; import ucar.unidata.io.RandomAccessFile; +import java.nio.ByteBuffer; import java.util.*; /** @@ -132,6 +135,16 @@ public class SigmetVolumeScan { "?", // AZDR16 (2 bytes) }; + public static final int REC_SIZE = 6144; + + public static final float MISSING_VALUE_FLOAT = -999.99f; + public static final double MISSING_VALUE_DOUBLE = -999.99; + public static final byte MISSING_VALUE_BYTE = 0; + + public static final ByteBuffer MISSING_VALUE_BYTE_ARRAY_BB = ByteBuffer.wrap(new byte[] {MISSING_VALUE_BYTE}); + public static final Array MISSING_VALUE_BYTE_ARRAY = + Array.factoryConstant(DataType.OPAQUE, new int[] {1}, new ByteBuffer[] {MISSING_VALUE_BYTE_ARRAY_BB}); + private HashMap>> allGroups = new HashMap<>(); private short[] data_type; @@ -153,12 +166,14 @@ public class SigmetVolumeScan { */ SigmetVolumeScan(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) throws java.io.IOException { - int REC_SIZE = 6144; int len = 12288; // ---- Read from the 3d record----------- 6144*2=12288 short nrec = 0, nsweep = 1, nray = 0, byteoff = 0; - int nwords, end_words, data_read = 0, num_zero, rays_count = 0, nb = 0, pos = 0, pos_ray_hdr = 0, t = 0; + int nwords, end_words, data_read = 0, num_zero, rays_count = 0, nb = 0, posInRay_relative = 0, + posInRay_absolute = 0, pos_ray_hdr = 0, t = 0; short a0, a00, dty; - short beg_az = 0, beg_elev = 0, end_az = 0, end_elev = 0, num_bins = 0, time_start_sw = 0; + short beg_az = 0, beg_elev = 0, end_az = 0, end_elev = 0, num_bins = 0; + // int because UINT2 data type (Unsigned 16-bit integer) + int time_start_sw = 0; float az, elev, d = 0.0f, step; // byte data = 0; boolean beg_rec = true, end_rec = true, read_ray_hdr = true, begin = true; @@ -270,10 +285,8 @@ public class SigmetVolumeScan { num_rays_exp[i] = raf.readShort(); beg += num_rays_exp[i]; // before num_rays_act[i] was used but it does seem not work num_rays_act[i] = raf.readShort(); - // beg += num_rays_act[i]; // idh_len+20 angl_swp[i] = raf.readShort(); // idh_len+22 - // TODO maybe use in stead of variable twoBytes? - bin_len[i] = raf.readShort(); // idh_len+24 + bin_len[i] = raf.readShort(); // idh_len+24 (Number of bits per bin for this data type) data_type[i] = raf.readShort(); // idh_len+26 } @@ -294,7 +307,8 @@ public class SigmetVolumeScan { end_rec = true; rays_count++; read_ray_hdr = true; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; len = cur_len; @@ -308,7 +322,7 @@ public class SigmetVolumeScan { } nwords = a0 & 0x7fff; - end_words = nwords - 6; + end_words = nwords - 6; // because of raw_prod_bhdr 12-byte structure data_read = end_words * 2; end_rec = false; @@ -325,19 +339,27 @@ public class SigmetVolumeScan { // ---Define output data files for each data_type (= nparams)/sweep --------- dty = data_type[0]; + int bitsToRead = bin_len[0]; if (nparams > 1) { kk = rays_count % nparams; dty = data_type[kk]; + bitsToRead = bin_len[kk]; + } else if (number_sweeps > 1) { } + if (bitsToRead <= 0 || bitsToRead % 8 != 0) + throw new IllegalStateException("Not compatible! Number of bits per bin for this data type = " + bitsToRead); + int bytesToRead = bitsToRead / 8; + String var_name = data_name[dty]; - // support for 2 byte data types is experimental - boolean twoBytes = var_name.endsWith("_2"); // --- read ray_header (size=12 bytes=6 words)--------------------------------------- if (read_ray_hdr) { + if (ray != null) + throw new IllegalStateException("ray != null"); + if (pos_ray_hdr < 2) { raf.seek(cur_len); beg_az = raf.readShort(); @@ -414,7 +436,7 @@ public class SigmetVolumeScan { if (pos_ray_hdr < 12) { raf.seek(cur_len); - time_start_sw = raf.readShort(); + time_start_sw = raf.readUnsignedShort(); cur_len = cur_len + 2; len = cur_len; } @@ -433,9 +455,9 @@ public class SigmetVolumeScan { continue; } - if (pos > 0) { - data_read = data_read - pos; - pos = 0; + if (posInRay_relative > 0) { + data_read = data_read - posInRay_relative; + posInRay_relative = 0; } if (data_read > 0) { @@ -447,14 +469,12 @@ public class SigmetVolumeScan { // data = raf.readByte(); // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); cur_len++; - nb++; - if (twoBytes) { - cur_len++; - // i++;? - } + posInRay_absolute++; + if (posInRay_absolute % bytesToRead == 0) + nb++; if (cur_len % REC_SIZE == 0) { - pos = i + 1; + posInRay_relative = i + 1; beg_rec = true; read_ray_hdr = false; len = cur_len; @@ -463,13 +483,13 @@ public class SigmetVolumeScan { } } raf.seek(cur_len); - if (pos > 0) { + if (posInRay_relative > 0) { continue; } } if (cur_len % REC_SIZE == 0) { - pos = 0; + posInRay_relative = 0; beg_rec = true; read_ray_hdr = false; data_read = 0; @@ -487,11 +507,8 @@ public class SigmetVolumeScan { // --- Check if the code=1 ("1" means an end of a ray) if (a00 == (short) 1) { - // for (int uk = 0; uk < (int) num_bins; uk++) { - // dd[uk] = -999.99f; - // } - ray = new Ray(-999.99f, -999.99f, -999.99f, -999.99f, num_bins, (short) (-99), -999, 0, -999, nsweep, - var_name, dty); + ray = new Ray(range_first, step, az, elev, num_bins, (short) nb, time_start_sw, rayoffset, datalen, + rayoffset1, nsweep, var_name, dty, bytesToRead); rays_count++; beg_rec = false; end_rec = true; @@ -504,7 +521,7 @@ public class SigmetVolumeScan { data_read = nwords * 2; if (cur_len % REC_SIZE == 0) { - pos = 0; + posInRay_relative = 0; beg_rec = true; end_rec = false; read_ray_hdr = false; @@ -517,15 +534,13 @@ public class SigmetVolumeScan { for (int ii = 0; ii < data_read; ii++) { // data = raf.readByte(); // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); - cur_len = cur_len + 1; - nb = nb + 1; - if (twoBytes) { - cur_len++; - // ii++;? - } + cur_len++; + posInRay_absolute++; + if (posInRay_absolute % bytesToRead == 0) + nb++; if (cur_len % REC_SIZE == 0) { - pos = ii + 1; + posInRay_relative = ii + 1; beg_rec = true; end_rec = false; read_ray_hdr = false; @@ -534,7 +549,7 @@ public class SigmetVolumeScan { } } raf.seek(cur_len); - if (pos > 0) { + if (posInRay_relative > 0) { break; } } else if (a00 > 0 & a00 != 1) { @@ -544,14 +559,19 @@ public class SigmetVolumeScan { // dd[nb + k] = SigmetIOServiceProvider.calcData(recHdr, dty, (byte) 0); // } - nb = nb + num_zero; - // TODO extra handling for twoBytes here, too? + int nb_before = posInRay_absolute / bytesToRead; + // sanity check + if (nb_before != nb) + throw new IllegalStateException("nb_before != nb"); + posInRay_absolute += num_zero; + int nb_after = posInRay_absolute / bytesToRead; + nb = nb + (nb_after - nb_before); if (cur_len % REC_SIZE == 0) { beg_rec = true; end_rec = false; read_ray_hdr = false; - pos = 0; + posInRay_relative = 0; data_read = 0; break; @@ -562,20 +582,52 @@ public class SigmetVolumeScan { if (cur_len % REC_SIZE == 0) { len = cur_len; + // will get lost otherwise + if (ray != null) { + beg_rec = true; + end_rec = true; + read_ray_hdr = true; + posInRay_relative = 0; + posInRay_absolute = 0; + data_read = 0; + nb = 0; + len = cur_len; + + if (ray == null) + throw new IllegalStateException("ray == null"); + + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); + } + varL.add(ray); + lastRay = ray; + ray = null; + } + continue; } raf.seek(cur_len); if (nb == (int) num_bins) { - a00 = raf.readShort(); + a00 = raf.readShort(); // should be 1 == end of ray + if (a00 != 1) + // should not be able to get here + logger.warn("nb == num_bins but a00 != 1 - something seems wrong"); cur_len = cur_len + 2; end_rec = true; - ray = new Ray(range_first, step, az, elev, num_bins, time_start_sw, rayoffset, datalen, rayoffset1, nsweep, - var_name, dty); + ray = new Ray(range_first, step, az, elev, num_bins, (short) nb, time_start_sw, rayoffset, datalen, rayoffset1, + nsweep, var_name, dty, bytesToRead); rays_count++; + // last ray of last sweep -> end of file if ((nsweep == number_sweeps) & (rays_count % beg == 0)) { + if (ray == null) + throw new IllegalStateException("ray == null"); + // using a universal structure that works for all data types List varL = all.get(var_name.trim()); if (varL == null) { @@ -583,6 +635,8 @@ public class SigmetVolumeScan { all.put(var_name.trim(), varL); } varL.add(ray); + lastRay = ray; + ray = null; break; } @@ -590,11 +644,15 @@ public class SigmetVolumeScan { beg_rec = true; end_rec = true; read_ray_hdr = true; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; len = cur_len; + if (ray == null) + throw new IllegalStateException("ray == null"); + // using a universal structure that works for all data types List varL = all.get(var_name.trim()); if (varL == null) { @@ -602,6 +660,8 @@ public class SigmetVolumeScan { all.put(var_name.trim(), varL); } varL.add(ray); + lastRay = ray; + ray = null; continue; } @@ -610,6 +670,9 @@ public class SigmetVolumeScan { if (firstRay == null) firstRay = ray; + if (ray == null) + throw new IllegalStateException("ray == null"); + // using a universal structure that works for all data types List additionalL = all.get(var_name.trim()); if (additionalL == null) { @@ -617,8 +680,11 @@ public class SigmetVolumeScan { all.put(var_name.trim(), additionalL); } additionalL.add(ray); + lastRay = ray; + ray = null; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; read_ray_hdr = true; @@ -635,7 +701,6 @@ public class SigmetVolumeScan { len = cur_len; } // ------------end of outer while --------------- - lastRay = ray; // using a universal structure that works for all data types for (String var_name : all.keySet()) { @@ -737,7 +802,7 @@ public int[] getStartSweep() { */ void checkSort(Ray[] r) { int j = 0, n = 0, n1, n2; - short time1, time2; + int time1, time2; int[] k1 = new int[300]; int[] k2 = new int[300]; // define the groups of rays with the same "time". For ex.: @@ -753,7 +818,7 @@ void checkSort(Ray[] r) { k1[j] = i + 1; } } - if (k2[j] < r.length - 1) { + if (k2[j] < r.length - 1 && j > 0) { k1[j] = k2[j - 1] + 1; k2[j] = r.length - 1; } From 9e09321763fd636c744bdc487e0648ab9fe06564 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 3 Feb 2024 11:08:04 +0100 Subject: [PATCH 12/16] cdm-radial: sigmet Fix for 2 byte data types (was able to find an example file finally) --- cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index 29f789abef..099c2488a0 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -332,7 +332,7 @@ else if (gateIdx >= bins_actual) if (ddx == null) ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); else { - int ddx2 = dd[2]; + int ddx2 = dd[offset+1]; int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); } From 1d671cb80e9a8d1d5874e890e3a779e3c3905fca Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sat, 3 Feb 2024 11:13:24 +0100 Subject: [PATCH 13/16] code style... :/ --- cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index 099c2488a0..41eb584fca 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -332,7 +332,7 @@ else if (gateIdx >= bins_actual) if (ddx == null) ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); else { - int ddx2 = dd[offset+1]; + int ddx2 = dd[offset + 1]; int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); } From b70e345f07c136e8fbb93e3f98c65056168d8786 Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Sun, 4 Feb 2024 12:25:30 +0100 Subject: [PATCH 14/16] cdm-radial: sigmet Fix for reading data from ray for only the number of bins available --- .../src/main/java/ucar/nc2/iosp/sigmet/Ray.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index 41eb584fca..be6185b9e0 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -268,7 +268,7 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th break; } raf.seek(cur_len); - for (int i = 0; i < dataRead1; i++) { + for (int i = 0; i < dataRead1 && nb < bins; i++) { dd[posInRay_absolute] = raf.readByte(); posInRay_absolute++; if (posInRay_absolute % bytesPerBin == 0) @@ -288,14 +288,15 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th } else if (a00 > 0 & a00 != 1) { int num_zero = a00 * 2; int dataRead1 = num_zero; - for (int k = 0; k < dataRead1; k++) { + for (int k = 0; k < dataRead1 && nb < bins; k++) { dd[posInRay_absolute] = 0; posInRay_absolute++; if (posInRay_absolute % bytesPerBin == 0) nb++; - } - if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { - break; + + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { + break; + } } } From b89cf7719e990f6aa23cc3b23ea880ba02b61938 Mon Sep 17 00:00:00 2001 From: Hailey Johnson Date: Mon, 11 Mar 2024 14:14:55 -0700 Subject: [PATCH 15/16] PR feedback and cleanup --- .../main/java/ucar/nc2/iosp/sigmet/Ray.java | 139 +++++++++--------- .../iosp/sigmet/SigmetIOServiceProvider.java | 130 +++++++--------- .../nc2/iosp/sigmet/SigmetVolumeScan.java | 8 +- 3 files changed, 121 insertions(+), 156 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index be6185b9e0..861003b0e9 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -5,6 +5,7 @@ package ucar.nc2.iosp.sigmet; +import ucar.ma2.DataType; import ucar.ma2.IndexIterator; import ucar.ma2.Range; import ucar.unidata.io.RandomAccessFile; @@ -32,11 +33,14 @@ public class Ray { short datatype; int bytesPerBin; + public Ray(float range, float step, float az, float elev, short bins, int time, int offset, int dataRead, int offset1, + int nsweep, String name, short datatype, int bytesPerBin) { + this(range, step, az, elev, bins, bins, time, offset, dataRead, offset1, nsweep, name, datatype, 1); + } public Ray(float range, float step, float az, float elev, short bins, short bins_actual, int time, int offset, int dataRead, int offset1, int nsweep, String name, short datatype, int bytesPerBin) { - // this.val = new float[bins]; setRange(range); setStep(step); setAz(az); @@ -162,15 +166,6 @@ public void setOffset1(int offset1) { this.offset1 = offset1; } - /* - * public float[] getVal() { - * return val; - * } - * - * public void setVal(float[] val) { - * System.arraycopy(val, 0, this.val, 0, bins); - * } - */ public void setName(String name) { this.varName = name; } @@ -223,7 +218,6 @@ public String toString() { * @throws IOException on read error */ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) throws IOException { - raf.seek(offset); // first need to read all the data (because bytesPerBin could be > 1) Byte[] dd = new Byte[bins * bytesPerBin]; int nb = 0; @@ -233,7 +227,6 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th int posInRay_absolute = 0; - // raf.readFully(data); if (dataRead > 0) { raf.seek(offset); @@ -303,80 +296,80 @@ public void readData(RandomAccessFile raf, Range gateRange, IndexIterator ii) th } // ------ end of while for num_bins--------------------------------- // only supporting float, double, byte and byte[]/Object for now - Class dtyClass = SigmetIOServiceProvider.calcDataClass(dty, bytesPerBin); - - if (dtyClass == Float.TYPE) { - for (int gateIdx : gateRange) { - if (gateIdx >= bins) - ii.setFloatNext(Float.NaN); - else if (gateIdx >= bins_actual) - ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); - else { - int offset = gateIdx * bytesPerBin; - Byte ddx = dd[offset]; - if (ddx == null) + DataType dType = SigmetIOServiceProvider.calcDataType(dty, bytesPerBin); + switch (dType) { + case FLOAT: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setFloatNext(Float.NaN); + else if (gateIdx >= bins_actual) ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); - else - ii.setFloatNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, ddx)); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); + else + ii.setFloatNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, ddx)); + } } - } - - } else if (dtyClass == Double.TYPE) { - for (int gateIdx : gateRange) { - if (gateIdx >= bins) - ii.setDoubleNext(Double.NaN); - else if (gateIdx >= bins_actual) - ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); - else { - int offset = gateIdx * bytesPerBin; - Byte ddx = dd[offset]; - if (ddx == null) + break; + case DOUBLE: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setDoubleNext(Double.NaN); + else if (gateIdx >= bins_actual) ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); else { - int ddx2 = dd[offset + 1]; - int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); - ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); + else { + int ddx2 = dd[offset + 1]; + int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); + ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); + } } } - } - - } else if (dtyClass == Byte.TYPE) { - for (int gateIdx : gateRange) { - if (gateIdx >= bins) - ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); - else if (gateIdx >= bins_actual) - ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); - else { - int offset = gateIdx * bytesPerBin; - Byte ddx = dd[offset]; - if (ddx == null) + break; + case BYTE: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else if (gateIdx >= bins_actual) ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); - else - ii.setByteNext(ddx); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else + ii.setByteNext(ddx); + } } - } - - } else { // byte[]/Object - for (int gateIdx : gateRange) { - if (gateIdx >= bins) - ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); - else if (gateIdx >= bins_actual) - ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); - else { - int offset = gateIdx * bytesPerBin; - Byte ddx = dd[offset]; - if (ddx == null) + break; + default: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else if (gateIdx >= bins_actual) ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); else { - byte[] b = new byte[bytesPerBin]; - for (int i = 0; i < bytesPerBin; i++) { - ddx = dd[offset + i]; - b[i] = ddx == null ? SigmetVolumeScan.MISSING_VALUE_BYTE : ddx; + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else { + byte[] b = new byte[bytesPerBin]; + for (int i = 0; i < bytesPerBin; i++) { + ddx = dd[offset + i]; + b[i] = ddx == null ? SigmetVolumeScan.MISSING_VALUE_BYTE : ddx; + } + ii.setObjectNext(ByteBuffer.wrap(b)); } - ii.setObjectNext(ByteBuffer.wrap(b)); } } - } } } // end of readData } // class Ray end------------------------------------------ diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index b3a07ca8ce..b5d0f9c9ea 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -5,6 +5,7 @@ package ucar.nc2.iosp.sigmet; +import org.checkerframework.checker.units.qual.A; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.ma2.*; @@ -126,12 +127,8 @@ public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ucar.nc2.util super.open(raf, ncfile, cancelTask); // java.util.Map recHdr=new java.util.HashMap(); Map hdrNames = new java.util.HashMap<>(); - volScan = new SigmetVolumeScan(raf, ncfile, varList); + volScan = new SigmetVolumeScan(raf); this.varList = init(raf, ncfile, hdrNames); - - // doData(raf, ncfile, varList); - // raf.close(); - // this.ncfile.close(); } /** @@ -357,35 +354,39 @@ public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile if (allRays.size() > 0 && allRays.get(0).size() > 0) bytesPerBin = allRays.get(0).get(0).getBytesPerBin(); - Class dtyClass = SigmetIOServiceProvider.calcDataClass(dty, bytesPerBin); + DataType dType = calcDataType(dty, bytesPerBin); + Number missingVal = null; + switch (dType) { + case FLOAT: + missingVal = SigmetVolumeScan.MISSING_VALUE_FLOAT; + break; + case DOUBLE: + missingVal = SigmetVolumeScan.MISSING_VALUE_DOUBLE; + break; + case BYTE: + missingVal = SigmetVolumeScan.MISSING_VALUE_BYTE; + + } for (int jj = 0; jj < number_sweeps; jj++) { if (number_sweeps > 1) { var_name = var_name_original + "_sweep_" + (jj + 1); } - v[j][jj] = new Variable(ncfile, null, null, var_name); - if (dtyClass == Float.TYPE) { - v[j][jj].setDataType(DataType.FLOAT); - v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_FLOAT)); - } else if (dtyClass == Double.TYPE) { - v[j][jj].setDataType(DataType.DOUBLE); - v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_DOUBLE)); - } else if (dtyClass == Byte.TYPE) { - v[j][jj].setDataType(DataType.BYTE); - v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_BYTE)); - } else { // byte[]/Object - v[j][jj].setDataType(DataType.OPAQUE); - Attribute a = new Attribute(CDM.MISSING_VALUE, DataType.OPAQUE); - v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY)); + Variable.Builder builder = Variable.builder().setNcfile(ncfile).setName(var_name).setDataType(dType); + Attribute.Builder attr = Attribute.builder().setName(CDM.MISSING_VALUE); + if (missingVal != null) { + attr.setNumericValue(missingVal, false); + } else { + attr.setValues(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY); } + builder.addAttribute(attr.build()); dims2.add(radial); dims2.add(gateR[jj]); - v[j][jj].setDimensions(dims2); - v[j][jj].addAttribute(new Attribute(CDM.LONG_NAME, var_name)); - v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[dty])); - String coordinates = "time elevationR azimuthR distanceR"; - v[j][jj].addAttribute(new Attribute(_Coordinate.Axes, coordinates)); + builder.setDimensions(dims2).addAttribute(new Attribute(CDM.LONG_NAME, var_name)) + .addAttribute(new Attribute(CDM.UNITS, unit[dty])) + .addAttribute(new Attribute(_Coordinate.Axes, "time elevationR azimuthR distanceR")); + v[j][jj] = builder.build(null); ncfile.addVariable(null, v[j][jj]); varList.add(v[j][jj]); dims2.clear(); @@ -550,23 +551,14 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ ArrayList varList, Map recHdr) { // prepare attribute values - String def_datafile = "SIGMET-IRIS"; - Short header_length = 80; - Short ray_header_length = 6; int ngates; - float radar_lat = recHdr.get("radar_lat").floatValue(); - float radar_lon = recHdr.get("radar_lon").floatValue(); - short ground_height = recHdr.get("ground_height").shortValue(); - short radar_height = recHdr.get("radar_height").shortValue(); - int radar_alt = (recHdr.get("radar_alt").intValue()) / 100; short num_rays = recHdr.get("num_rays").shortValue(); float range_first = (recHdr.get("range_first").intValue()) * 0.01f; float range_last = (recHdr.get("range_last").intValue()) * 0.01f; short number_sweeps = recHdr.get("number_sweeps").shortValue(); int nparams = (recHdr.get("nparams").intValue()); - // define date/time - // int last_t=(int)(ray[nparams*number_sweeps-1][num_rays-1].getTime()); + int last_t = volScan.lastRay.getTime(); String sss1 = Short.toString(m[0]); if (sss1.length() < 2) @@ -589,11 +581,6 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ // set all of Variables try { - int sz = varList.size(); - - ArrayFloat.D2[] dataArr = new ArrayFloat.D2[nparams * number_sweeps]; - Index[] dataIndex = new Index[nparams * number_sweeps]; - Ray[] rtemp = new Ray[(int) num_rays]; Variable[] distanceR = new Variable[number_sweeps]; @@ -869,29 +856,28 @@ private void readOneRadial(Ray r, Range gateRange, IndexIterator ii) throws IOEx r.readData(volScan.raf, gateRange, ii); } - private void readOneRadialMissing(short datatype, int bytesPerBin, Range gateRange, IndexIterator ii) - throws IOException { - Class dtyClass = calcDataClass(datatype, bytesPerBin); - - if (dtyClass == Float.TYPE) { - for (int i = 0; i < gateRange.length(); i++) { - ii.setFloatNext(Float.NaN); - } - - } else if (dtyClass == Double.TYPE) { - for (int i = 0; i < gateRange.length(); i++) { - ii.setDoubleNext(Double.NaN); - } - - } else if (dtyClass == Byte.TYPE) { - for (int i = 0; i < gateRange.length(); i++) { - ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); - } - - } else { // byte[]/Object - for (int i = 0; i < gateRange.length(); i++) { - ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); - } + private void readOneRadialMissing(short datatype, int bytesPerBin, Range gateRange, IndexIterator ii) { + DataType dType = calcDataType(datatype, bytesPerBin); + switch (dType) { + case FLOAT: + for (int i = 0; i < gateRange.length(); i++) { + ii.setFloatNext(Float.NaN); + } + break; + case DOUBLE: + for (int i = 0; i < gateRange.length(); i++) { + ii.setDoubleNext(Double.NaN); + } + break; + case BYTE: + for (int i = 0; i < gateRange.length(); i++) { + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + } + break; + default: // byte[]/Object + for (int i = 0; i < gateRange.length(); i++) { + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + } } } @@ -1058,36 +1044,28 @@ static float calcAz(short az0, short az1) { return result.floatValue(); } - /** - * Calculate Object class for data type - * - * @see Ray - * - * @param dty type of data (@see SigmetVolumeScan.data_name) - * @return float value with precision of two decimal - */ - static Class calcDataClass(short dty, int bytes) { + static DataType calcDataType(short dty, int bytes) { switch (dty) { case 1:// dty=1,2 -total_power, reflectivity (dBZ) case 2: case 3: // dty=3 - mean velocity (m/sec) case 4: // dty=4 - spectrum width (m/sec) case 5: // dty=5 - differential reflectivity (dB) - return Float.TYPE; + return DataType.FLOAT; case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) case 8: case 9: // dty=9 - mean velocity 2 (m/sec) case 10: // dty=10 - spectrum width 2 (m/sec) case 11: // dty=11 - differential reflectivity 2 (dB) - return Double.TYPE; + return DataType.DOUBLE; default: // TODO implement for more SigmetVolumeScan.data_name (see chapter 4.4) if (bytes == 1) - return Byte.TYPE; + return DataType.BYTE; - return byte[].class; + return DataType.OBJECT; } } diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 8437b1e629..ddd4529889 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -6,8 +6,6 @@ package ucar.nc2.iosp.sigmet; -// ~--- non-JDK imports -------------------------------------------------------- - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.ma2.Array; @@ -160,12 +158,8 @@ public class SigmetVolumeScan { /** * Read all the values from SIGMET-IRIS file which are necessary to fill in the ncfile. * - * @param raf ucar.unidata.io.RandomAccessFile corresponds to SIGMET datafile. - * @param ncfile an empty NetcdfFile object which will be filled. - * @param varList ArrayList of Variables of ncfile */ - SigmetVolumeScan(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) - throws java.io.IOException { + SigmetVolumeScan(RandomAccessFile raf) throws java.io.IOException { int len = 12288; // ---- Read from the 3d record----------- 6144*2=12288 short nrec = 0, nsweep = 1, nray = 0, byteoff = 0; int nwords, end_words, data_read = 0, num_zero, rays_count = 0, nb = 0, posInRay_relative = 0, From d17603ce2ffbf10d2ae8f7aaec255e11033c967a Mon Sep 17 00:00:00 2001 From: Michael Diener Date: Tue, 12 Mar 2024 10:28:00 +0100 Subject: [PATCH 16/16] cdm-radial: sigmet Fixes for change "PR feedback and cleanup" b89cf77 --- .../java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index b5d0f9c9ea..2c812a95d9 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -386,7 +386,7 @@ public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile builder.setDimensions(dims2).addAttribute(new Attribute(CDM.LONG_NAME, var_name)) .addAttribute(new Attribute(CDM.UNITS, unit[dty])) .addAttribute(new Attribute(_Coordinate.Axes, "time elevationR azimuthR distanceR")); - v[j][jj] = builder.build(null); + v[j][jj] = builder.build(ncfile.getRootGroup()); ncfile.addVariable(null, v[j][jj]); varList.add(v[j][jj]); dims2.clear(); @@ -1065,7 +1065,7 @@ static DataType calcDataType(short dty, int bytes) { if (bytes == 1) return DataType.BYTE; - return DataType.OBJECT; + return DataType.OPAQUE; } }