diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java index f6e557231b..861003b0e9 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/Ray.java @@ -5,10 +5,13 @@ package ucar.nc2.iosp.sigmet; +import ucar.ma2.DataType; import ucar.ma2.IndexIterator; import ucar.ma2.Range; import ucar.unidata.io.RandomAccessFile; + import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Formatter; /** @@ -16,27 +19,34 @@ * @since Apr 7, 2010 */ public class Ray { - private short bins; + + private short bins, bins_actual; int dataRead; int offset; int offset1; private float range, step, az, elev; - private short time; + // int because UINT2 data type (Unsigned 16-bit integer) + private int time; // private float[] val; String varName; int nsweep; short datatype; + int bytesPerBin; + public Ray(float range, float step, float az, float elev, short bins, int time, int offset, int dataRead, int offset1, + int nsweep, String name, short datatype, int bytesPerBin) { + this(range, step, az, elev, bins, bins, time, offset, dataRead, offset1, nsweep, name, datatype, 1); + } - public Ray(float range, float step, float az, float elev, short bins, short time, int offset, int dataRead, - int offset1, int nsweep, String name, short datatype) { + public Ray(float range, float step, float az, float elev, short bins, short bins_actual, int time, int offset, + int dataRead, int offset1, int nsweep, String name, short datatype, int bytesPerBin) { - // this.val = new float[bins]; setRange(range); setStep(step); setAz(az); setElev(elev); setBins(bins); + setBinsActual(bins_actual); setTime(time); setOffset(offset); setDataRead(dataRead); @@ -45,7 +55,7 @@ public Ray(float range, float step, float az, float elev, short bins, short time setName(name); setNsweep(nsweep); setDataType(datatype); - + setBytesPerBin(bytesPerBin); } public short getDataType() { @@ -56,6 +66,14 @@ public void setDataType(short datatype) { this.datatype = datatype; } + public int getBytesPerBin() { + return bytesPerBin; + } + + public void setBytesPerBin(int bytesPerBin) { + this.bytesPerBin = bytesPerBin; + } + public float getRange() { return range; } @@ -108,11 +126,19 @@ public void setBins(short bins) { this.bins = bins; } - public short getTime() { + public short getBinsActual() { + return bins_actual; + } + + public void setBinsActual(short bins_actual) { + this.bins_actual = bins_actual; + } + + public int getTime() { return time; } - public void setTime(short time) { + public void setTime(int time) { this.time = time; } @@ -140,15 +166,6 @@ public void setOffset1(int offset1) { this.offset1 = offset1; } - /* - * public float[] getVal() { - * return val; - * } - * - * public void setVal(float[] val) { - * System.arraycopy(val, 0, this.val, 0, bins); - * } - */ public void setName(String name) { this.varName = name; } @@ -163,7 +180,8 @@ public boolean equals(Object o) { } else if (o instanceof Ray) { Ray oo = (Ray) o; - return (range == oo.range & step == oo.step & az == oo.az & elev == oo.elev & bins == oo.bins & time == oo.time); + return (range == oo.range & step == oo.step & az == oo.az & elev == oo.elev & bins == oo.bins + & bins_actual == oo.bins_actual & time == oo.time & bytesPerBin == oo.bytesPerBin); } else { return false; } @@ -171,7 +189,8 @@ public boolean equals(Object o) { public int hashCode() { return new Float(range).hashCode() + new Float(step).hashCode() + new Float(az).hashCode() - + new Float(elev).hashCode() + new Short(bins).hashCode() + new Short(time).hashCode(); + + new Float(elev).hashCode() + new Short(bins).hashCode() + new Short(bins_actual).hashCode() + + new Integer(time).hashCode() + new Integer(bytesPerBin).hashCode(); // val.hashCode(); } @@ -184,7 +203,7 @@ public String toString() { az = 360.0f + az; } - sb.format(" Az=%f Elev=%f Bins=%d Time=%d", az, elev, bins, time); + sb.format(" Az=%f Elev=%f Bins=%d (%d) Time=%d", az, elev, bins, bins_actual, time); // for (int i=0; i 1) + Byte[] dd = new Byte[bins * bytesPerBin]; int nb = 0; short a00; - // raf.readFully(data); + + short dty = getDataType(); + + int posInRay_absolute = 0; + if (dataRead > 0) { raf.seek(offset); for (int i = 0; i < dataRead; i++) { - d = raf.readByte(); - dd[i] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), d); - nb++; + dd[i] = raf.readByte(); + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; } } raf.seek(offset1); int cur_len = offset1; + // this only works for 1 additional record, not for more. Only a theoretical possibility for now. + // (relevant only for data types with a lot of bytes per bin) while (nb < (int) bins) { // --- Check if the code=1 ("1" means an end of a ray) a00 = raf.readShort(); cur_len = cur_len + 2; + // end of ray if (a00 == (short) 1) { - for (int uk = 0; uk < (int) bins; uk++) { - dd[uk] = -999.99f; - } + // don't need to do anything as the rest of dd is already null break; } if (a00 < 0) { // -- This is data int nwords = a00 & 0x7fff; int dataRead1 = nwords * 2; - int pos = 0; - if (cur_len % REC_SIZE == 0) { + boolean breakOuterLoop = false; + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { break; } raf.seek(cur_len); - for (int i = 0; i < dataRead1; i++) { - d = raf.readByte(); - dd[nb] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), d); - nb = nb + 1; - cur_len = cur_len + 1; - if (nb % REC_SIZE == 0) { - pos = i + 1; + for (int i = 0; i < dataRead1 && nb < bins; i++) { + dd[posInRay_absolute] = raf.readByte(); + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; + + cur_len++; + + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { + breakOuterLoop = true; break; } } - - if (pos > 0) { + if (breakOuterLoop) { break; } + } else if (a00 > 0 & a00 != 1) { int num_zero = a00 * 2; int dataRead1 = num_zero; - for (int k = 0; k < dataRead1; k++) { - dd[nb + k] = SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, getDataType(), (byte) 0); - } - nb = nb + dataRead1; - if (cur_len % REC_SIZE == 0) { - break; - } + for (int k = 0; k < dataRead1 && nb < bins; k++) { + dd[posInRay_absolute] = 0; + posInRay_absolute++; + if (posInRay_absolute % bytesPerBin == 0) + nb++; + if (cur_len % SigmetVolumeScan.REC_SIZE == 0) { + break; + } + } } + } // ------ end of while for num_bins--------------------------------- - for (int gateIdx : gateRange) { - if (gateIdx >= bins) - ii.setFloatNext(Float.NaN); - else - ii.setFloatNext(dd[gateIdx]); + // only supporting float, double, byte and byte[]/Object for now + DataType dType = SigmetIOServiceProvider.calcDataType(dty, bytesPerBin); + switch (dType) { + case FLOAT: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setFloatNext(Float.NaN); + else if (gateIdx >= bins_actual) + ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setFloatNext(SigmetVolumeScan.MISSING_VALUE_FLOAT); + else + ii.setFloatNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, ddx)); + } + } + break; + case DOUBLE: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setDoubleNext(Double.NaN); + else if (gateIdx >= bins_actual) + ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setDoubleNext(SigmetVolumeScan.MISSING_VALUE_DOUBLE); + else { + int ddx2 = dd[offset + 1]; + int rawValue = (ddx & 0xFF) | ((ddx2 & 0xFF) << 8); + ii.setDoubleNext(SigmetIOServiceProvider.calcData(SigmetIOServiceProvider.recHdr, dty, rawValue)); + } + } + } + break; + case BYTE: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else if (gateIdx >= bins_actual) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + else + ii.setByteNext(ddx); + } + } + break; + default: + for (int gateIdx : gateRange) { + if (gateIdx >= bins) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else if (gateIdx >= bins_actual) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else { + int offset = gateIdx * bytesPerBin; + Byte ddx = dd[offset]; + if (ddx == null) + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + else { + byte[] b = new byte[bytesPerBin]; + for (int i = 0; i < bytesPerBin; i++) { + ddx = dd[offset + i]; + b[i] = ddx == null ? SigmetVolumeScan.MISSING_VALUE_BYTE : ddx; + } + ii.setObjectNext(ByteBuffer.wrap(b)); + } + } + } } - } // end of readData } // class Ray end------------------------------------------ diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java index 39599c325d..2c812a95d9 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetIOServiceProvider.java @@ -5,25 +5,10 @@ package ucar.nc2.iosp.sigmet; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import org.checkerframework.checker.units.qual.A; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.ma2.Range; -import ucar.ma2.Array; -import ucar.ma2.ArrayFloat; -import ucar.ma2.ArrayInt; -import ucar.ma2.DataType; -import ucar.ma2.Index; -import ucar.ma2.IndexIterator; -import ucar.ma2.InvalidRangeException; -import ucar.ma2.Section; +import ucar.ma2.*; import ucar.nc2.Attribute; import ucar.nc2.Dimension; import ucar.nc2.Variable; @@ -35,6 +20,15 @@ import ucar.nc2.iosp.LayoutRegular; import ucar.unidata.io.RandomAccessFile; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + /** * Implementation of the ServerProvider pattern provides input/output * of the SIGMET-IRIS dataset. IOSP are managed by the NetcdfFile class. @@ -78,12 +72,16 @@ public class SigmetIOServiceProvider extends AbstractIOServiceProvider { private static Logger logger = LoggerFactory.getLogger(SigmetIOServiceProvider.class); + + // meaning unprocessed + public static String RAW_VARIABLE_PREFIX = "raw_"; + private ArrayList varList; private int[] tsu_sec; private int[] sweep_bins; private String date0; - public static java.util.Map recHdr = new java.util.HashMap<>(); + public static Map recHdr = new java.util.HashMap<>(); private SigmetVolumeScan volScan; public String getFileTypeDescription() { @@ -101,7 +99,7 @@ public String getFileTypeId() { /** * Check if this is a valid SIGMET-IRIS file for this IOServiceProvider. */ - public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) { + public boolean isValidFile(RandomAccessFile raf) { try { raf.order(RandomAccessFile.LITTLE_ENDIAN); // The first struct in the file is the product_hdr, which will have the @@ -124,25 +122,21 @@ public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) { /** * Open existing file, and populate ncfile with it. */ - public void open(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - ucar.nc2.util.CancelTask cancelTask) throws java.io.IOException { + public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ucar.nc2.util.CancelTask cancelTask) + throws IOException { super.open(raf, ncfile, cancelTask); // java.util.Map recHdr=new java.util.HashMap(); - java.util.Map hdrNames = new java.util.HashMap<>(); - volScan = new SigmetVolumeScan(raf, ncfile, varList); + Map hdrNames = new java.util.HashMap<>(); + volScan = new SigmetVolumeScan(raf); this.varList = init(raf, ncfile, hdrNames); - - // doData(raf, ncfile, varList); - // raf.close(); - // this.ncfile.close(); } /** * Read some global data from SIGMET file. The SIGMET file consists of records with * fixed length=6144 bytes. */ - public static java.util.Map readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) { - java.util.Map recHdr1 = new java.util.HashMap<>(); + public static Map readRecordsHdr(RandomAccessFile raf) { + Map recHdr1 = new java.util.HashMap<>(); try { int nparams = 0; // -- Read from of the 1st record -- 12+320+120 @@ -164,17 +158,51 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando short num_rays = raf.readShort(); // 6340 raf.skipBytes(2); int radar_alt = raf.readInt(); // 6344 - raf.seek(6648); - int time_beg = raf.readInt(); + // end of ingest_config would be 6636 + + // next is 2612 Bytes + raf.seek(6648); // + task_configuration 12 + // inner 120 Bytes + int time_beg = raf.readInt(); // Start time (seconds within a day) raf.seek(6652); - int time_end = raf.readInt(); - raf.seek(6772); + int time_end = raf.readInt(); // Stop time (seconds within a day) + // end inner would be 6768 + + // next inner is 320 Bytes. + raf.seek(6772); // + 2 Major mode + 2 DSP type + + // 4 - 24 - Current Data type mask + // Mask word 0 int data_mask = raf.readInt(); for (int j = 0; j < 32; j++) { nparams += ((data_mask >> j) & (0x1)); } + raf.readInt(); // Extended header type + // Mask word 1 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 2 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 3 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + // Mask word 4 + data_mask = raf.readInt(); + for (int j = 0; j < 32; j++) { + nparams += ((data_mask >> j) & (0x1)); + } + raf.seek(6912); short multiprf = raf.readShort(); + // end inner would be 7088 + raf.seek(7408); int range_first = raf.readInt(); // cm 7408 int range_last = raf.readInt(); // cm 7412 @@ -186,6 +214,9 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando int step = raf.readInt(); // cm 7424 raf.seek(7574); short number_sweeps = raf.readShort(); // 7574 + // end of task_configuration would be 9248 + + raf.seek(12312); int base_time = raf.readInt(); // 3d rec raf.skipBytes(2); @@ -218,8 +249,8 @@ public static java.util.Map readRecordsHdr(ucar.unidata.io.Rando /** * Read StationName strings */ - public java.util.Map readStnNames(ucar.unidata.io.RandomAccessFile raf) { - java.util.Map hdrNames = new java.util.HashMap<>(); + public Map readStnNames(RandomAccessFile raf) { + Map hdrNames = new java.util.HashMap<>(); try { raf.seek(6288); String stnName = raf.readString(16); @@ -241,12 +272,11 @@ public java.util.Map readStnNames(ucar.unidata.io.RandomAccessFi * @param hdrNames java.util.Map with values for "StationName.." Attributes * @return ArrayList of Variables of ncfile */ - public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, - java.util.Map hdrNames) { + public ArrayList init(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, Map hdrNames) { // prepare attribute values - String[] data_name = {" ", "TotalPower", "Reflectivity", "Velocity", "Width", "Differential_Reflectivity"}; - String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; - int[] type = {1, 2, 3, 4, 5}; + String[] data_name = SigmetVolumeScan.data_name; + String[] unit = SigmetVolumeScan.data_unit; + String def_datafile = "SIGMET-IRIS"; String tim = ""; int ngates = 0; @@ -305,25 +335,58 @@ public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.N ArrayList varList = new ArrayList<>(); + short[] dataTypes = volScan.getDataTypes(); + Variable[][] v = new Variable[nparams][number_sweeps]; String var_name; for (int j = 0; j < nparams; j++) { - int tp = type[j]; - var_name = data_name[tp]; + short dty = dataTypes[j]; + String var_name_original = data_name[dty]; + + // see also calcData - a lot of data types are raw values + if (dty == 6 || dty > 11) + // this is make it obvious to the user + var_name_original = RAW_VARIABLE_PREFIX + var_name_original; + + var_name = var_name_original; + int bytesPerBin = 1; + List> allRays = volScan.getGroup(data_name[dty]); + if (allRays.size() > 0 && allRays.get(0).size() > 0) + bytesPerBin = allRays.get(0).get(0).getBytesPerBin(); + + DataType dType = calcDataType(dty, bytesPerBin); + Number missingVal = null; + switch (dType) { + case FLOAT: + missingVal = SigmetVolumeScan.MISSING_VALUE_FLOAT; + break; + case DOUBLE: + missingVal = SigmetVolumeScan.MISSING_VALUE_DOUBLE; + break; + case BYTE: + missingVal = SigmetVolumeScan.MISSING_VALUE_BYTE; + + } + for (int jj = 0; jj < number_sweeps; jj++) { if (number_sweeps > 1) { - var_name = data_name[tp] + "_sweep_" + (jj + 1); + var_name = var_name_original + "_sweep_" + (jj + 1); + } + Variable.Builder builder = Variable.builder().setNcfile(ncfile).setName(var_name).setDataType(dType); + Attribute.Builder attr = Attribute.builder().setName(CDM.MISSING_VALUE); + if (missingVal != null) { + attr.setNumericValue(missingVal, false); + } else { + attr.setValues(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY); } - v[j][jj] = new Variable(ncfile, null, null, var_name); - v[j][jj].setDataType(DataType.FLOAT); + builder.addAttribute(attr.build()); + dims2.add(radial); dims2.add(gateR[jj]); - v[j][jj].setDimensions(dims2); - v[j][jj].addAttribute(new Attribute(CDM.LONG_NAME, var_name)); - v[j][jj].addAttribute(new Attribute(CDM.UNITS, unit[tp])); - String coordinates = "time elevationR azimuthR distanceR"; - v[j][jj].addAttribute(new Attribute(_Coordinate.Axes, coordinates)); - v[j][jj].addAttribute(new Attribute(CDM.MISSING_VALUE, -999.99f)); + builder.setDimensions(dims2).addAttribute(new Attribute(CDM.LONG_NAME, var_name)) + .addAttribute(new Attribute(CDM.UNITS, unit[dty])) + .addAttribute(new Attribute(_Coordinate.Axes, "time elevationR azimuthR distanceR")); + v[j][jj] = builder.build(ncfile.getRootGroup()); ncfile.addVariable(null, v[j][jj]); varList.add(v[j][jj]); dims2.clear(); @@ -485,27 +548,17 @@ public ArrayList init(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.N * @param recHdr java.util.Map with values for Attributes */ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[] yr, short[] m, short[] dda, - ArrayList varList, java.util.Map recHdr) { + ArrayList varList, Map recHdr) { // prepare attribute values - String[] unit = {" ", "dbZ", "dbZ", "m/sec", "m/sec", "dB"}; - String def_datafile = "SIGMET-IRIS"; - Short header_length = 80; - Short ray_header_length = 6; int ngates; - float radar_lat = recHdr.get("radar_lat").floatValue(); - float radar_lon = recHdr.get("radar_lon").floatValue(); - short ground_height = recHdr.get("ground_height").shortValue(); - short radar_height = recHdr.get("radar_height").shortValue(); - int radar_alt = (recHdr.get("radar_alt").intValue()) / 100; short num_rays = recHdr.get("num_rays").shortValue(); float range_first = (recHdr.get("range_first").intValue()) * 0.01f; float range_last = (recHdr.get("range_last").intValue()) * 0.01f; short number_sweeps = recHdr.get("number_sweeps").shortValue(); int nparams = (recHdr.get("nparams").intValue()); - // define date/time - // int last_t=(int)(ray[nparams*number_sweeps-1][num_rays-1].getTime()); + int last_t = volScan.lastRay.getTime(); String sss1 = Short.toString(m[0]); if (sss1.length() < 2) @@ -528,11 +581,6 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ // set all of Variables try { - int sz = varList.size(); - - ArrayFloat.D2[] dataArr = new ArrayFloat.D2[nparams * number_sweeps]; - Index[] dataIndex = new Index[nparams * number_sweeps]; - Ray[] rtemp = new Ray[(int) num_rays]; Variable[] distanceR = new Variable[number_sweeps]; @@ -559,14 +607,33 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ distArr[i].setFloat(distIndex[i].set(ii), (range_first + ii * stp)); } } - List rgp = volScan.getTotalPowerGroups(); - if (rgp.isEmpty()) - rgp = volScan.getReflectivityGroups(); + + // used for lists of distance, time, azimuth, elevation + // probably there is a better way to do this, but I'm keeping the old logic + List rgp = volScan.getGroup("TotalPower"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("TotalPower_2"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("Reflectivity"); + if (rgp == null || rgp.isEmpty()) + rgp = volScan.getGroup("Reflectivity_2"); + List[] sgp = new ArrayList[number_sweeps]; - for (int i = 0; i < number_sweeps; i++) { - sgp[i] = (List) rgp.get((short) i); - } + for (int sweepMinus1 = 0; sweepMinus1 < number_sweeps; sweepMinus1++) { + List found = null; + // in case some rays are missing/empty this is a safer way to find them + for (int i = 0; i < rgp.size() && found == null; i++) { + List rlist = (List) rgp.get(i); + if (rlist.size() > 0) { + Ray r = rlist.get(0); + if (r.getNsweep() == sweepMinus1 + 1) { + found = rlist; + } + } + } + sgp[sweepMinus1] = found; + } Variable[] time = new Variable[number_sweeps]; ArrayInt.D1[] timeArr = new ArrayInt.D1[number_sweeps]; @@ -586,12 +653,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ timeArr[i] = (ArrayInt.D1) Array.factory(DataType.INT, time[i].getShape()); timeIndex[i] = timeArr[i].getIndex(); List rlist = sgp[i]; - - for (int jj = 0; jj < num_rays; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { - timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + timeArr[i].setInt(timeIndex[i].set(jj), -999); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + timeArr[i].setInt(timeIndex[i].set(jj), rtemp[jj].getTime()); + } } } @@ -612,12 +685,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ azimArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, azimuthR[i].getShape()); azimIndex[i] = azimArr[i].getIndex(); List rlist = sgp[i]; - - for (int jj = 0; jj < num_rays; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { - azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + azimArr[i].setFloat(azimIndex[i].set(jj), -999.99f); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + azimArr[i].setFloat(azimIndex[i].set(jj), rtemp[jj].getAz()); + } } } @@ -638,12 +717,18 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ elevArr[i] = (ArrayFloat.D1) Array.factory(DataType.FLOAT, elevationR[i].getShape()); elevIndex[i] = elevArr[i].getIndex(); List rlist = sgp[i]; - - for (int jj = 0; jj < num_rays; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - for (int jj = 0; jj < num_rays; jj++) { - elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); + if (rlist == null) { + for (int jj = 0; jj < num_rays; jj++) { + elevArr[i].setFloat(elevIndex[i].set(jj), -999.99f); + } + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + for (int jj = 0; jj < num_rays_actual; jj++) { + elevArr[i].setFloat(elevIndex[i].set(jj), rtemp[jj].getElev()); + } } } @@ -661,11 +746,16 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ for (int i = 0; i < number_sweeps; i++) { List rlist = sgp[i]; - for (int jj = 0; jj < num_rays; jj++) { - rtemp[jj] = (Ray) rlist.get(jj); - } // ray[i][jj]; } - ngates = rtemp[0].getBins(); - gatesArr.setInt(gatesIndex.set(i), ngates); + if (rlist == null) { + gatesArr.setInt(gatesIndex.set(i), -999); + } else { + int num_rays_actual = Math.min(num_rays, rlist.size()); + for (int jj = 0; jj < num_rays_actual; jj++) { + rtemp[jj] = (Ray) rlist.get(jj); + } // ray[i][jj]; } + ngates = rtemp[0].getBins(); + gatesArr.setInt(gatesIndex.set(i), ngates); + } } for (int i = 0; i < number_sweeps; i++) { @@ -689,7 +779,7 @@ public void doNetcdfFileCoordinate(ucar.nc2.NetcdfFile ncfile, int[] bst, short[ * of ucar.ma2.Range which define the requested data subset. * @return Array of data which will be read from Variable through this call. */ - public Array readData1(ucar.nc2.Variable v2, Section section) throws IOException, InvalidRangeException { + public Array readData1(Variable v2, Section section) throws IOException, InvalidRangeException { // doData(raf, ncfile, varList); int[] sh = section.getShape(); Array temp = Array.factory(v2.getDataType(), sh); @@ -715,17 +805,16 @@ public Array readData(Variable v2, Section section) throws IOException { List> groups; String shortName = v2.getShortName(); - if (shortName.startsWith("Reflectivity")) - groups = volScan.getReflectivityGroups(); - else if (shortName.startsWith("Velocity")) - groups = volScan.getVelocityGroups(); - else if (shortName.startsWith("TotalPower")) - groups = volScan.getTotalPowerGroups(); - else if (shortName.startsWith("Width")) - groups = volScan.getWidthGroups(); - else if (shortName.startsWith("DiffReflectivity")) - groups = volScan.getDifferentialReflectivityGroups(); - else + String groupName = shortName; + + int posSweep = groupName.indexOf("_sweep_"); + if (posSweep > 0) + groupName = groupName.substring(0, posSweep); + if (groupName.startsWith(RAW_VARIABLE_PREFIX)) + groupName = groupName.substring(RAW_VARIABLE_PREFIX.length()); + + groups = volScan.getGroup(groupName); + if (groups == null) throw new IllegalStateException("Illegal variable name = " + shortName); if (section.getRank() == 2) { @@ -748,25 +837,50 @@ else if (shortName.startsWith("DiffReflectivity")) private void readOneScan(List mapScan, Range radialRange, Range gateRange, IndexIterator ii) throws IOException { int siz = mapScan.size(); + short dataType = 1; // TotalPower as fallback + int bytesPerBin = 1; + for (int radialIdx : radialRange) { if (radialIdx >= siz) - readOneRadial(null, gateRange, ii); + readOneRadialMissing(dataType, bytesPerBin, gateRange, ii); else { Ray r = mapScan.get(radialIdx); + dataType = r.datatype; + bytesPerBin = r.bytesPerBin; readOneRadial(r, gateRange, ii); } } } private void readOneRadial(Ray r, Range gateRange, IndexIterator ii) throws IOException { - if (r == null) { - for (int i = 0; i < gateRange.length(); i++) - ii.setFloatNext(Float.NaN); - return; - } r.readData(volScan.raf, gateRange, ii); } + private void readOneRadialMissing(short datatype, int bytesPerBin, Range gateRange, IndexIterator ii) { + DataType dType = calcDataType(datatype, bytesPerBin); + switch (dType) { + case FLOAT: + for (int i = 0; i < gateRange.length(); i++) { + ii.setFloatNext(Float.NaN); + } + break; + case DOUBLE: + for (int i = 0; i < gateRange.length(); i++) { + ii.setDoubleNext(Double.NaN); + } + break; + case BYTE: + for (int i = 0; i < gateRange.length(); i++) { + ii.setByteNext(SigmetVolumeScan.MISSING_VALUE_BYTE); + } + break; + default: // byte[]/Object + for (int i = 0; i < gateRange.length(); i++) { + ii.setObjectNext(SigmetVolumeScan.MISSING_VALUE_BYTE_ARRAY_BB); + } + } + } + /** * Read data from a top level Variable of INTEGER data type and return a memory resident Array. * @@ -811,8 +925,7 @@ public Array readFloatData(LayoutRegular index, Variable v2) throws IOException * @param channel WritableByteChannel object - channel that can write bytes. * @return the number of bytes written, possibly zero. */ - public long readToByteChannel11(ucar.nc2.Variable v2, Section section, WritableByteChannel channel) - throws java.io.IOException { + public long readToByteChannel11(Variable v2, Section section, WritableByteChannel channel) throws IOException { Array data = readData(v2, section); float[] ftdata = new float[(int) data.getSize()]; byte[] bytedata = new byte[(int) data.getSize() * 4]; @@ -931,12 +1044,36 @@ static float calcAz(short az0, short az1) { return result.floatValue(); } + static DataType calcDataType(short dty, int bytes) { + switch (dty) { + case 1:// dty=1,2 -total_power, reflectivity (dBZ) + case 2: + case 3: // dty=3 - mean velocity (m/sec) + case 4: // dty=4 - spectrum width (m/sec) + case 5: // dty=5 - differential reflectivity (dB) + return DataType.FLOAT; + + case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) + case 8: + case 9: // dty=9 - mean velocity 2 (m/sec) + case 10: // dty=10 - spectrum width 2 (m/sec) + case 11: // dty=11 - differential reflectivity 2 (dB) + return DataType.DOUBLE; + + default: + // TODO implement for more SigmetVolumeScan.data_name (see chapter 4.4) + if (bytes == 1) + return DataType.BYTE; + + return DataType.OPAQUE; + } + } + /** * Calculate data values from raw ingest data * * @param recHdr java.util.Map object with values for calculation - * @param dty type of data ( "Total_Power", "Reflectivity", "Velocity", - * "Width", "Differential_Reflectivity") + * @param dty type of data (@see SigmetVolumeScan.data_name) * @param data 1-byte input value * @return float value with precision of two decimal */ @@ -944,35 +1081,99 @@ static float calcData(Map recHdr, short dty, byte data) { short[] coef = {1, 2, 3, 4}; // MultiPRF modes short multiprf = recHdr.get("multiprf").shortValue(); float vNyq = recHdr.get("vNyq").floatValue(); - double temp = -999.99; + double temp = SigmetVolumeScan.MISSING_VALUE_FLOAT; + + // see chapter 4.4 switch (dty) { - default: // dty=1,2 -total_power, reflectivity (dBZ) - if (data != 0) { + case 1:// dty=1,2 -total_power, reflectivity (dBZ) + case 2: + if (data != 0 && data != (byte) 255) { temp = (((int) data & 0xFF) - 64) * 0.5; } break; case 3: // dty=3 - mean velocity (m/sec) if (data != 0) { + // seems incorrect according to "4.4.44 1-byte Velocity Format (DB_VEL)" + // TODO needs more research temp = ((((int) data & 0xFF) - 128) / 127.0) * vNyq * coef[multiprf]; } break; case 4: // dty=4 - spectrum width (m/sec) - if (data != 0) { + if (data != 0 && data != (byte) 255) { + // seems incorrect according to "4.4.48 1-byte Width Format (DB_WIDTH)" + // TODO needs more research double v = ((((int) data & 0xFF) - 128) / 127.0) * vNyq * coef[multiprf]; temp = (((int) data & 0xFF) / 256.0) * v; } break; case 5: // dty=5 - differential reflectivity (dB) - if (data != 0) { + if (data != 0 && data != (byte) 255) { temp = ((((int) data & 0xFF) - 128) / 16.0); } break; + default: + // TODO implement for more SigmetVolumeScan.data_name (only 1 byte) (see chapter 4.4) + // using only the raw value + temp = (int) data & 0xFF; + + // doing no rounding in this case as below for the other cases + BigDecimal bd = new BigDecimal(temp); + return bd.floatValue(); + // logger.warn("calcData: unimplemented 1 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); } BigDecimal bd = new BigDecimal(temp); + // this should be reviewed, not sure why would you want a loss of precision here? BigDecimal result = bd.setScale(2, RoundingMode.HALF_DOWN); return result.floatValue(); } + /** + * Calculate data values from raw ingest data + * + * @param recHdr java.util.Map object with values for calculation + * @param dty type of data (@see SigmetVolumeScan.data_name) + * @param data 2-byte input value (unsigned short) + * @return double value + */ + static double calcData(Map recHdr, short dty, int data) { + double temp = SigmetVolumeScan.MISSING_VALUE_DOUBLE; + + // see chapter 4.4 + switch (dty) { + case 7:// dty=7,8 -total_power 2, reflectivity 2 (dBZ) + case 8: + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; + } + break; + case 9: // dty=9 - mean velocity 2 (m/sec) + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; + } + break; + case 10: // dty=10 - spectrum width 2 (m/sec) + if (data != 0 && data != 65535) { + temp = data / 100d; + } + break; + case 11: // dty=11 - differential reflectivity 2 (dB) + if (data != 0 && data != 65535) { + temp = (data - 32768) / 100d; + } + break; + default: + // TODO implement for more SigmetVolumeScan.data_name (only 2 bytes) (see chapter 4.4) + // using only the raw value + temp = data; + // logger.warn("calcData: unimplemented 2 byte data type = " + dty + " " + SigmetVolumeScan.data_name[dty]); + break; + } + + BigDecimal bd = new BigDecimal(temp); + // full precision + return bd.doubleValue(); + } + /** * Calculate time as hh:mm:ss * diff --git a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java index 679c970eb5..ddd4529889 100644 --- a/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java +++ b/cdm/radial/src/main/java/ucar/nc2/iosp/sigmet/SigmetVolumeScan.java @@ -6,12 +6,14 @@ package ucar.nc2.iosp.sigmet; -// ~--- non-JDK imports -------------------------------------------------------- - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ucar.ma2.Array; +import ucar.ma2.DataType; import ucar.nc2.Variable; import ucar.unidata.io.RandomAccessFile; + +import java.nio.ByteBuffer; import java.util.*; /** @@ -20,13 +22,130 @@ */ public class SigmetVolumeScan { private static Logger logger = LoggerFactory.getLogger(SigmetVolumeScan.class); - String[] data_name = {" ", "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity"}; - private List> differentialReflectivityGroups; - private List> reflectivityGroups; - private List> totalPowerGroups; - private List> velocityGroups; - private List> widthGroups; - private List> timeGroups; + /** + * See IRIS-Programming-Guide-M211318EN: 4.9 Constants + * empty or undefined types are set to their index number to avoid duplicates + * if suffix is "_2" it means a two byte value instead of just one byte + * (2 bytes are experimental at this point - didn't have a file to test) + */ + public static final String[] data_name = {"ExtendedHeaders", // ? bytes + "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity", "[6]", "CorrectedReflectivity", // 1 + // byte + "TotalPower_2", "Reflectivity_2", "Velocity_2", "Width_2", "DifferentialReflectivity_2", // 2 bytes + "RainfallRate_2" /* 2 bytes */, "KDPDifferentialPhase" /* 1 byte */, "KDPDifferentialPhase_2" /* 2 bytes */, + "PhiDPDifferentialPhase" /* 1 byte */, "CorrectedVelocity" /* 1 byte */, "SQI" /* 1 byte */, "RhoHV" /* 1 byte */, + "RhoHV_2" /* 2 bytes */, "CorrectedReflectivity_2" /* 2 bytes */, "CorrectedVelocity_2" /* 2 bytes */, + "SQI_2" /* 2 bytes */, "PhiDPDifferentialPhase_2" /* 2 bytes */, "LDRH" /* 1 byte */, "LDRH_2" /* 2 bytes */, + "LDRV" /* 1 byte */, "LDRV_2" /* 2 bytes */, "[29]", "[30]", "[31]", "Height" /* 1 byte */, + "LinearLiquid_2" /* 2 bytes */, "RawData" /* ? */, "WindShear" /* 1 byte */, "Divergence_2" /* 2 bytes */, + "FloatedLiquid_2" /* 2 bytes */, "UserType" /* 1 byte */, "UnspecifiedData" /* 1 byte */, + "Deformation_2" /* 2 bytes */, "VerticalVelocity_2" /* 2 bytes */, "HorizontalVelocity_2" /* 2 bytes */, + "HorizontalWindDirection_2" /* 2 bytes */, "AxisOfDilatation_2" /* 2 bytes */, "TimeInSeconds_2" /* 2 bytes */, + "RHOH" /* 1 byte */, "RHOH_2" /* 2 bytes */, "RHOV" /* 1 byte */, "RHOV_2" /* 2 bytes */, "PHIH" /* 1 byte */, + "PHIH_2" /* 2 bytes */, "PHIV" /* 1 byte */, "PHIV_2" /* 2 bytes */, "UserType_2" /* 2 bytes */, + "HydrometeorClass" /* 1 byte */, "HydrometeorClass_2" /* 2 bytes */, + "CorrectedDifferentialReflectivity" /* 1 byte */, "CorrectedDifferentialReflectivity_2" /* 2 bytes */, + // 16 empty + "[59]", "[60]", "[61]", "[62]", "[63]", "[64]", "[65]", "[66]", "[67]", "[68]", "[69]", "[70]", "[71]", "[72]", + "[73]", "[74]", "PolarimetricMeteoIndex" /* 1 byte */, "PolarimetricMeteoIndex_2" /* 2 bytes */, + "LOG8" /* 1 byte */, "LOG16_2" /* 2 bytes */, "CSP8" /* 1 byte */, "CSP16_2" /* 2 bytes */, "CCOR8" /* 1 byte */, + "CCOR16_2" /* 2 bytes */, "AH8" /* 1 byte */, "AH16_2" /* 2 bytes */, "AV8" /* 1 byte */, "AV16_2" /* 2 bytes */, + "AZDR8" /* 1 byte */, "AZDR16_2" /* 2 bytes */,}; + + /* + * Some units are unknown, some were correlated and assumed by ChatGPT. + * Not all might be correct! + * Just extending the initial list here from previous version of SigmetIOServiceProvider + * // TODO fill and double check + */ + public static final String[] data_unit = {"?", // DB_XHDR: Extended Headers + "dBm", // DB_DBT: Total Power (1 byte) + "dBZ", // DB_DBZ: Reflectivity (1 byte) + "m/s", // DB_VEL: Velocity (1 byte) + "m/s", // DB_WIDTH: Width (1 byte) + "dB", // DB_ZDR: Differential Reflectivity (1 byte) + "?", // empty + "dBZ", // DB_DBZC: Corrected Reflectivity (1 byte) + "dBm", // DB_DBT2: Total Power (2 byte) + "dBZ", // DB_DBZ2: Reflectivity (2 byte) + "m/s", // DB_VEL2: Velocity (2 byte) + "m/s", // DB_WIDTH2: Width (2 byte) + "dB", // DB_ZDR2: Differential Reflectivity (2 byte) + "mm/hr", // DB_RAINRATE2: Rainfall Rate (2 byte) + "°/km", // DB_KDP: KDP (Differential Phase) (1 byte) + "°/km", // DB_KDP2: KDP (Differential Phase) (2 byte) + "°", // DB_PHIDP: PhiDP (Differential Phase) (1 byte) + "m/s", // DB_VELC: Corrected Velocity (1 byte) + "?", // DB_SQI: SQI (Signal Quality Index) (1 byte) + "?", // DB_RHOHV: RhoHV (1 byte) + "?", // DB_RHOHV2: RhoHV (2 byte) + "dBZ", // DB_DBZC2: Corrected Reflectivity (2 byte) + "m/s", // DB_VELC2: Corrected Velocity (2 byte) + "?", // DB_SQI2: SQI (Signal Quality Index) (2 byte) + "°", // DB_PHIDP2: PhiDP (Differential Phase) (2 byte) + "?", // DB_LDRH: LDR xmt H rcv V (1 byte) + "?", // DB_LDRH2: LDR xmt H rcv V (2 byte) + "?", // DB_LDRV: LDR xmt V rcv H (1 byte) + "?", // DB_LDRV2: LDR xmt V rcv H (2 byte) + // 3 empty + "?", "?", "?", "1/10 km", // DB_HEIGHT: Height (1/10 km) (1 byte) + ".001mm", // DB_VIL2: Linear liquid (.001mm) (2 byte) + "?", // DB_RAW: Unknown unit or unitless (Raw Data) + "m/s", // DB_SHEAR: Shear (Velocity difference) + "?", // DB_DIVERGE2: Divergence (2 byte) + "mm", // DB_FLIQUID2: Liquid equivalent (2 byte) + "?", // DB_USER: User-defined (unit depends on definition) + "?", // DB_OTHER: Other data type (unit depends on definition) + "1/s", // DB_DEFORM2: Deformation (2 byte) + "m/s", // DB_VVEL2: Vertical Velocity (2 byte) + "m/s", // DB_HVEL2: Horizontal Velocity (2 byte) + "°", // DB_HDIR2: Horizontal Direction (2 byte) + "1/s", // DB_AXDIL2: Axis of Dilation (2 byte) + "s", // DB_TIME2: Time (2 byte) + "?", // DB_RHOH: RhoH (1 byte) + "?", // DB_RHOH2: RhoH (2 byte) + "?", // DB_RHOV: RhoV (1 byte) + "?", // DB_RHOV2: RhoV (2 byte) + "°", // DB_PHIH: PhiH (1 byte) + "°", // DB_PHIH2: PhiH (2 byte) + "°", // DB_PHIV: PhiV (1 byte) + "°", // DB_PHIV2: PhiV (2 byte) + "?", // DB_USER2: User-defined (2 byte) + "?", // DB_HCLASS: Hydrometeor Classification (1 byte) + "?", // DB_HCLASS2: Hydrometeor Classification (2 byte) + "dB", // DB_ZDRC: Corrected Differential Reflectivity (1 byte) + "dB", // DB_ZDRC2: Corrected Differential Reflectivity (2 byte) + // 16 empty + "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", // PolarimetricMeteoIndex (1 + // byte) + "?", // PolarimetricMeteoIndex2 (2 bytes) + "?", // LOG8 (1 byte) + "?", // LOG16 (2 bytes) + "?", // CSP8 (1 byte) + "?", // CSP16 (2 bytes) + "?", // CCOR8 (1 byte) + "?", // CCOR16 (2 bytes) + "?", // AH8 (1 byte) + "?", // AH16 (2 bytes) + "?", // AV8 (1 byte) + "?", // AV16 (2 bytes) + "?", // AZDR8 (1 byte) + "?", // AZDR16 (2 bytes) + }; + + public static final int REC_SIZE = 6144; + + public static final float MISSING_VALUE_FLOAT = -999.99f; + public static final double MISSING_VALUE_DOUBLE = -999.99; + public static final byte MISSING_VALUE_BYTE = 0; + + public static final ByteBuffer MISSING_VALUE_BYTE_ARRAY_BB = ByteBuffer.wrap(new byte[] {MISSING_VALUE_BYTE}); + public static final Array MISSING_VALUE_BYTE_ARRAY = + Array.factoryConstant(DataType.OPAQUE, new int[] {1}, new ByteBuffer[] {MISSING_VALUE_BYTE_ARRAY_BB}); + + private HashMap>> allGroups = new HashMap<>(); + + private short[] data_type; private int[] num_gates; public int[] base_time; public short[] year; @@ -34,29 +153,21 @@ public class SigmetVolumeScan { public short[] day; public Ray firstRay; public Ray lastRay; - public ucar.unidata.io.RandomAccessFile raf; - public boolean hasReflectivity; - public boolean hasVelocity; - public boolean hasWidth; - public boolean hasTotalPower; - public boolean hasDifferentialReflectivity; - public boolean hasTime; + public RandomAccessFile raf; /** * Read all the values from SIGMET-IRIS file which are necessary to fill in the ncfile. * - * @param raf ucar.unidata.io.RandomAccessFile corresponds to SIGMET datafile. - * @param ncfile an empty NetcdfFile object which will be filled. - * @param varList ArrayList of Variables of ncfile */ - SigmetVolumeScan(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ArrayList varList) - throws java.io.IOException { - int REC_SIZE = 6144; + SigmetVolumeScan(RandomAccessFile raf) throws java.io.IOException { int len = 12288; // ---- Read from the 3d record----------- 6144*2=12288 short nrec = 0, nsweep = 1, nray = 0, byteoff = 0; - int nwords, end_words, data_read = 0, num_zero, rays_count = 0, nb = 0, pos = 0, pos_ray_hdr = 0, t = 0; + int nwords, end_words, data_read = 0, num_zero, rays_count = 0, nb = 0, posInRay_relative = 0, + posInRay_absolute = 0, pos_ray_hdr = 0, t = 0; short a0, a00, dty; - short beg_az = 0, beg_elev = 0, end_az = 0, end_elev = 0, num_bins = 0, time_start_sw = 0; + short beg_az = 0, beg_elev = 0, end_az = 0, end_elev = 0, num_bins = 0; + // int because UINT2 data type (Unsigned 16-bit integer) + int time_start_sw = 0; float az, elev, d = 0.0f, step; // byte data = 0; boolean beg_rec = true, end_rec = true, read_ray_hdr = true, begin = true; @@ -68,7 +179,7 @@ public class SigmetVolumeScan { raf.order(RandomAccessFile.LITTLE_ENDIAN); int fileLength = (int) raf.length(); - java.util.Map recHdr = SigmetIOServiceProvider.readRecordsHdr(raf); + Map recHdr = SigmetIOServiceProvider.readRecordsHdr(raf); int nparams = recHdr.get("nparams").intValue(); short number_sweeps = recHdr.get("number_sweeps").shortValue(); @@ -84,10 +195,11 @@ public class SigmetVolumeScan { short[] num_sweep = new short[nparams]; short[] num_rays_swp = new short[nparams]; short[] indx_1ray = new short[nparams]; + short[] num_rays_exp = new short[nparams]; short[] num_rays_act = new short[nparams]; short[] angl_swp = new short[nparams]; short[] bin_len = new short[nparams]; - short[] data_type = new short[nparams]; + data_type = new short[nparams]; // float[] dd = new float[bins]; num_gates = new int[number_sweeps]; base_time = new int[nparams * number_sweeps]; @@ -97,15 +209,10 @@ public class SigmetVolumeScan { // Array of Ray objects is 2D. Number of columns=number of rays // Number of raws = number of types of data if number_sweeps=1, // or number of raws = number_sweeps - List totalPower = new ArrayList<>(); - List velocity = new ArrayList<>(); - List reflectivity = new ArrayList<>(); - List width = new ArrayList<>(); - List diffReflectivity = new ArrayList<>(); - List time = new ArrayList<>(); + HashMap> all = new HashMap<>(); + int irays = (int) num_rays; Ray ray = null; - int two = 0; // init array float[] val = new float[bins]; @@ -142,7 +249,20 @@ public class SigmetVolumeScan { beg = 0; for (int i = 0; i < nparams; i++) { - int idh_len = cur_len + 12 + i * 76; + int idh_len = cur_len + 12 + i * 76; // + 12 == skipping over + + /* + * debug structure_header + * raf.seek(idh_len-12); + * // Structure identifier + * short si = raf.readShort(); + * raf.readShort(); // format version + * int nob = raf.readInt(); + * if (si != (short)24) + * throw new IllegalStateException("not a Ingest_header"); + * raf.readShort(); // reserved + * raf.readShort(); // flags + */ raf.seek(idh_len); @@ -156,11 +276,11 @@ public class SigmetVolumeScan { num_sweep[i] = raf.readShort(); // idh_len+12 num_rays_swp[i] = raf.readShort(); // idh_len+14 indx_1ray[i] = raf.readShort(); // idh_len+16 - raf.skipBytes(2); + num_rays_exp[i] = raf.readShort(); + beg += num_rays_exp[i]; // before num_rays_act[i] was used but it does seem not work num_rays_act[i] = raf.readShort(); - beg += num_rays_act[i]; // idh_len+20 angl_swp[i] = raf.readShort(); // idh_len+22 - bin_len[i] = raf.readShort(); // idh_len+24 + bin_len[i] = raf.readShort(); // idh_len+24 (Number of bits per bin for this data type) data_type[i] = raf.readShort(); // idh_len+26 } @@ -181,7 +301,8 @@ public class SigmetVolumeScan { end_rec = true; rays_count++; read_ray_hdr = true; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; len = cur_len; @@ -195,7 +316,7 @@ public class SigmetVolumeScan { } nwords = a0 & 0x7fff; - end_words = nwords - 6; + end_words = nwords - 6; // because of raw_prod_bhdr 12-byte structure data_read = end_words * 2; end_rec = false; @@ -212,17 +333,27 @@ public class SigmetVolumeScan { // ---Define output data files for each data_type (= nparams)/sweep --------- dty = data_type[0]; + int bitsToRead = bin_len[0]; if (nparams > 1) { kk = rays_count % nparams; dty = data_type[kk]; + bitsToRead = bin_len[kk]; + } else if (number_sweeps > 1) { } + if (bitsToRead <= 0 || bitsToRead % 8 != 0) + throw new IllegalStateException("Not compatible! Number of bits per bin for this data type = " + bitsToRead); + int bytesToRead = bitsToRead / 8; + String var_name = data_name[dty]; // --- read ray_header (size=12 bytes=6 words)--------------------------------------- if (read_ray_hdr) { + if (ray != null) + throw new IllegalStateException("ray != null"); + if (pos_ray_hdr < 2) { raf.seek(cur_len); beg_az = raf.readShort(); @@ -299,7 +430,7 @@ public class SigmetVolumeScan { if (pos_ray_hdr < 12) { raf.seek(cur_len); - time_start_sw = raf.readShort(); + time_start_sw = raf.readUnsignedShort(); cur_len = cur_len + 2; len = cur_len; } @@ -318,9 +449,9 @@ public class SigmetVolumeScan { continue; } - if (pos > 0) { - data_read = data_read - pos; - pos = 0; + if (posInRay_relative > 0) { + data_read = data_read - posInRay_relative; + posInRay_relative = 0; } if (data_read > 0) { @@ -332,10 +463,12 @@ public class SigmetVolumeScan { // data = raf.readByte(); // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); cur_len++; - nb++; + posInRay_absolute++; + if (posInRay_absolute % bytesToRead == 0) + nb++; if (cur_len % REC_SIZE == 0) { - pos = i + 1; + posInRay_relative = i + 1; beg_rec = true; read_ray_hdr = false; len = cur_len; @@ -344,13 +477,13 @@ public class SigmetVolumeScan { } } raf.seek(cur_len); - if (pos > 0) { + if (posInRay_relative > 0) { continue; } } if (cur_len % REC_SIZE == 0) { - pos = 0; + posInRay_relative = 0; beg_rec = true; read_ray_hdr = false; data_read = 0; @@ -368,11 +501,8 @@ public class SigmetVolumeScan { // --- Check if the code=1 ("1" means an end of a ray) if (a00 == (short) 1) { - // for (int uk = 0; uk < (int) num_bins; uk++) { - // dd[uk] = -999.99f; - // } - ray = new Ray(-999.99f, -999.99f, -999.99f, -999.99f, num_bins, (short) (-99), -999, 0, -999, nsweep, - var_name, dty); + ray = new Ray(range_first, step, az, elev, num_bins, (short) nb, time_start_sw, rayoffset, datalen, + rayoffset1, nsweep, var_name, dty, bytesToRead); rays_count++; beg_rec = false; end_rec = true; @@ -385,7 +515,7 @@ public class SigmetVolumeScan { data_read = nwords * 2; if (cur_len % REC_SIZE == 0) { - pos = 0; + posInRay_relative = 0; beg_rec = true; end_rec = false; read_ray_hdr = false; @@ -398,11 +528,13 @@ public class SigmetVolumeScan { for (int ii = 0; ii < data_read; ii++) { // data = raf.readByte(); // dd[nb] = SigmetIOServiceProvider.calcData(recHdr, dty, data); - cur_len = cur_len + 1; - nb = nb + 1; + cur_len++; + posInRay_absolute++; + if (posInRay_absolute % bytesToRead == 0) + nb++; if (cur_len % REC_SIZE == 0) { - pos = ii + 1; + posInRay_relative = ii + 1; beg_rec = true; end_rec = false; read_ray_hdr = false; @@ -411,7 +543,7 @@ public class SigmetVolumeScan { } } raf.seek(cur_len); - if (pos > 0) { + if (posInRay_relative > 0) { break; } } else if (a00 > 0 & a00 != 1) { @@ -421,13 +553,19 @@ public class SigmetVolumeScan { // dd[nb + k] = SigmetIOServiceProvider.calcData(recHdr, dty, (byte) 0); // } - nb = nb + num_zero; + int nb_before = posInRay_absolute / bytesToRead; + // sanity check + if (nb_before != nb) + throw new IllegalStateException("nb_before != nb"); + posInRay_absolute += num_zero; + int nb_after = posInRay_absolute / bytesToRead; + nb = nb + (nb_after - nb_before); if (cur_len % REC_SIZE == 0) { beg_rec = true; end_rec = false; read_ray_hdr = false; - pos = 0; + posInRay_relative = 0; data_read = 0; break; @@ -438,33 +576,61 @@ public class SigmetVolumeScan { if (cur_len % REC_SIZE == 0) { len = cur_len; + // will get lost otherwise + if (ray != null) { + beg_rec = true; + end_rec = true; + read_ray_hdr = true; + posInRay_relative = 0; + posInRay_absolute = 0; + data_read = 0; + nb = 0; + len = cur_len; + + if (ray == null) + throw new IllegalStateException("ray == null"); + + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); + } + varL.add(ray); + lastRay = ray; + ray = null; + } + continue; } raf.seek(cur_len); if (nb == (int) num_bins) { - a00 = raf.readShort(); + a00 = raf.readShort(); // should be 1 == end of ray + if (a00 != 1) + // should not be able to get here + logger.warn("nb == num_bins but a00 != 1 - something seems wrong"); cur_len = cur_len + 2; end_rec = true; - ray = new Ray(range_first, step, az, elev, num_bins, time_start_sw, rayoffset, datalen, rayoffset1, nsweep, - var_name, dty); + ray = new Ray(range_first, step, az, elev, num_bins, (short) nb, time_start_sw, rayoffset, datalen, rayoffset1, + nsweep, var_name, dty, bytesToRead); rays_count++; - two++; + + // last ray of last sweep -> end of file if ((nsweep == number_sweeps) & (rays_count % beg == 0)) { - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found1 {}", var_name); + if (ray == null) + throw new IllegalStateException("ray == null"); + + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); } + varL.add(ray); + lastRay = ray; + ray = null; break; } @@ -472,46 +638,47 @@ public class SigmetVolumeScan { beg_rec = true; end_rec = true; read_ray_hdr = true; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; len = cur_len; - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found2 {}", var_name); + + if (ray == null) + throw new IllegalStateException("ray == null"); + + // using a universal structure that works for all data types + List varL = all.get(var_name.trim()); + if (varL == null) { + varL = new ArrayList<>(); + all.put(var_name.trim(), varL); } + varL.add(ray); + lastRay = ray; + ray = null; + continue; } } - // "TotalPower", "Reflectivity", "Velocity", "Width", "DifferentialReflectivity" if (firstRay == null) firstRay = ray; - if (var_name.trim().equalsIgnoreCase("TotalPower")) { - totalPower.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Reflectivity")) { - reflectivity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Velocity")) { - velocity.add(ray); - } else if (var_name.trim().equalsIgnoreCase("Width")) { - width.add(ray); - } else if (var_name.trim().equalsIgnoreCase("DifferentialReflectivity")) { - diffReflectivity.add(ray); - } else { - logger.warn(" Error: Unknown Radial Variable found3 {}", var_name); + if (ray == null) + throw new IllegalStateException("ray == null"); + + // using a universal structure that works for all data types + List additionalL = all.get(var_name.trim()); + if (additionalL == null) { + additionalL = new ArrayList<>(); + all.put(var_name.trim(), additionalL); } + additionalL.add(ray); + lastRay = ray; + ray = null; - pos = 0; + posInRay_relative = 0; + posInRay_absolute = 0; data_read = 0; nb = 0; read_ray_hdr = true; @@ -528,32 +695,12 @@ public class SigmetVolumeScan { len = cur_len; } // ------------end of outer while --------------- - lastRay = ray; - if (!reflectivity.isEmpty()) { - reflectivityGroups = sortScans("reflectivity", reflectivity, 1000); - hasReflectivity = true; - } - if (!velocity.isEmpty()) { - velocityGroups = sortScans("velocity", velocity, 1000); - hasVelocity = true; - } - if (!totalPower.isEmpty()) { - totalPowerGroups = sortScans("totalPower", totalPower, 1000); - hasTotalPower = true; - } - if (!width.isEmpty()) { - widthGroups = sortScans("width", width, 1000); - hasWidth = true; - } - if (!diffReflectivity.isEmpty()) { - differentialReflectivityGroups = sortScans("diffReflectivity", diffReflectivity, 1000); - hasDifferentialReflectivity = true; - } - - if (!time.isEmpty()) { - timeGroups = sortScans("diffReflectivity", diffReflectivity, 1000); - hasTime = true; + // using a universal structure that works for all data types + for (String var_name : all.keySet()) { + List additionalL = all.get(var_name); + if (!additionalL.isEmpty()) + allGroups.put(var_name, sortScans(var_name, additionalL, 1000)); } // --------- fill all of values in the ncfile ------ @@ -624,24 +771,13 @@ public int compare(List group1, List group2) { } } - public List> getTotalPowerGroups() { - return totalPowerGroups; - } - - public List> getVelocityGroups() { - return velocityGroups; - } - - public List> getWidthGroups() { - return widthGroups; - } - - public List> getReflectivityGroups() { - return reflectivityGroups; + // using a universal structure that works for all data types + public List> getGroup(String name) { + return allGroups.get(name); } - public List> getDifferentialReflectivityGroups() { - return differentialReflectivityGroups; + public short[] getDataTypes() { + return data_type; } public int[] getNumberGates() { @@ -660,7 +796,7 @@ public int[] getStartSweep() { */ void checkSort(Ray[] r) { int j = 0, n = 0, n1, n2; - short time1, time2; + int time1, time2; int[] k1 = new int[300]; int[] k2 = new int[300]; // define the groups of rays with the same "time". For ex.: @@ -676,7 +812,7 @@ void checkSort(Ray[] r) { k1[j] = i + 1; } } - if (k2[j] < r.length - 1) { + if (k2[j] < r.length - 1 && j > 0) { k1[j] = k2[j - 1] + 1; k2[j] = r.length - 1; }