diff --git a/.github/workflows/tds-check.yml b/.github/workflows/tds-check.yml index 03cb1437f2..b9f7b2aff1 100644 --- a/.github/workflows/tds-check.yml +++ b/.github/workflows/tds-check.yml @@ -44,9 +44,7 @@ jobs: sed -i 's/versions\["ncj"\] =.*$/versions["ncj"] ="'${NCJ_VERSION}'"/g' ./gradle/any/dependencies.gradle # setup env vars for tds buildTestDSR CONTENT_ROOT="-Dtds.content.root.path=${TDS_BUILD_DIR}/tds/src/test/content" - DOWNLOAD_DIR="-Dtds.download.dir=/tmp/download" - UPLOAD_DIR="-Dtds.upload.dir=/tmp/upload" - SYSTEM_PROPS="$CONTENT_ROOT $DOWNLOAD_DIR $UPLOAD_DIR" + SYSTEM_PROPS="$CONTENT_ROOT" # run tds tests echo "Run the TDS tests" ./gradlew $SYSTEM_PROPS --info --stacktrace testAll @@ -96,10 +94,8 @@ jobs: sed -i 's/versions\["ncj"\] =.*$/versions["ncj"] ="'${NCJ_VERSION}'"/g' ./gradle/any/dependencies.gradle # setup env vars for tds buildTestDSR CONTENT_ROOT="-Dtds.content.root.path=${TDS_BUILD_DIR}/tds/src/test/content" - DOWNLOAD_DIR="-Dtds.download.dir=/tmp/download" - UPLOAD_DIR="-Dtds.upload.dir=/tmp/upload" BUILDERS="-Dthredds.test.experimental.useNetcdfJavaBuilders=true" - SYSTEM_PROPS="$CONTENT_ROOT $DOWNLOAD_DIR $UPLOAD_DIR $BUILDERS" + SYSTEM_PROPS="$CONTENT_ROOT $BUILDERS" # run tds tests echo "Run the TDS tests" ./gradlew $SYSTEM_PROPS --info --stacktrace testAll --refresh-dependencies diff --git a/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageDatasetFactory.java b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageDatasetFactory.java new file mode 100644 index 0000000000..3a643e7fc1 --- /dev/null +++ b/cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageDatasetFactory.java @@ -0,0 +1,61 @@ +package ucar.nc2.ft.coverage; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.ft2.coverage.CoverageDatasetFactory; +import ucar.nc2.ft2.coverage.FeatureDatasetCoverage; +import ucar.unidata.io.RandomAccessFile; + +public class TestCoverageDatasetFactory { + + @BeforeClass + public static void setupCaches() { + RandomAccessFile.enableDefaultGlobalFileCache(); + NetcdfDatasets.initNetcdfFileCache(1, 10, 15, 200); + } + + @AfterClass + public static void shutdownCaches() { + NetcdfDatasets.shutdown(); + RandomAccessFile.setGlobalFileCache(null); + } + + @After + public void cleanupAfterEach() { + NetcdfDatasets.getNetcdfFileCache().clearCache(true); + RandomAccessFile.getGlobalFileCache().clearCache(true); + } + + @Test + public void shouldReleaseCacheResources() throws Exception { + final String filename = "src/test/data/ucar/nc2/ft/coverage/coverageWithLambertConformalConic_m.nc"; + + try (FeatureDatasetCoverage featureDatasetCoverage = CoverageDatasetFactory.open(filename)) { + assertThat(featureDatasetCoverage).isNotNull(); + } + assertThatCacheFileIsNotLocked(); + } + + @Test + public void shouldReleaseCacheResourcesWhenGridsAreEmpty() throws Exception { + final String filename = "../cdm/core/src/test/data/pointPre1.6/pointUnlimited.nc"; + + try (FeatureDatasetCoverage featureDatasetCoverage = CoverageDatasetFactory.open(filename)) { + assertThat(featureDatasetCoverage).isNull(); + } + assertThatCacheFileIsNotLocked(); + } + + private static void assertThatCacheFileIsNotLocked() { + assertThat(RandomAccessFile.getGlobalFileCache().showCache().size()).isEqualTo(1); + assertThat(RandomAccessFile.getGlobalFileCache().showCache().get(0)).startsWith("false"); + + assertThat(NetcdfDatasets.getNetcdfFileCache().showCache().size()).isEqualTo(1); + assertThat(NetcdfDatasets.getNetcdfFileCache().showCache().get(0)).startsWith("false"); + } +} diff --git a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java index a3ab1aba73..32835fcb59 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java +++ b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java @@ -79,6 +79,7 @@ public static Optional openCoverageDataset(String endpoi return Optional.empty(errlog.toString()); } + gds.close(); return Optional.empty("Could not open as a coverage dataset"); } diff --git a/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CF1Convention.java b/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CF1Convention.java index 00bc65c6a0..bbb1d5b89c 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CF1Convention.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CF1Convention.java @@ -193,7 +193,9 @@ protected void augmentGroup(Group.Builder gb, CancelTask cancelTask) throws IOEx } // look for time variables and check to see if they have a calendar attribute. if not, add the default - checkTimeVarForCalendar((VariableDS.Builder) vb); + if (vb instanceof VariableDS.Builder) { + checkTimeVarForCalendar((VariableDS.Builder) vb); + } } // look for horiz transforms. only ones that are referenced by another variable. diff --git a/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CoardsConventions.java b/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CoardsConventions.java index b28298041d..49f425b513 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CoardsConventions.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/dataset/conv/CoardsConventions.java @@ -48,12 +48,12 @@ public CoordSystemBuilder open(NetcdfDataset.Builder datasetBuilder) { * time in such cases. */ - CoardsConventions(NetcdfDataset.Builder datasetBuilder) { + CoardsConventions(NetcdfDataset.Builder datasetBuilder) { super(datasetBuilder); this.conventionName = CONVENTION_NAME; } - boolean checkTimeVarForCalendar(VariableDS.Builder vb) { + boolean checkTimeVarForCalendar(VariableDS.Builder vb) { boolean hasChanged = false; String unit = vb.getUnits(); if (unit != null) { @@ -72,9 +72,9 @@ boolean checkTimeVarForCalendar(VariableDS.Builder vb) { @Override protected void augmentDataset(CancelTask cancelTask) throws IOException { - for (Variable.Builder vb : rootGroup.vbuilders) { + for (Variable.Builder vb : rootGroup.vbuilders) { if (vb instanceof VariableDS.Builder) { - checkTimeVarForCalendar((VariableDS.Builder) vb); + checkTimeVarForCalendar((VariableDS.Builder) vb); } } } diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java index a9bc19fd00..181a5e9d47 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5headerNew.java @@ -161,7 +161,6 @@ public static boolean isValidFile(RandomAccessFile raf) throws IOException { * 3) all variables' dimensions have a dimension scale */ - private final RandomAccessFile raf; private final Group.Builder root; private final H5iospNew h5iosp; @@ -185,8 +184,7 @@ public static boolean isValidFile(RandomAccessFile raf) throws IOException { private final Charset valueCharset; - H5headerNew(RandomAccessFile myRaf, Group.Builder root, H5iospNew h5iosp) { - this.raf = myRaf; + H5headerNew(Group.Builder root, H5iospNew h5iosp) { this.root = root; this.h5iosp = h5iosp; valueCharset = h5iosp.getValueCharset().orElse(StandardCharsets.UTF_8); @@ -212,7 +210,7 @@ public void read(PrintWriter debugPS) throws IOException { } h5objects = new H5objects(this, debugOut, memTracker); - long actualSize = raf.length(); + long actualSize = getRandomAccessFile().length(); if (debugTracker) memTracker = new MemTracker(actualSize); @@ -221,8 +219,8 @@ public void read(PrintWriter debugPS) throws IOException { boolean ok = false; long filePos = 0; while ((filePos < actualSize - 8)) { - raf.seek(filePos); - String magic = raf.readString(8); + getRandomAccessFile().seek(filePos); + String magic = getRandomAccessFile().readString(8); if (magic.equals(magicString)) { ok = true; break; @@ -233,19 +231,19 @@ public void read(PrintWriter debugPS) throws IOException { throw new IOException("Not a netCDF4/HDF5 file "); } if (debug1) { - log.debug("H5header opened file to read:'{}' size= {}", raf.getLocation(), actualSize); + log.debug("H5header opened file to read:'{}' size= {}", getRandomAccessFile().getLocation(), actualSize); } // now we are positioned right after the header // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); - long superblockStart = raf.getFilePointer() - 8; + long superblockStart = getRandomAccessFile().getFilePointer() - 8; if (debugTracker) memTracker.add("header", 0, superblockStart); // superblock version - byte versionSB = raf.readByte(); + byte versionSB = getRandomAccessFile().readByte(); if (versionSB < 2) { readSuperBlock1(superblockStart, versionSB); @@ -281,43 +279,43 @@ private void readSuperBlock1(long superblockStart, byte versionSB) throws IOExce long eofAddress; long driverBlockAddress; - versionFSS = raf.readByte(); - versionGroup = raf.readByte(); - raf.readByte(); // skip 1 byte - versionSHMF = raf.readByte(); + versionFSS = getRandomAccessFile().readByte(); + versionGroup = getRandomAccessFile().readByte(); + getRandomAccessFile().readByte(); // skip 1 byte + versionSHMF = getRandomAccessFile().readByte(); if (debugDetail) { log.debug(" versionSB= " + versionSB + " versionFSS= " + versionFSS + " versionGroup= " + versionGroup + " versionSHMF= " + versionSHMF); } - sizeOffsets = raf.readByte(); + sizeOffsets = getRandomAccessFile().readByte(); isOffsetLong = (sizeOffsets == 8); - sizeLengths = raf.readByte(); + sizeLengths = getRandomAccessFile().readByte(); isLengthLong = (sizeLengths == 8); if (debugDetail) { log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths); log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong); } - raf.read(); // skip 1 byte + getRandomAccessFile().read(); // skip 1 byte // log.debug(" position="+mapBuffer.position()); - btreeLeafNodeSize = raf.readShort(); - btreeInternalNodeSize = raf.readShort(); + btreeLeafNodeSize = getRandomAccessFile().readShort(); + btreeInternalNodeSize = getRandomAccessFile().readShort(); if (debugDetail) { log.debug(" btreeLeafNodeSize= {} btreeInternalNodeSize= {}", btreeLeafNodeSize, btreeInternalNodeSize); } // log.debug(" position="+mapBuffer.position()); - fileFlags = raf.readInt(); + fileFlags = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags)); } if (versionSB == 1) { - short storageInternalNodeSize = raf.readShort(); - raf.skipBytes(2); + short storageInternalNodeSize = getRandomAccessFile().readShort(); + getRandomAccessFile().skipBytes(2); } baseAddress = readOffset(); @@ -337,36 +335,36 @@ private void readSuperBlock1(long superblockStart, byte versionSB) throws IOExce log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress)); log.debug(" global free space heap Address= 0x{}", Long.toHexString(heapAddress)); log.debug(" eof Address={}", eofAddress); - log.debug(" raf length= {}", raf.length()); + log.debug(" raf length= {}", getRandomAccessFile().length()); log.debug(" driver BlockAddress= 0x{}", Long.toHexString(driverBlockAddress)); log.debug(""); } if (debugTracker) - memTracker.add("superblock", superblockStart, raf.getFilePointer()); + memTracker.add("superblock", superblockStart, getRandomAccessFile().getFilePointer()); // look for file truncation - long fileSize = raf.length(); + long fileSize = getRandomAccessFile().length(); if (fileSize < eofAddress) - throw new IOException( - "File is truncated should be= " + eofAddress + " actual = " + fileSize + "%nlocation= " + raf.getLocation()); + throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize + "%nlocation= " + + getRandomAccessFile().getLocation()); // next comes the root object's SymbolTableEntry // extract the root group object, recursively read all objects - h5rootGroup = h5objects.readRootSymbolTable(raf.getFilePointer()); + h5rootGroup = h5objects.readRootSymbolTable(getRandomAccessFile().getFilePointer()); } private void readSuperBlock2(long superblockStart) throws IOException { - sizeOffsets = raf.readByte(); + sizeOffsets = getRandomAccessFile().readByte(); isOffsetLong = (sizeOffsets == 8); - sizeLengths = raf.readByte(); + sizeLengths = getRandomAccessFile().readByte(); isLengthLong = (sizeLengths == 8); if (debugDetail) { log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths); log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong); } - byte fileFlags = raf.readByte(); + byte fileFlags = getRandomAccessFile().readByte(); if (debugDetail) { log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags)); } @@ -375,7 +373,7 @@ private void readSuperBlock2(long superblockStart) throws IOException { long extensionAddress = readOffset(); long eofAddress = readOffset(); long rootObjectAddress = readOffset(); - int checksum = raf.readInt(); + int checksum = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress)); @@ -386,7 +384,7 @@ private void readSuperBlock2(long superblockStart) throws IOException { } if (debugTracker) - memTracker.add("superblock", superblockStart, raf.getFilePointer()); + memTracker.add("superblock", superblockStart, getRandomAccessFile().getFilePointer()); if (baseAddress != superblockStart) { baseAddress = superblockStart; @@ -397,7 +395,7 @@ private void readSuperBlock2(long superblockStart) throws IOException { } // look for file truncation - long fileSize = raf.length(); + long fileSize = getRandomAccessFile().length(); if (fileSize < eofAddress) { throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize); } @@ -981,7 +979,7 @@ private void makeAttributes(Structure.Builder sb, MessageAttribute matt, Attr // reading attribute values might change byte order during a read // put back to little endian for further header processing - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); } private Attribute makeAttribute(MessageAttribute matt) throws IOException { @@ -1001,7 +999,7 @@ private Attribute makeAttribute(MessageAttribute matt) throws IOException { attData = readAttributeData(matt, vinfo, dtype); } catch (InvalidRangeException e) { - log.warn("failed to read Attribute " + matt.name + " HDF5 file=" + raf.getLocation()); + log.warn("failed to read Attribute " + matt.name + " HDF5 file=" + getRandomAccessFile().getLocation()); return null; } @@ -1019,7 +1017,7 @@ private Attribute makeAttribute(MessageAttribute matt) throws IOException { result = new Attribute(matt.name, attData); } - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); return result; } @@ -1082,8 +1080,8 @@ private Array readAttributeData(MessageAttribute matt, H5headerNew.Vinfo vinfo, } // copy bytes directly into the underlying byte[] - raf.seek(chunk.getSrcPos()); - raf.readFully(byteArray, (int) chunk.getDestElem() * recsize, chunk.getNelems() * recsize); + getRandomAccessFile().seek(chunk.getSrcPos()); + getRandomAccessFile().readFully(byteArray, (int) chunk.getDestElem() * recsize, chunk.getNelems() * recsize); } // strings are stored on the heap, and must be read separately @@ -1700,7 +1698,7 @@ public long[] countStorageSize(Formatter f) throws IOException { isvlen = this.mdt.isVlen(); if (!facade.dobj.mdt.isOK && warnings) { - log.debug("WARNING HDF5 file " + raf.getLocation() + " not handling " + facade.dobj.mdt); + log.debug("WARNING HDF5 file " + getRandomAccessFile().getLocation() + " not handling " + facade.dobj.mdt); return; // not a supported datatype } @@ -1728,7 +1726,7 @@ public long[] countStorageSize(Formatter f) throws IOException { this.dataPos = dataPos; if (!mdt.isOK && warnings) { - log.debug("WARNING HDF5 file " + raf.getLocation() + " not handling " + mdt); + log.debug("WARNING HDF5 file " + getRandomAccessFile().getLocation() + " not handling " + mdt); return; // not a supported datatype } @@ -1929,7 +1927,8 @@ Array readArray() throws IOException { // cant happen because we use null for wantSection throw new IllegalStateException(); } - Object data = IospHelper.readDataFill(raf, layout, dataType, getFillValue(), typeInfo.endian, false); + Object data = + IospHelper.readDataFill(getRandomAccessFile(), layout, dataType, getFillValue(), typeInfo.endian, false); return Array.factory(dataType, shape, data); } @@ -1948,7 +1947,8 @@ String readString() throws IOException { // cant happen because we use null for wantSection throw new IllegalStateException(); } - Object data = IospHelper.readDataFill(raf, layout, dataType, getFillValue(), typeInfo.endian, true); + Object data = + IospHelper.readDataFill(getRandomAccessFile(), layout, dataType, getFillValue(), typeInfo.endian, true); String result = ""; if (data instanceof String) { @@ -1987,10 +1987,10 @@ else if (size == 4) else if (size == 8) return DataType.LONG.withSignedness(signedness); else if (warnings) { - log.debug("WARNING HDF5 file " + raf.getLocation() + " not handling hdf integer type (" + hdfType + log.debug("WARNING HDF5 file " + getRandomAccessFile().getLocation() + " not handling hdf integer type (" + + hdfType + ") with size= " + size); + log.warn("HDF5 file " + getRandomAccessFile().getLocation() + " not handling hdf integer type (" + hdfType + ") with size= " + size); - log.warn( - "HDF5 file " + raf.getLocation() + " not handling hdf integer type (" + hdfType + ") with size= " + size); return null; } @@ -2000,8 +2000,10 @@ else if (warnings) { else if (size == 8) return DataType.DOUBLE; else if (warnings) { - log.debug("WARNING HDF5 file " + raf.getLocation() + " not handling hdf float type with size= " + size); - log.warn("HDF5 file " + raf.getLocation() + " not handling hdf float type with size= " + size); + log.debug("WARNING HDF5 file " + getRandomAccessFile().getLocation() + + " not handling hdf float type with size= " + size); + log.warn( + "HDF5 file " + getRandomAccessFile().getLocation() + " not handling hdf float type with size= " + size); return null; } @@ -2018,9 +2020,11 @@ else if (warnings) { return null; // dunno } else if (warnings) { - log.warn("HDF5 file " + raf.getLocation() + " not handling hdf type = " + hdfType + " size= " + size); + log.warn("HDF5 file " + getRandomAccessFile().getLocation() + " not handling hdf type = " + hdfType + " size= " + + size); } else { - log.debug("HDF5 file " + raf.getLocation() + " not handling hdf type = " + hdfType + " size= " + size); + log.debug("HDF5 file " + getRandomAccessFile().getLocation() + " not handling hdf type = " + hdfType + " size= " + + size); } return null; } @@ -2085,43 +2089,43 @@ Array getHeapDataArray(HeapIdentifier heapId, DataType dataType, int endian) log.debug(" HeapObject= {}", ho); } if (endian >= 0) { - raf.order(endian); + getRandomAccessFile().order(endian); } if (DataType.FLOAT == dataType) { float[] pa = new float[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readFloat(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readFloat(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (DataType.DOUBLE == dataType) { double[] pa = new double[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readDouble(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readDouble(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == byte.class) { byte[] pa = new byte[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readFully(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readFully(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == short.class) { short[] pa = new short[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readShort(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readShort(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == int.class) { int[] pa = new int[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readInt(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readInt(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == long.class) { long[] pa = new long[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readLong(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readLong(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } @@ -2145,8 +2149,8 @@ String readHeapString(long heapIdAddress) throws IOException { throw new IllegalStateException("Cant find Heap Object,heapId=" + heapId); if (ho.dataSize > 1000 * 1000) return String.format("Bad HeapObject.dataSize=%s", ho); - raf.seek(ho.dataPos); - return raf.readString((int) ho.dataSize, valueCharset); + getRandomAccessFile().seek(ho.dataPos); + return getRandomAccessFile().readString((int) ho.dataSize, valueCharset); } /** @@ -2165,8 +2169,8 @@ String readHeapString(ByteBuffer bb, int pos) throws IOException { GlobalHeap.HeapObject ho = heapId.getHeapObject(); if (ho == null) throw new IllegalStateException("Cant find Heap Object,heapId=" + heapId); - raf.seek(ho.dataPos); - return raf.readString((int) ho.dataSize, valueCharset); + getRandomAccessFile().seek(ho.dataPos); + return getRandomAccessFile().readString((int) ho.dataSize, valueCharset); } Array readHeapVlen(ByteBuffer bb, int pos, DataType dataType, int endian) throws IOException, InvalidRangeException { @@ -2243,12 +2247,12 @@ public boolean isOffsetLong() { @Override public long readLength() throws IOException { - return isLengthLong ? raf.readLong() : (long) raf.readInt(); + return isLengthLong ? getRandomAccessFile().readLong() : (long) getRandomAccessFile().readInt(); } @Override public long readOffset() throws IOException { - return isOffsetLong ? raf.readLong() : (long) raf.readInt(); + return isOffsetLong ? getRandomAccessFile().readLong() : (long) getRandomAccessFile().readInt(); } @Override @@ -2276,17 +2280,17 @@ public int getNumBytesFromMax(long maxNumber) { public long readVariableSizeUnsigned(int size) throws IOException { long vv; if (size == 1) { - vv = DataType.unsignedByteToShort(raf.readByte()); + vv = DataType.unsignedByteToShort(getRandomAccessFile().readByte()); } else if (size == 2) { if (debugPos) { - log.debug("position={}", raf.getFilePointer()); + log.debug("position={}", getRandomAccessFile().getFilePointer()); } - short s = raf.readShort(); + short s = getRandomAccessFile().readShort(); vv = DataType.unsignedShortToInt(s); } else if (size == 4) { - vv = DataType.unsignedIntToLong(raf.readInt()); + vv = DataType.unsignedIntToLong(getRandomAccessFile().readInt()); } else if (size == 8) { - vv = raf.readLong(); + vv = getRandomAccessFile().readLong(); } else { vv = readVariableSizeN(size); } @@ -2297,7 +2301,7 @@ public long readVariableSizeUnsigned(int size) throws IOException { private long readVariableSizeN(int nbytes) throws IOException { int[] ch = new int[nbytes]; for (int i = 0; i < nbytes; i++) - ch[i] = raf.read(); + ch[i] = getRandomAccessFile().read(); long result = ch[nbytes - 1]; for (int i = nbytes - 2; i >= 0; i--) { @@ -2310,7 +2314,7 @@ private long readVariableSizeN(int nbytes) throws IOException { @Override public RandomAccessFile getRandomAccessFile() { - return raf; + return h5iosp.getRandomAccessFile(); } @Override @@ -2323,6 +2327,10 @@ public byte getSizeOffsets() { return sizeOffsets; } + H5objects getH5objects() { + return h5objects; + } + boolean isNetcdf4() { return isNetcdf4; } @@ -2340,7 +2348,7 @@ public void close() { } public void getEosInfo(Formatter f) throws IOException { - HdfEos.getEosInfo(raf.getLocation(), this, root, f); + HdfEos.getEosInfo(getRandomAccessFile().getLocation(), this, root, f); } // debug - hdf5Table diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java index 5b0039d2ae..e881ee9103 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java @@ -97,6 +97,10 @@ public String getFileTypeDescription() { return "Hierarchical Data Format, version 5"; } + RandomAccessFile getRandomAccessFile() { + return raf; + } + public static void useHdfEos(boolean val) { useHdfEos = val; } @@ -118,7 +122,7 @@ public void build(RandomAccessFile raf, Group.Builder rootGroup, CancelTask canc super.open(raf, rootGroup.getNcfile(), cancelTask); raf.order(RandomAccessFile.BIG_ENDIAN); - header = new H5headerNew(raf, rootGroup, this); + header = new H5headerNew(rootGroup, this); header.read(null); // check if its an HDF5-EOS file @@ -171,7 +175,7 @@ protected void setValueCharset(@Nullable Charset charset) { public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException { super.open(raf, ncfile, cancelTask); Group.Builder rootGroup = Group.builder().setName("").setNcfile(ncfile); - header = new H5headerNew(raf, rootGroup, this); + header = new H5headerNew(rootGroup, this); header.read(null); ncfile.setRootGroup(rootGroup.build()); diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5objects.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5objects.java index 89b5439bf7..110885a280 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5objects.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5objects.java @@ -37,7 +37,6 @@ public class H5objects { private static boolean debugDimensionScales; private final H5headerNew header; - private final RandomAccessFile raf; private final PrintWriter debugOut; private final MemTracker memTracker; @@ -45,11 +44,14 @@ public class H5objects { H5objects(H5headerNew header, PrintWriter debugOut, MemTracker memTracker) { this.header = header; - this.raf = header.getRandomAccessFile(); this.debugOut = debugOut; this.memTracker = memTracker; } + RandomAccessFile getRandomAccessFile() { + return header.getRandomAccessFile(); + } + H5Group readRootSymbolTable(long pos) throws IOException { // The root object's SymbolTableEntry SymbolTableEntry rootEntry = new SymbolTableEntry(pos); @@ -308,63 +310,63 @@ private DataObject(long address, String who) throws IOException { log.debug("\n--> DataObject.read parsing <" + who + "> object ID/address=" + address); } if (debugPos) { - log.debug(" DataObject.read now at position=" + raf.getFilePointer() + " for <" + who + "> reposition to " - + header.getFileOffset(address)); + log.debug(" DataObject.read now at position=" + getRandomAccessFile().getFilePointer() + " for <" + who + + "> reposition to " + header.getFileOffset(address)); } // if (offset < 0) return null; - raf.seek(header.getFileOffset(address)); + getRandomAccessFile().seek(header.getFileOffset(address)); - version = raf.readByte(); + version = getRandomAccessFile().readByte(); if (version == 1) { // Level 2A1 (first part, before the messages) - raf.readByte(); // skip byte - short nmess = raf.readShort(); + getRandomAccessFile().readByte(); // skip byte + short nmess = getRandomAccessFile().readShort(); if (debugDetail) { log.debug(" version=" + version + " nmess=" + nmess); } - int referenceCount = raf.readInt(); - int headerSize = raf.readInt(); + int referenceCount = getRandomAccessFile().readInt(); + int headerSize = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" referenceCount=" + referenceCount + " headerSize=" + headerSize); } // if (referenceCount > 1) // log.debug("WARNING referenceCount="+referenceCount); - raf.skipBytes(4); // header messages multiples of 8 + getRandomAccessFile().skipBytes(4); // header messages multiples of 8 - long posMess = raf.getFilePointer(); + long posMess = getRandomAccessFile().getFilePointer(); int count = readMessagesVersion1(posMess, nmess, Integer.MAX_VALUE, this.who); if (debugContinueMessage) { log.debug(" nmessages read = {}", count); } if (debugPos) { - log.debug("<--done reading messages for <" + who + ">; position=" + raf.getFilePointer()); + log.debug("<--done reading messages for <" + who + ">; position=" + getRandomAccessFile().getFilePointer()); } if (debugTracker) memTracker.addByLen("Object " + who, header.getFileOffset(address), headerSize + 16); } else { // level 2A2 (first part, before the messages) // first byte was already read - String magic = raf.readString(3); + String magic = getRandomAccessFile().readString(3); if (!magic.equals("HDR")) throw new IllegalStateException("DataObject doesnt start with OHDR"); - version = raf.readByte(); - byte flags = raf.readByte(); // data object header flags (version 2) + version = getRandomAccessFile().readByte(); + byte flags = getRandomAccessFile().readByte(); // data object header flags (version 2) if (debugDetail) { log.debug(" version=" + version + " flags=" + Integer.toBinaryString(flags)); } // raf.skipBytes(2); if (((flags >> 5) & 1) == 1) { - int accessTime = raf.readInt(); - int modTime = raf.readInt(); - int changeTime = raf.readInt(); - int birthTime = raf.readInt(); + int accessTime = getRandomAccessFile().readInt(); + int modTime = getRandomAccessFile().readInt(); + int changeTime = getRandomAccessFile().readInt(); + int birthTime = getRandomAccessFile().readInt(); } if (((flags >> 4) & 1) == 1) { - short maxCompactAttributes = raf.readShort(); - short minDenseAttributes = raf.readShort(); + short maxCompactAttributes = getRandomAccessFile().readShort(); + short minDenseAttributes = getRandomAccessFile().readShort(); } long sizeOfChunk = readVariableSizeFactor(flags & 3); @@ -372,13 +374,13 @@ private DataObject(long address, String who) throws IOException { log.debug(" sizeOfChunk=" + sizeOfChunk); } - long posMess = raf.getFilePointer(); + long posMess = getRandomAccessFile().getFilePointer(); int count = readMessagesVersion2(posMess, sizeOfChunk, (flags & 4) != 0, this.who); if (debugContinueMessage) { log.debug(" nmessages read = {}", count); } if (debugPos) { - log.debug("<--done reading messages for <" + who + ">; position=" + raf.getFilePointer()); + log.debug("<--done reading messages for <" + who + ">; position=" + getRandomAccessFile().getFilePointer()); } } @@ -520,7 +522,7 @@ private int readMessagesVersion2(long filePos, long maxBytes, boolean creationOr if (debugContinueMessage) debugOut.println(" ---ObjectHeaderContinuation filePos= " + continuationBlockFilePos); - raf.seek(continuationBlockFilePos); + getRandomAccessFile().seek(continuationBlockFilePos); String sig = readStringFixedLength(4); if (!sig.equals("OCHK")) throw new IllegalStateException(" ObjectHeaderContinuation Missing signature"); @@ -664,28 +666,28 @@ public long getStart() { */ int read(long filePos, int version, boolean creationOrderPresent, String objectName) throws IOException { this.start = filePos; - raf.seek(filePos); + getRandomAccessFile().seek(filePos); if (debugPos) { - log.debug(" --> Message Header starts at =" + raf.getFilePointer()); + log.debug(" --> Message Header starts at =" + getRandomAccessFile().getFilePointer()); } if (version == 1) { - type = raf.readShort(); - size = DataType.unsignedShortToInt(raf.readShort()); - headerMessageFlags = raf.readByte(); - raf.skipBytes(3); + type = getRandomAccessFile().readShort(); + size = DataType.unsignedShortToInt(getRandomAccessFile().readShort()); + headerMessageFlags = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); header_length = 8; } else { - type = raf.readByte(); - size = DataType.unsignedShortToInt(raf.readShort()); + type = getRandomAccessFile().readByte(); + size = DataType.unsignedShortToInt(getRandomAccessFile().readShort()); // if (size > Short.MAX_VALUE) // log.debug("HEY"); - headerMessageFlags = raf.readByte(); + headerMessageFlags = getRandomAccessFile().readByte(); header_length = 4; if (creationOrderPresent) { - creationOrder = raf.readShort(); + creationOrder = getRandomAccessFile().readShort(); header_length += 2; } } @@ -697,7 +699,7 @@ int read(long filePos, int version, boolean creationOrderPresent, String objectN } } if (debugPos) { - log.debug(" --> Message Data starts at=" + raf.getFilePointer()); + log.debug(" --> Message Data starts at=" + getRandomAccessFile().getFilePointer()); } if ((headerMessageFlags & 2) != 0) { // shared @@ -755,7 +757,7 @@ int read(long filePos, int version, boolean creationOrderPresent, String objectN } else if (mtype == MessageType.Attribute) { // 12 MessageAttribute data = new MessageAttribute(); - data.read(raf.getFilePointer()); + data.read(getRandomAccessFile().getFilePointer()); messData = data; } else if (mtype == MessageType.Comment) { // 13 @@ -835,17 +837,17 @@ public void showCompression(Formatter f) { } private DataObject getSharedDataObject(MessageType mtype) throws IOException { - byte sharedVersion = raf.readByte(); - byte sharedType = raf.readByte(); + byte sharedVersion = getRandomAccessFile().readByte(); + byte sharedType = getRandomAccessFile().readByte(); if (sharedVersion == 1) - raf.skipBytes(6); + getRandomAccessFile().skipBytes(6); if ((sharedVersion == 3) && (sharedType == 1)) { - long heapId = raf.readLong(); + long heapId = getRandomAccessFile().readLong(); if (debug1) { log.debug(" Shared Message " + sharedVersion + " type=" + sharedType + " heapId = " + heapId); } if (debugPos) { - log.debug(" --> Shared Message reposition to =" + raf.getFilePointer()); + log.debug(" --> Shared Message reposition to =" + getRandomAccessFile().getFilePointer()); } // dunno where is the file's shared object header heap ?? throw new UnsupportedOperationException("****SHARED MESSAGE type = " + mtype + " heapId = " + heapId); @@ -907,20 +909,20 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageSimpleDataspace start pos= " + raf.getFilePointer()); + log.debug(" *MessageSimpleDataspace start pos= " + getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); + byte version = getRandomAccessFile().readByte(); if (version == 1) { - ndims = raf.readByte(); - flags = raf.readByte(); + ndims = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); type = (byte) ((ndims == 0) ? 0 : 1); - raf.skipBytes(5); // skip 5 bytes + getRandomAccessFile().skipBytes(5); // skip 5 bytes } else if (version == 2) { - ndims = raf.readByte(); - flags = raf.readByte(); - type = raf.readByte(); + ndims = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); + type = getRandomAccessFile().readByte(); } else { throw new IllegalStateException("MessageDataspace: unknown version= " + version); @@ -1009,12 +1011,12 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageGroupNew start pos= " + raf.getFilePointer()); + log.debug(" *MessageGroupNew start pos= " + getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); - byte flags = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + byte flags = getRandomAccessFile().readByte(); if ((flags & 1) != 0) { - maxCreationIndex = raf.readLong(); + maxCreationIndex = getRandomAccessFile().readLong(); } fractalHeapAddress = header.readOffset(); @@ -1052,19 +1054,19 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageGroupInfo start pos= " + raf.getFilePointer()); + log.debug(" *MessageGroupInfo start pos= " + getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); - flags = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); if ((flags & 1) != 0) { - maxCompactValue = raf.readShort(); - minDenseValue = raf.readShort(); + maxCompactValue = getRandomAccessFile().readShort(); + minDenseValue = getRandomAccessFile().readShort(); } if ((flags & 2) != 0) { - estNumEntries = raf.readShort(); - estLengthEntryName = raf.readShort(); + estNumEntries = getRandomAccessFile().readShort(); + estLengthEntryName = getRandomAccessFile().readShort(); } if (debug1) { @@ -1103,19 +1105,19 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageLink start pos= {}", raf.getFilePointer()); + log.debug(" *MessageLink start pos= {}", getRandomAccessFile().getFilePointer()); } - version = raf.readByte(); - flags = raf.readByte(); + version = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); if ((flags & 8) != 0) - linkType = raf.readByte(); + linkType = getRandomAccessFile().readByte(); if ((flags & 4) != 0) - creationOrder = raf.readLong(); + creationOrder = getRandomAccessFile().readLong(); if ((flags & 0x10) != 0) - encoding = raf.readByte(); + encoding = getRandomAccessFile().readByte(); int linkNameLength = (int) readVariableSizeFactor(flags & 3); linkName = readStringFixedLength(linkNameLength); @@ -1124,11 +1126,11 @@ void read() throws IOException { linkAddress = header.readOffset(); } else if (linkType == 1) { - short len = raf.readShort(); + short len = getRandomAccessFile().readShort(); link = readStringFixedLength(len); } else if (linkType == 64) { - short len = raf.readShort(); + short len = getRandomAccessFile().readShort(); link = readStringFixedLength(len); // actually 2 strings - see docs } @@ -1223,15 +1225,15 @@ public String getType() { void read(String objectName) throws IOException { if (debugPos) { - log.debug(" *MessageDatatype start pos= {}", raf.getFilePointer()); + log.debug(" *MessageDatatype start pos= {}", getRandomAccessFile().getFilePointer()); } - byte tandv = raf.readByte(); + byte tandv = getRandomAccessFile().readByte(); type = (tandv & 0xf); version = ((tandv & 0xf0) >> 4); - raf.readFully(flags); - byteSize = raf.readInt(); + getRandomAccessFile().readFully(flags); + byteSize = getRandomAccessFile().readInt(); endian = ((flags[0] & 1) == 0) ? RandomAccessFile.LITTLE_ENDIAN : RandomAccessFile.BIG_ENDIAN; if (debug1) { @@ -1242,8 +1244,8 @@ void read(String objectName) throws IOException { if (type == 0) { // fixed point unsigned = ((flags[0] & 8) == 0); - short bitOffset = raf.readShort(); - short bitPrecision = raf.readShort(); + short bitOffset = getRandomAccessFile().readShort(); + short bitPrecision = getRandomAccessFile().readShort(); if (debug1) { log.debug(" type 0 (fixed point): bitOffset= " + bitOffset + " bitPrecision= " + bitPrecision + " unsigned= " + unsigned); @@ -1251,20 +1253,20 @@ void read(String objectName) throws IOException { isOK = (bitOffset == 0) && (bitPrecision % 8 == 0); } else if (type == 1) { // floating point - short bitOffset = raf.readShort(); - short bitPrecision = raf.readShort(); - byte expLocation = raf.readByte(); - byte expSize = raf.readByte(); - byte manLocation = raf.readByte(); - byte manSize = raf.readByte(); - int expBias = raf.readInt(); + short bitOffset = getRandomAccessFile().readShort(); + short bitPrecision = getRandomAccessFile().readShort(); + byte expLocation = getRandomAccessFile().readByte(); + byte expSize = getRandomAccessFile().readByte(); + byte manLocation = getRandomAccessFile().readByte(); + byte manSize = getRandomAccessFile().readByte(); + int expBias = getRandomAccessFile().readInt(); if (debug1) { log.debug(" type 1 (floating point): bitOffset= " + bitOffset + " bitPrecision= " + bitPrecision + " expLocation= " + expLocation + " expSize= " + expSize + " manLocation= " + manLocation + " manSize= " + manSize + " expBias= " + expBias); } } else if (type == 2) { // time - short bitPrecision = raf.readShort(); + short bitPrecision = getRandomAccessFile().readShort(); if (bitPrecision == 16) timeType = DataType.SHORT; else if (bitPrecision == 32) @@ -1283,8 +1285,8 @@ else if (bitPrecision == 64) } } else if (type == 4) { // bit field - short bitOffset = raf.readShort(); - short bitPrecision = raf.readShort(); + short bitOffset = getRandomAccessFile().readShort(); + short bitPrecision = getRandomAccessFile().readShort(); if (debug1) { log.debug(" type 4 (bit field): bitOffset= " + bitOffset + " bitPrecision= " + bitPrecision); } @@ -1292,7 +1294,7 @@ else if (bitPrecision == 64) } else if (type == 5) { // opaque byte len = flags[0]; - opaque_desc = (len > 0) ? readString(raf).trim() : null; + opaque_desc = (len > 0) ? readString(getRandomAccessFile()).trim() : null; if (debug1) { log.debug(" type 5 (opaque): len= " + len + " desc= " + opaque_desc); } @@ -1331,20 +1333,20 @@ else if (bitPrecision == 64) String[] enumName = new String[nmembers]; for (int i = 0; i < nmembers; i++) { if (version < 3) - enumName[i] = readString8(raf); // padding + enumName[i] = readString8(getRandomAccessFile()); // padding else - enumName[i] = readString(raf); // no padding + enumName[i] = readString(getRandomAccessFile()); // no padding } // read the enum values; must switch to base byte order (!) if (base.endian >= 0) { - raf.order(base.endian); + getRandomAccessFile().order(base.endian); } int[] enumValue = new int[nmembers]; for (int i = 0; i < nmembers; i++) { enumValue[i] = (int) header.readVariableSizeUnsigned(base.byteSize); // assume size is 1, 2, or 4 } - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); enumTypeName = objectName; map = new TreeMap<>(); @@ -1373,14 +1375,14 @@ else if (bitPrecision == 64) if (debug1) { debugOut.print(" type 10(array) lengths= "); } - int ndims = raf.readByte(); + int ndims = getRandomAccessFile().readByte(); if (version < 3) { - raf.skipBytes(3); + getRandomAccessFile().skipBytes(3); } dim = new int[ndims]; for (int i = 0; i < ndims; i++) { - dim[i] = raf.readInt(); + dim[i] = getRandomAccessFile().readInt(); if (debug1) { debugOut.print(" " + dim[i]); } @@ -1389,7 +1391,7 @@ else if (bitPrecision == 64) if (version < 3) { // not present in version 3, never used anyway int[] pdim = new int[ndims]; for (int i = 0; i < ndims; i++) - pdim[i] = raf.readInt(); + pdim[i] = getRandomAccessFile().readInt(); } if (debug1) { log.debug(""); @@ -1432,13 +1434,13 @@ class StructureMember { StructureMember(int version, int byteSize) throws IOException { if (debugPos) { - log.debug(" *StructureMember now at position={}", raf.getFilePointer()); + log.debug(" *StructureMember now at position={}", getRandomAccessFile().getFilePointer()); } - name = readString(raf); + name = readString(getRandomAccessFile()); if (version < 3) { - raf.skipBytes(padding(name.length() + 1, 8)); - offset = raf.readInt(); + getRandomAccessFile().skipBytes(padding(name.length() + 1, 8)); + offset = getRandomAccessFile().readInt(); } else { offset = (int) readVariableSizeMax(byteSize); } @@ -1448,9 +1450,9 @@ class StructureMember { } if (version == 1) { - dims = raf.readByte(); - raf.skipBytes(3); - raf.skipBytes(24); // ignore dimension info for now + dims = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); + getRandomAccessFile().skipBytes(24); // ignore dimension info for now } // HDFdumpWithCount(buffer, raf.getFilePointer(), 16); @@ -1481,9 +1483,9 @@ class MessageFillValueOld implements Named { int size; void read() throws IOException { - size = raf.readInt(); + size = getRandomAccessFile().readInt(); value = new byte[size]; - raf.readFully(value); + getRandomAccessFile().readFully(value); if (debug1) { log.debug("{}", this); @@ -1518,25 +1520,25 @@ class MessageFillValue implements Named { byte flags; void read() throws IOException { - version = raf.readByte(); + version = getRandomAccessFile().readByte(); if (version < 3) { - spaceAllocateTime = raf.readByte(); - fillWriteTime = raf.readByte(); - hasFillValue = raf.readByte() != 0; + spaceAllocateTime = getRandomAccessFile().readByte(); + fillWriteTime = getRandomAccessFile().readByte(); + hasFillValue = getRandomAccessFile().readByte() != 0; } else { - flags = raf.readByte(); + flags = getRandomAccessFile().readByte(); spaceAllocateTime = (byte) (flags & 3); fillWriteTime = (byte) ((flags >> 2) & 3); hasFillValue = (flags & 32) != 0; } if (hasFillValue) { - size = raf.readInt(); + size = getRandomAccessFile().readInt(); if (size > 0) { value = new byte[size]; - raf.readFully(value); + getRandomAccessFile().readFully(value); hasFillValue = true; } else { hasFillValue = false; @@ -1640,41 +1642,41 @@ public String getName() { void read() throws IOException { int ndims; - byte version = raf.readByte(); + byte version = getRandomAccessFile().readByte(); if (version < 3) { - ndims = raf.readByte(); - type = raf.readByte(); - raf.skipBytes(5); // skip 5 bytes + ndims = getRandomAccessFile().readByte(); + type = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(5); // skip 5 bytes boolean isCompact = (type == 0); if (!isCompact) dataAddress = header.readOffset(); chunkSize = new int[ndims]; for (int i = 0; i < ndims; i++) - chunkSize[i] = raf.readInt(); + chunkSize[i] = getRandomAccessFile().readInt(); if (isCompact) { - dataSize = raf.readInt(); - dataAddress = raf.getFilePointer(); + dataSize = getRandomAccessFile().readInt(); + dataAddress = getRandomAccessFile().getFilePointer(); } } else { - type = raf.readByte(); + type = getRandomAccessFile().readByte(); if (type == 0) { - dataSize = raf.readShort(); - dataAddress = raf.getFilePointer(); + dataSize = getRandomAccessFile().readShort(); + dataAddress = getRandomAccessFile().getFilePointer(); } else if (type == 1) { dataAddress = header.readOffset(); contiguousSize = header.readLength(); } else if (type == 2) { - ndims = raf.readByte(); + ndims = getRandomAccessFile().readByte(); dataAddress = header.readOffset(); chunkSize = new int[ndims]; for (int i = 0; i < ndims; i++) - chunkSize[i] = raf.readInt(); + chunkSize[i] = getRandomAccessFile().readInt(); } } @@ -1690,11 +1692,11 @@ class MessageFilter implements Named { void read() throws IOException { - byte version = raf.readByte(); - byte nfilters = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + byte nfilters = getRandomAccessFile().readByte(); // version 1 has 6 bytes reserved if (version == 1) { - raf.skipBytes(6); + getRandomAccessFile().skipBytes(6); } // create filters list @@ -1703,32 +1705,32 @@ void read() throws IOException { // create properties map Map props = new HashMap<>(); // read filter description - short id = raf.readShort(); + short id = getRandomAccessFile().readShort(); props.put(Filters.Keys.ID, id); // if the filter id < 256 then this field is not stored - short nameSize = ((version > 1) && (id < 256)) ? 0 : raf.readShort(); - short flags = raf.readShort(); + short nameSize = ((version > 1) && (id < 256)) ? 0 : getRandomAccessFile().readShort(); + short flags = getRandomAccessFile().readShort(); props.put(Filters.Keys.OPTIONAL, (flags != 0)); // save as boolean - short nValues = raf.readShort(); + short nValues = getRandomAccessFile().readShort(); // get filter name, if present String name = null; if (nameSize > 0) { // version 1 name is padded to multiple of 8 - name = version == 1 ? readString8(raf) : readStringFixedLength(nameSize); + name = version == 1 ? readString8(getRandomAccessFile()) : readStringFixedLength(nameSize); } props.put(Filters.Keys.NAME, name); // read data int[] data = new int[nValues]; for (int n = 0; n < nValues; n++) { - data[n] = raf.readInt(); + data[n] = getRandomAccessFile().readInt(); } props.put(Filters.Keys.DATA, data); // add padding if nValues is odd if ((version == 1) && (nValues & 1) != 0) { - raf.skipBytes(4); + getRandomAccessFile().skipBytes(4); } // add to filters list @@ -1826,51 +1828,52 @@ public String getName() { } boolean read(long pos) throws IOException { - raf.seek(pos); + getRandomAccessFile().seek(pos); if (debugPos) { - log.debug(" *MessageAttribute start pos= {}", raf.getFilePointer()); + log.debug(" *MessageAttribute start pos= {}", getRandomAccessFile().getFilePointer()); } short nameSize, typeSize, spaceSize; byte flags = 0; byte encoding = 0; // 0 = ascii, 1 = UTF-8 - version = raf.readByte(); + version = getRandomAccessFile().readByte(); if (version == 1) { - raf.read(); // skip byte - nameSize = raf.readShort(); - typeSize = raf.readShort(); - spaceSize = raf.readShort(); + getRandomAccessFile().read(); // skip byte + nameSize = getRandomAccessFile().readShort(); + typeSize = getRandomAccessFile().readShort(); + spaceSize = getRandomAccessFile().readShort(); } else if ((version == 2) || (version == 3)) { - flags = raf.readByte(); - nameSize = raf.readShort(); - typeSize = raf.readShort(); - spaceSize = raf.readShort(); + flags = getRandomAccessFile().readByte(); + nameSize = getRandomAccessFile().readShort(); + typeSize = getRandomAccessFile().readShort(); + spaceSize = getRandomAccessFile().readShort(); if (version == 3) - encoding = raf.readByte(); + encoding = getRandomAccessFile().readByte(); } else if (version == 72) { - flags = raf.readByte(); - nameSize = raf.readShort(); - typeSize = raf.readShort(); - spaceSize = raf.readShort(); - log.error("HDF5 MessageAttribute found bad version " + version + " at filePos " + raf.getFilePointer()); + flags = getRandomAccessFile().readByte(); + nameSize = getRandomAccessFile().readShort(); + typeSize = getRandomAccessFile().readShort(); + spaceSize = getRandomAccessFile().readShort(); + log.error("HDF5 MessageAttribute found bad version " + version + " at filePos " + + getRandomAccessFile().getFilePointer()); // G:/work/galibert/IMOS_ANMN-NSW_AETVZ_20131127T230000Z_PH100_FV01_PH100-1311-Workhorse-ADCP-109.5_END-20140306T010000Z_C-20140521T053527Z.nc // E:/work/antonio/2014_ch.nc // return false; } else { - log.error("bad version " + version + " at filePos " + raf.getFilePointer()); // buggery, may be HDF5 "more than - // 8 attributes" error + // buggery, may be HDF5 "more than 8 attributes" error + log.error("bad version " + version + " at filePos " + getRandomAccessFile().getFilePointer()); return false; // throw new IllegalStateException("MessageAttribute unknown version " + version); } // read the attribute name - long filePos = raf.getFilePointer(); - name = readString(raf); // read at current pos + long filePos = getRandomAccessFile().getFilePointer(); + name = readString(getRandomAccessFile()); // read at current pos if (version == 1) nameSize += padding(nameSize, 8); - raf.seek(filePos + nameSize); // make it more robust for errors + getRandomAccessFile().seek(filePos + nameSize); // make it more robust for errors if (debug1) { log.debug(" MessageAttribute version= " + version + " flags = " + Integer.toBinaryString(flags) @@ -1878,7 +1881,7 @@ boolean read(long pos) throws IOException { } // read the datatype - filePos = raf.getFilePointer(); + filePos = getRandomAccessFile().getFilePointer(); if (debugPos) { log.debug(" *MessageAttribute before mdt pos= {}", filePos); } @@ -1893,20 +1896,20 @@ boolean read(long pos) throws IOException { if (version == 1) typeSize += padding(typeSize, 8); } - raf.seek(filePos + typeSize); // make it more robust for errors + getRandomAccessFile().seek(filePos + typeSize); // make it more robust for errors // read the dataspace - filePos = raf.getFilePointer(); + filePos = getRandomAccessFile().getFilePointer(); if (debugPos) { log.debug(" *MessageAttribute before mds = {}", filePos); } mds.read(); if (version == 1) spaceSize += padding(spaceSize, 8); - raf.seek(filePos + spaceSize); // make it more robust for errors + getRandomAccessFile().seek(filePos + spaceSize); // make it more robust for errors // the data starts immediately afterward - ie in the message - dataPos = raf.getFilePointer(); // note this is absolute position (no offset needed) + dataPos = getRandomAccessFile().getFilePointer(); // note this is absolute position (no offset needed) if (debug1) { log.debug(" *MessageAttribute dataPos= {}", dataPos); } @@ -1991,12 +1994,12 @@ String trunc(String s, int max) { void read() throws IOException { if (debugPos) { - log.debug(" *MessageAttributeInfo start pos= {}", raf.getFilePointer()); + log.debug(" *MessageAttributeInfo start pos= {}", getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); - byte flags = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + byte flags = getRandomAccessFile().readByte(); if ((flags & 1) != 0) - maxCreationIndex = raf.readShort(); + maxCreationIndex = getRandomAccessFile().readShort(); fractalHeapAddress = header.readOffset(); v2BtreeAddress = header.readOffset(); @@ -2015,7 +2018,7 @@ class MessageComment implements Named { String comment; void read() throws IOException { - comment = readString(raf); + comment = readString(getRandomAccessFile()); } public String toString() { @@ -2034,9 +2037,9 @@ class MessageLastModified implements Named { int secs; void read() throws IOException { - version = raf.readByte(); - raf.skipBytes(3); // skip byte - secs = raf.readInt(); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); // skip byte + secs = getRandomAccessFile().readInt(); } public String toString() { @@ -2054,7 +2057,7 @@ class MessageLastModifiedOld implements Named { String datemod; void read() throws IOException { - datemod = raf.readString(14); + datemod = getRandomAccessFile().readString(14); if (debug1) { log.debug(" MessageLastModifiedOld={}", datemod); } @@ -2091,8 +2094,8 @@ class MessageObjectReferenceCount implements Named { int refCount; void read() throws IOException { - int version = raf.readByte(); - refCount = raf.readInt(); + int version = getRandomAccessFile().readByte(); + refCount = getRandomAccessFile().readInt(); if (debug1) { log.debug(" ObjectReferenceCount={}", refCount); } @@ -2141,7 +2144,7 @@ private void readGroupNew(H5Group group, MessageGroupNew groupNewMessage, DataOb long pos = fractalHeapId.getPos(); if (pos < 0) continue; - raf.seek(pos); + getRandomAccessFile().seek(pos); MessageLink linkMessage = new MessageLink(); linkMessage.read(); if (debugBtree2) { @@ -2227,18 +2230,18 @@ List getSymbolTableEntries() { // recursively read all entries, place them in order in list void readAllEntries(long address, List entryList) throws IOException { - raf.seek(header.getFileOffset(address)); + getRandomAccessFile().seek(header.getFileOffset(address)); if (debugGroupBtree) { - log.debug("\n--> GroupBTree read tree at position={}", raf.getFilePointer()); + log.debug("\n--> GroupBTree read tree at position={}", getRandomAccessFile().getFilePointer()); } - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("TREE")) throw new IllegalStateException("BtreeGroup doesnt start with TREE"); - int type = raf.readByte(); - int level = raf.readByte(); - int nentries = raf.readShort(); + int type = getRandomAccessFile().readByte(); + int level = getRandomAccessFile().readByte(); + int nentries = getRandomAccessFile().readShort(); if (debugGroupBtree) { log.debug(" type=" + type + " level=" + level + " nentries=" + nentries); } @@ -2299,25 +2302,25 @@ class GroupNode { GroupNode(long address) throws IOException { this.address = address; - raf.seek(header.getFileOffset(address)); + getRandomAccessFile().seek(header.getFileOffset(address)); if (debugDetail) { - log.debug("--Group Node position={}", raf.getFilePointer()); + log.debug("--Group Node position={}", getRandomAccessFile().getFilePointer()); } // header - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("SNOD")) { throw new IllegalStateException(magic + " should equal SNOD"); } - version = raf.readByte(); - raf.readByte(); // skip byte - nentries = raf.readShort(); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().readByte(); // skip byte + nentries = getRandomAccessFile().readShort(); if (debugDetail) { log.debug(" version={} nentries={}", version, nentries); } - long posEntry = raf.getFilePointer(); + long posEntry = getRandomAccessFile().getFilePointer(); for (int i = 0; i < nentries; i++) { SymbolTableEntry entry = new SymbolTableEntry(posEntry); posEntry += entry.getSize(); @@ -2333,7 +2336,7 @@ class GroupNode { } } if (debugDetail) { - log.debug("-- Group Node end position={}", raf.getFilePointer()); + log.debug("-- Group Node end position={}", getRandomAccessFile().getFilePointer()); } long size = 8 + nentries * 40; if (debugTracker) @@ -2358,22 +2361,22 @@ class SymbolTableEntry { boolean isSymbolicLink; SymbolTableEntry(long filePos) throws IOException { - raf.seek(filePos); + getRandomAccessFile().seek(filePos); if (debugSymbolTable) { - log.debug("--> readSymbolTableEntry position={}", raf.getFilePointer()); + log.debug("--> readSymbolTableEntry position={}", getRandomAccessFile().getFilePointer()); } nameOffset = header.readOffset(); objectHeaderAddress = header.readOffset(); - cacheType = raf.readInt(); - raf.skipBytes(4); + cacheType = getRandomAccessFile().readInt(); + getRandomAccessFile().skipBytes(4); if (debugSymbolTable) { log.debug(" nameOffset={} objectHeaderAddress={} cacheType={}", nameOffset, objectHeaderAddress, cacheType); } // "scratch pad" - posData = raf.getFilePointer(); + posData = getRandomAccessFile().getFilePointer(); if (debugSymbolTable) dump("Group Entry scratch pad", posData, 16, false); @@ -2387,7 +2390,7 @@ class SymbolTableEntry { // check for symbolic link if (cacheType == 2) { - linkOffset = raf.readInt(); // offset in local heap + linkOffset = getRandomAccessFile().readInt(); // offset in local heap if (debugSymbolTable) { log.debug("WARNING Symbolic Link linkOffset={}", linkOffset); } @@ -2395,7 +2398,7 @@ class SymbolTableEntry { } if (debugSymbolTable) { - log.debug("<-- end readSymbolTableEntry position={}", raf.getFilePointer()); + log.debug("<-- end readSymbolTableEntry position={}", getRandomAccessFile().getFilePointer()); } if (debugTracker) @@ -2455,43 +2458,43 @@ private Array getHeapDataArray(HeapIdentifier heapId, DataType dataType, int end log.debug(" HeapObject= {}", ho); } if (endian >= 0) { - raf.order(endian); + getRandomAccessFile().order(endian); } if (DataType.FLOAT == dataType) { float[] pa = new float[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readFloat(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readFloat(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (DataType.DOUBLE == dataType) { double[] pa = new double[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readDouble(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readDouble(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == byte.class) { byte[] pa = new byte[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readFully(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readFully(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == short.class) { short[] pa = new short[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readShort(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readShort(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == int.class) { int[] pa = new int[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readInt(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readInt(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == long.class) { long[] pa = new long[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readLong(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readLong(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } @@ -2517,11 +2520,11 @@ class HeapIdentifier { // address must be absolute, getFileOffset already added HeapIdentifier(long address) throws IOException { // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(address); - nelems = raf.readInt(); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(address); + nelems = getRandomAccessFile().readInt(); heapAddress = header.readOffset(); - index = raf.readInt(); + index = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" read HeapIdentifier address=" + address + this); } @@ -2572,10 +2575,10 @@ class RegionReference { RegionReference(long filePos) throws IOException { // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(filePos); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(filePos); heapAddress = header.readOffset(); - index = raf.readInt(); + index = getRandomAccessFile().readInt(); GlobalHeap gheap; if (null == (gheap = heapMap.get(heapAddress))) { @@ -2595,8 +2598,8 @@ class RegionReference { * src/H5S.c, where foo = {all, hyper, point, none}. * There is _no_ datatype information stored for these kind of selections currently. */ - raf.seek(want.dataPos); - long objId = raf.readLong(); + getRandomAccessFile().seek(want.dataPos); + long objId = getRandomAccessFile().readLong(); DataObject ndo = header.getDataObject(objId, null); // String what = (ndo == null) ? "none" : ndo.getName(); if (debugRegionReference) { @@ -2616,35 +2619,35 @@ class GlobalHeap { GlobalHeap(long address) throws IOException { // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(header.getFileOffset(address)); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(header.getFileOffset(address)); // header - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("GCOL")) throw new IllegalStateException(magic + " should equal GCOL"); - version = raf.readByte(); - raf.skipBytes(3); - sizeBytes = raf.readInt(); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); + sizeBytes = getRandomAccessFile().readInt(); if (debugDetail) { log.debug("-- readGlobalHeap address=" + address + " version= " + version + " size = " + sizeBytes); // log.debug("-- readGlobalHeap address=" + address + " version= " + version + " size = " + sizeBytes); } - raf.skipBytes(4); // pad to 8 + getRandomAccessFile().skipBytes(4); // pad to 8 int count = 0; int countBytes = 0; while (true) { - long startPos = raf.getFilePointer(); + long startPos = getRandomAccessFile().getFilePointer(); GlobalHeap.HeapObject o = new GlobalHeap.HeapObject(); - o.id = raf.readShort(); + o.id = getRandomAccessFile().readShort(); if (o.id == 0) break; // ?? look - o.refCount = raf.readShort(); - raf.skipBytes(4); + o.refCount = getRandomAccessFile().readShort(); + getRandomAccessFile().skipBytes(4); o.dataSize = header.readLength(); - o.dataPos = raf.getFilePointer(); + o.dataPos = getRandomAccessFile().getFilePointer(); int dsize = ((int) o.dataSize) + padding((int) o.dataSize, 8); countBytes += dsize + 16; @@ -2661,7 +2664,7 @@ class GlobalHeap { + o.dataSize + " dataPos = " + o.dataPos + " count= " + count + " countBytes= " + countBytes); } - raf.skipBytes(dsize); + getRandomAccessFile().skipBytes(dsize); hos.put(o.id, o); count++; @@ -2670,7 +2673,7 @@ class GlobalHeap { } if (debugDetail) { - log.debug("-- endGlobalHeap position=" + raf.getFilePointer()); + log.debug("-- endGlobalHeap position=" + getRandomAccessFile().getFilePointer()); } if (debugTracker) memTracker.addByLen("GlobalHeap", address, sizeBytes); @@ -2705,21 +2708,21 @@ class LocalHeap { this.group = group; // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(header.getFileOffset(address)); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(header.getFileOffset(address)); if (debugDetail) { - log.debug("-- readLocalHeap position={}", raf.getFilePointer()); + log.debug("-- readLocalHeap position={}", getRandomAccessFile().getFilePointer()); } // header - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("HEAP")) { throw new IllegalStateException(magic + " should equal HEAP"); } - version = raf.readByte(); - raf.skipBytes(3); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); size = (int) header.readLength(); freelistOffset = header.readLength(); dataAddress = header.readOffset(); @@ -2728,17 +2731,17 @@ class LocalHeap { + " heap starts at dataAddress=" + dataAddress); } if (debugPos) { - log.debug(" *now at position={}", raf.getFilePointer()); + log.debug(" *now at position={}", getRandomAccessFile().getFilePointer()); } // data - raf.seek(header.getFileOffset(dataAddress)); + getRandomAccessFile().seek(header.getFileOffset(dataAddress)); heap = new byte[size]; - raf.readFully(heap); + getRandomAccessFile().readFully(heap); // if (debugHeap) printBytes( out, "heap", heap, size, true); if (debugDetail) { - log.debug("-- endLocalHeap position={}", raf.getFilePointer()); + log.debug("-- endLocalHeap position={}", getRandomAccessFile().getFilePointer()); } int hsize = 8 + 2 * header.sizeLengths + header.sizeOffsets; if (debugTracker) @@ -2812,7 +2815,7 @@ private String readString8(RandomAccessFile raf) throws IOException { * @throws IOException on io error */ private String readStringFixedLength(int size) throws IOException { - return raf.readString(size); + return getRandomAccessFile().readString(size); } // size of data depends on "maximum possible number" @@ -2841,13 +2844,13 @@ private int makeUnsignedIntFromBytes(byte upper, byte lower) { private void dump(String head, long filePos, int nbytes, boolean count) throws IOException { if (debugOut == null) return; - long savePos = raf.getFilePointer(); + long savePos = getRandomAccessFile().getFilePointer(); if (filePos >= 0) - raf.seek(filePos); + getRandomAccessFile().seek(filePos); byte[] mess = new byte[nbytes]; - raf.readFully(mess); + getRandomAccessFile().readFully(mess); printBytes(head, mess, nbytes, false, debugOut); - raf.seek(savePos); + getRandomAccessFile().seek(savePos); } private static void printBytes(String head, byte[] buff, int n, boolean count, PrintWriter ps) { diff --git a/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/DataBTree.java b/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/DataBTree.java index 9293d81bf2..d23ae99bbb 100644 --- a/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/DataBTree.java +++ b/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/DataBTree.java @@ -38,7 +38,6 @@ public class DataBTree { private static java.io.PrintStream debugOut = System.out; private final H5headerIF h5; - private final RandomAccessFile raf; private final MemTracker memTracker; private final long rootNodeAddress; @@ -49,7 +48,6 @@ public class DataBTree { public DataBTree(H5headerIF h5, long rootNodeAddress, int[] varShape, int[] storageSize, MemTracker memTracker) { this.h5 = h5; - this.raf = h5.getRandomAccessFile(); this.rootNodeAddress = rootNodeAddress; this.tiling = new Tiling(varShape, storageSize); this.ndimStorage = storageSize.length; @@ -58,6 +56,10 @@ public DataBTree(H5headerIF h5, long rootNodeAddress, int[] varShape, int[] stor wantType = 1; } + RandomAccessFile getRandomAccessFile() { + return h5.getRandomAccessFile(); + } + public void setOwner(Object owner) { this.owner = owner; } @@ -159,17 +161,17 @@ class Node { if (debugDataBtree) debugOut.println("\n--> DataBTree read tree at address=" + address + " parent= " + parent + " owner= " + owner); - raf.order(RandomAccessFile.LITTLE_ENDIAN); // header information is in le byte order - raf.seek(h5.getFileOffset(address)); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); // header information is in le byte order + getRandomAccessFile().seek(h5.getFileOffset(address)); this.address = address; - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("TREE")) throw new IllegalStateException("DataBTree doesnt start with TREE"); - int type = raf.readByte(); - level = raf.readByte(); - nentries = raf.readShort(); + int type = getRandomAccessFile().readByte(); + level = getRandomAccessFile().readByte(); + nentries = getRandomAccessFile().readShort(); if (type != wantType) throw new IllegalStateException("DataBTree must be type " + wantType); @@ -198,9 +200,9 @@ class Node { offset = new int[nentries + 1][ndimStorage]; childPointer = new long[nentries + 1]; for (int i = 0; i <= nentries; i++) { - raf.skipBytes(8); // skip size, filterMask + getRandomAccessFile().skipBytes(8); // skip size, filterMask for (int j = 0; j < ndimStorage; j++) { - long loffset = raf.readLong(); + long loffset = getRandomAccessFile().readLong(); assert loffset < Integer.MAX_VALUE; offset[i][j] = (int) loffset; } @@ -301,11 +303,11 @@ public class DataChunk { public final long filePos; // filePos of a single raw data chunk, already shifted by the offset if needed DataChunk(int ndim, boolean last) throws IOException { - this.size = raf.readInt(); - this.filterMask = raf.readInt(); + this.size = getRandomAccessFile().readInt(); + this.filterMask = getRandomAccessFile().readInt(); offset = new int[ndim]; for (int i = 0; i < ndim; i++) { - long loffset = raf.readLong(); + long loffset = getRandomAccessFile().readLong(); assert loffset < Integer.MAX_VALUE; offset[i] = (int) loffset; } diff --git a/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5header.java b/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5header.java index 6dea73a27e..3db0bd88c1 100644 --- a/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5header.java +++ b/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5header.java @@ -96,7 +96,6 @@ public static boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws I //////////////////////////////////////////////////////////////////////////////// - RandomAccessFile raf; ucar.nc2.NetcdfFile ncfile; private H5iosp h5iosp; @@ -121,9 +120,8 @@ public static boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) throws I private java.io.PrintWriter debugOut; private MemTracker memTracker; - H5header(RandomAccessFile myRaf, ucar.nc2.NetcdfFile ncfile, H5iosp h5iosp) { + H5header(ucar.nc2.NetcdfFile ncfile, H5iosp h5iosp) { this.ncfile = ncfile; - this.raf = myRaf; this.h5iosp = h5iosp; } @@ -154,7 +152,7 @@ public void read(java.io.PrintWriter debugPS) throws IOException { debugOut = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)); } - long actualSize = raf.length(); + long actualSize = getRandomAccessFile().length(); if (debugTracker) memTracker = new MemTracker(actualSize); @@ -163,8 +161,8 @@ public void read(java.io.PrintWriter debugPS) throws IOException { boolean ok = false; long filePos = 0; while ((filePos < actualSize - 8)) { - raf.seek(filePos); - String magic = raf.readString(8); + getRandomAccessFile().seek(filePos); + String magic = getRandomAccessFile().readString(8); if (magic.equals(hdf5magic)) { ok = true; break; @@ -175,19 +173,19 @@ public void read(java.io.PrintWriter debugPS) throws IOException { throw new IOException("Not a netCDF4/HDF5 file "); } if (debug1) { - log.debug("H5header opened file to read:'{}' size= {}", raf.getLocation(), actualSize); + log.debug("H5header opened file to read:'{}' size= {}", getRandomAccessFile().getLocation(), actualSize); } // now we are positioned right after the header // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); - long superblockStart = raf.getFilePointer() - 8; + long superblockStart = getRandomAccessFile().getFilePointer() - 8; if (debugTracker) memTracker.add("header", 0, superblockStart); // superblock version - byte versionSB = raf.readByte(); + byte versionSB = getRandomAccessFile().readByte(); if (versionSB < 2) { readSuperBlock1(superblockStart, versionSB); @@ -223,43 +221,43 @@ private void readSuperBlock1(long superblockStart, byte versionSB) throws IOExce long eofAddress; long driverBlockAddress; - versionFSS = raf.readByte(); - versionGroup = raf.readByte(); - raf.readByte(); // skip 1 byte - versionSHMF = raf.readByte(); + versionFSS = getRandomAccessFile().readByte(); + versionGroup = getRandomAccessFile().readByte(); + getRandomAccessFile().readByte(); // skip 1 byte + versionSHMF = getRandomAccessFile().readByte(); if (debugDetail) { log.debug(" versionSB= " + versionSB + " versionFSS= " + versionFSS + " versionGroup= " + versionGroup + " versionSHMF= " + versionSHMF); } - sizeOffsets = raf.readByte(); + sizeOffsets = getRandomAccessFile().readByte(); isOffsetLong = (sizeOffsets == 8); - sizeLengths = raf.readByte(); + sizeLengths = getRandomAccessFile().readByte(); isLengthLong = (sizeLengths == 8); if (debugDetail) { log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths); log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong); } - raf.read(); // skip 1 byte + getRandomAccessFile().read(); // skip 1 byte // log.debug(" position="+mapBuffer.position()); - btreeLeafNodeSize = raf.readShort(); - btreeInternalNodeSize = raf.readShort(); + btreeLeafNodeSize = getRandomAccessFile().readShort(); + btreeInternalNodeSize = getRandomAccessFile().readShort(); if (debugDetail) { log.debug(" btreeLeafNodeSize= {} btreeInternalNodeSize= {}", btreeLeafNodeSize, btreeInternalNodeSize); } // log.debug(" position="+mapBuffer.position()); - fileFlags = raf.readInt(); + fileFlags = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags)); } if (versionSB == 1) { - short storageInternalNodeSize = raf.readShort(); - raf.skipBytes(2); + short storageInternalNodeSize = getRandomAccessFile().readShort(); + getRandomAccessFile().skipBytes(2); } baseAddress = readOffset(); @@ -279,21 +277,21 @@ private void readSuperBlock1(long superblockStart, byte versionSB) throws IOExce log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress)); log.debug(" global free space heap Address= 0x{}", Long.toHexString(heapAddress)); log.debug(" eof Address={}", eofAddress); - log.debug(" raf length= {}", raf.length()); + log.debug(" raf length= {}", getRandomAccessFile().length()); log.debug(" driver BlockAddress= 0x{}", Long.toHexString(driverBlockAddress)); log.debug(""); } if (debugTracker) - memTracker.add("superblock", superblockStart, raf.getFilePointer()); + memTracker.add("superblock", superblockStart, getRandomAccessFile().getFilePointer()); // look for file truncation - long fileSize = raf.length(); + long fileSize = getRandomAccessFile().length(); if (fileSize < eofAddress) - throw new IOException( - "File is truncated should be= " + eofAddress + " actual = " + fileSize + "%nlocation= " + raf.getLocation()); + throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize + "%nlocation= " + + getRandomAccessFile().getLocation()); // next comes the root objext's SymbolTableEntry - SymbolTableEntry rootEntry = new SymbolTableEntry(raf.getFilePointer()); + SymbolTableEntry rootEntry = new SymbolTableEntry(getRandomAccessFile().getFilePointer()); // extract the root group object, recursively read all objects long rootObjectAddress = rootEntry.getObjectAddress(); @@ -314,17 +312,17 @@ private void readSuperBlock1(long superblockStart, byte versionSB) throws IOExce } private void readSuperBlock2(long superblockStart) throws IOException { - sizeOffsets = raf.readByte(); + sizeOffsets = getRandomAccessFile().readByte(); isOffsetLong = (sizeOffsets == 8); - sizeLengths = raf.readByte(); + sizeLengths = getRandomAccessFile().readByte(); isLengthLong = (sizeLengths == 8); if (debugDetail) { log.debug(" sizeOffsets= {} sizeLengths= {}", sizeOffsets, sizeLengths); log.debug(" isLengthLong= {} isOffsetLong= {}", isLengthLong, isOffsetLong); } - byte fileFlags = raf.readByte(); + byte fileFlags = getRandomAccessFile().readByte(); if (debugDetail) { log.debug(" fileFlags= 0x{}", Integer.toHexString(fileFlags)); } @@ -333,7 +331,7 @@ private void readSuperBlock2(long superblockStart) throws IOException { long extensionAddress = readOffset(); long eofAddress = readOffset(); long rootObjectAddress = readOffset(); - int checksum = raf.readInt(); + int checksum = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" baseAddress= 0x{}", Long.toHexString(baseAddress)); @@ -344,7 +342,7 @@ private void readSuperBlock2(long superblockStart) throws IOException { } if (debugTracker) - memTracker.add("superblock", superblockStart, raf.getFilePointer()); + memTracker.add("superblock", superblockStart, getRandomAccessFile().getFilePointer()); if (baseAddress != superblockStart) { baseAddress = superblockStart; @@ -355,7 +353,7 @@ private void readSuperBlock2(long superblockStart) throws IOException { } // look for file truncation - long fileSize = raf.length(); + long fileSize = getRandomAccessFile().length(); if (fileSize < eofAddress) { throw new IOException("File is truncated should be= " + eofAddress + " actual = " + fileSize); } @@ -993,7 +991,7 @@ private void makeAttributes(Structure s, MessageAttribute matt, AttributeContain // reading attribute values might change byte order during a read // put back to little endian for further header processing - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); } private Attribute makeAttribute(MessageAttribute matt) throws IOException { @@ -1013,7 +1011,7 @@ private Attribute makeAttribute(MessageAttribute matt) throws IOException { attData = readAttributeData(matt, vinfo, dtype); } catch (InvalidRangeException e) { - log.warn("failed to read Attribute " + matt.name + " HDF5 file=" + raf.getLocation()); + log.warn("failed to read Attribute " + matt.name + " HDF5 file=" + getRandomAccessFile().getLocation()); return null; } @@ -1031,7 +1029,7 @@ private Attribute makeAttribute(MessageAttribute matt) throws IOException { result = new Attribute(matt.name, attData); } - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); return result; } @@ -1093,8 +1091,8 @@ private Array readAttributeData(H5header.MessageAttribute matt, H5header.Vinfo v } // copy bytes directly into the underlying byte[] - raf.seek(chunk.getSrcPos()); - raf.readFully(byteArray, (int) chunk.getDestElem() * recsize, chunk.getNelems() * recsize); + getRandomAccessFile().seek(chunk.getSrcPos()); + getRandomAccessFile().readFully(byteArray, (int) chunk.getDestElem() * recsize, chunk.getNelems() * recsize); } // strings are stored on the heap, and must be read separately @@ -2458,63 +2456,63 @@ private DataObject(long address, String who) throws IOException { log.debug("\n--> DataObject.read parsing <" + who + "> object ID/address=" + address); } if (debugPos) { - log.debug(" DataObject.read now at position=" + raf.getFilePointer() + " for <" + who + "> reposition to " - + getFileOffset(address)); + log.debug(" DataObject.read now at position=" + getRandomAccessFile().getFilePointer() + " for <" + who + + "> reposition to " + getFileOffset(address)); } // if (offset < 0) return null; - raf.seek(getFileOffset(address)); + getRandomAccessFile().seek(getFileOffset(address)); - version = raf.readByte(); + version = getRandomAccessFile().readByte(); if (version == 1) { // Level 2A1 (first part, before the messages) - raf.readByte(); // skip byte - short nmess = raf.readShort(); + getRandomAccessFile().readByte(); // skip byte + short nmess = getRandomAccessFile().readShort(); if (debugDetail) { log.debug(" version=" + version + " nmess=" + nmess); } - int referenceCount = raf.readInt(); - int headerSize = raf.readInt(); + int referenceCount = getRandomAccessFile().readInt(); + int headerSize = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" referenceCount=" + referenceCount + " headerSize=" + headerSize); } // if (referenceCount > 1) // log.debug("WARNING referenceCount="+referenceCount); - raf.skipBytes(4); // header messages multiples of 8 + getRandomAccessFile().skipBytes(4); // header messages multiples of 8 - long posMess = raf.getFilePointer(); + long posMess = getRandomAccessFile().getFilePointer(); int count = readMessagesVersion1(posMess, nmess, Integer.MAX_VALUE, this.who); if (debugContinueMessage) { log.debug(" nmessages read = {}", count); } if (debugPos) { - log.debug("<--done reading messages for <" + who + ">; position=" + raf.getFilePointer()); + log.debug("<--done reading messages for <" + who + ">; position=" + getRandomAccessFile().getFilePointer()); } if (debugTracker) memTracker.addByLen("Object " + who, getFileOffset(address), headerSize + 16); } else { // level 2A2 (first part, before the messages) // first byte was already read - String magic = raf.readString(3); + String magic = getRandomAccessFile().readString(3); if (!magic.equals("HDR")) throw new IllegalStateException("DataObject doesnt start with OHDR"); - version = raf.readByte(); - byte flags = raf.readByte(); // data object header flags (version 2) + version = getRandomAccessFile().readByte(); + byte flags = getRandomAccessFile().readByte(); // data object header flags (version 2) if (debugDetail) { log.debug(" version=" + version + " flags=" + Integer.toBinaryString(flags)); } // raf.skipBytes(2); if (((flags >> 5) & 1) == 1) { - int accessTime = raf.readInt(); - int modTime = raf.readInt(); - int changeTime = raf.readInt(); - int birthTime = raf.readInt(); + int accessTime = getRandomAccessFile().readInt(); + int modTime = getRandomAccessFile().readInt(); + int changeTime = getRandomAccessFile().readInt(); + int birthTime = getRandomAccessFile().readInt(); } if (((flags >> 4) & 1) == 1) { - short maxCompactAttributes = raf.readShort(); - short minDenseAttributes = raf.readShort(); + short maxCompactAttributes = getRandomAccessFile().readShort(); + short minDenseAttributes = getRandomAccessFile().readShort(); } long sizeOfChunk = readVariableSizeFactor(flags & 3); @@ -2522,13 +2520,13 @@ private DataObject(long address, String who) throws IOException { log.debug(" sizeOfChunk=" + sizeOfChunk); } - long posMess = raf.getFilePointer(); + long posMess = getRandomAccessFile().getFilePointer(); int count = readMessagesVersion2(posMess, sizeOfChunk, (flags & 4) != 0, this.who); if (debugContinueMessage) { log.debug(" nmessages read = {}", count); } if (debugPos) { - log.debug("<--done reading messages for <" + who + ">; position=" + raf.getFilePointer()); + log.debug("<--done reading messages for <" + who + ">; position=" + getRandomAccessFile().getFilePointer()); } } @@ -2670,7 +2668,7 @@ private int readMessagesVersion2(long filePos, long maxBytes, boolean creationOr if (debugContinueMessage) debugOut.println(" ---ObjectHeaderContinuation filePos= " + continuationBlockFilePos); - raf.seek(continuationBlockFilePos); + getRandomAccessFile().seek(continuationBlockFilePos); String sig = readStringFixedLength(4); if (!sig.equals("OCHK")) throw new IllegalStateException(" ObjectHeaderContinuation Missing signature"); @@ -2831,28 +2829,28 @@ public long getStart() { */ int read(long filePos, int version, boolean creationOrderPresent, String objectName) throws IOException { this.start = filePos; - raf.seek(filePos); + getRandomAccessFile().seek(filePos); if (debugPos) { - log.debug(" --> Message Header starts at =" + raf.getFilePointer()); + log.debug(" --> Message Header starts at =" + getRandomAccessFile().getFilePointer()); } if (version == 1) { - type = raf.readShort(); - size = DataType.unsignedShortToInt(raf.readShort()); - headerMessageFlags = raf.readByte(); - raf.skipBytes(3); + type = getRandomAccessFile().readShort(); + size = DataType.unsignedShortToInt(getRandomAccessFile().readShort()); + headerMessageFlags = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); header_length = 8; } else { - type = (short) raf.readByte(); - size = DataType.unsignedShortToInt(raf.readShort()); + type = (short) getRandomAccessFile().readByte(); + size = DataType.unsignedShortToInt(getRandomAccessFile().readShort()); // if (size > Short.MAX_VALUE) // log.debug("HEY"); - headerMessageFlags = raf.readByte(); + headerMessageFlags = getRandomAccessFile().readByte(); header_length = 4; if (creationOrderPresent) { - creationOrder = raf.readShort(); + creationOrder = getRandomAccessFile().readShort(); header_length += 2; } } @@ -2864,7 +2862,7 @@ int read(long filePos, int version, boolean creationOrderPresent, String objectN } } if (debugPos) { - log.debug(" --> Message Data starts at=" + raf.getFilePointer()); + log.debug(" --> Message Data starts at=" + getRandomAccessFile().getFilePointer()); } if ((headerMessageFlags & 2) != 0) { // shared @@ -2922,7 +2920,7 @@ int read(long filePos, int version, boolean creationOrderPresent, String objectN } else if (mtype == MessageType.Attribute) { // 12 MessageAttribute data = new MessageAttribute(); - data.read(raf.getFilePointer()); + data.read(getRandomAccessFile().getFilePointer()); messData = data; } else if (mtype == MessageType.Comment) { // 13 @@ -3002,17 +3000,17 @@ public void showCompression(Formatter f) { } private DataObject getSharedDataObject(MessageType mtype) throws IOException { - byte sharedVersion = raf.readByte(); - byte sharedType = raf.readByte(); + byte sharedVersion = getRandomAccessFile().readByte(); + byte sharedType = getRandomAccessFile().readByte(); if (sharedVersion == 1) - raf.skipBytes(6); + getRandomAccessFile().skipBytes(6); if ((sharedVersion == 3) && (sharedType == 1)) { - long heapId = raf.readLong(); + long heapId = getRandomAccessFile().readLong(); if (debug1) { log.debug(" Shared Message " + sharedVersion + " type=" + sharedType + " heapId = " + heapId); } if (debugPos) { - log.debug(" --> Shared Message reposition to =" + raf.getFilePointer()); + log.debug(" --> Shared Message reposition to =" + getRandomAccessFile().getFilePointer()); } // dunno where is the file's shared object header heap ?? throw new UnsupportedOperationException("****SHARED MESSAGE type = " + mtype + " heapId = " + heapId); @@ -3074,20 +3072,20 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageSimpleDataspace start pos= " + raf.getFilePointer()); + log.debug(" *MessageSimpleDataspace start pos= " + getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); + byte version = getRandomAccessFile().readByte(); if (version == 1) { - ndims = raf.readByte(); - flags = raf.readByte(); + ndims = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); type = (byte) ((ndims == 0) ? 0 : 1); - raf.skipBytes(5); // skip 5 bytes + getRandomAccessFile().skipBytes(5); // skip 5 bytes } else if (version == 2) { - ndims = raf.readByte(); - flags = raf.readByte(); - type = raf.readByte(); + ndims = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); + type = getRandomAccessFile().readByte(); } else { throw new IllegalStateException("MessageDataspace: unknown version= " + version); @@ -3177,12 +3175,12 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageGroupNew start pos= " + raf.getFilePointer()); + log.debug(" *MessageGroupNew start pos= " + getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); - byte flags = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + byte flags = getRandomAccessFile().readByte(); if ((flags & 1) != 0) { - maxCreationIndex = raf.readLong(); + maxCreationIndex = getRandomAccessFile().readLong(); } fractalHeapAddress = readOffset(); @@ -3220,19 +3218,19 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageGroupInfo start pos= " + raf.getFilePointer()); + log.debug(" *MessageGroupInfo start pos= " + getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); - flags = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); if ((flags & 1) != 0) { - maxCompactValue = raf.readShort(); - minDenseValue = raf.readShort(); + maxCompactValue = getRandomAccessFile().readShort(); + minDenseValue = getRandomAccessFile().readShort(); } if ((flags & 2) != 0) { - estNumEntries = raf.readShort(); - estLengthEntryName = raf.readShort(); + estNumEntries = getRandomAccessFile().readShort(); + estLengthEntryName = getRandomAccessFile().readShort(); } if (debug1) { @@ -3271,19 +3269,19 @@ public String toString() { void read() throws IOException { if (debugPos) { - log.debug(" *MessageLink start pos= {}", raf.getFilePointer()); + log.debug(" *MessageLink start pos= {}", getRandomAccessFile().getFilePointer()); } - version = raf.readByte(); - flags = raf.readByte(); + version = getRandomAccessFile().readByte(); + flags = getRandomAccessFile().readByte(); if ((flags & 8) != 0) - linkType = raf.readByte(); + linkType = getRandomAccessFile().readByte(); if ((flags & 4) != 0) - creationOrder = raf.readLong(); + creationOrder = getRandomAccessFile().readLong(); if ((flags & 0x10) != 0) - encoding = raf.readByte(); + encoding = getRandomAccessFile().readByte(); int linkNameLength = (int) readVariableSizeFactor(flags & 3); linkName = readStringFixedLength(linkNameLength); @@ -3292,11 +3290,11 @@ void read() throws IOException { linkAddress = readOffset(); } else if (linkType == 1) { - short len = raf.readShort(); + short len = getRandomAccessFile().readShort(); link = readStringFixedLength(len); } else if (linkType == 64) { - short len = raf.readShort(); + short len = getRandomAccessFile().readShort(); link = readStringFixedLength(len); // actually 2 strings - see docs } @@ -3389,15 +3387,15 @@ public String getType() { void read(String objectName) throws IOException { if (debugPos) { - log.debug(" *MessageDatatype start pos= {}", raf.getFilePointer()); + log.debug(" *MessageDatatype start pos= {}", getRandomAccessFile().getFilePointer()); } - byte tandv = raf.readByte(); + byte tandv = getRandomAccessFile().readByte(); type = (tandv & 0xf); version = ((tandv & 0xf0) >> 4); - raf.readFully(flags); - byteSize = raf.readInt(); + getRandomAccessFile().readFully(flags); + byteSize = getRandomAccessFile().readInt(); endian = ((flags[0] & 1) == 0) ? RandomAccessFile.LITTLE_ENDIAN : RandomAccessFile.BIG_ENDIAN; if (debug1) { @@ -3408,8 +3406,8 @@ void read(String objectName) throws IOException { if (type == 0) { // fixed point unsigned = ((flags[0] & 8) == 0); - short bitOffset = raf.readShort(); - short bitPrecision = raf.readShort(); + short bitOffset = getRandomAccessFile().readShort(); + short bitPrecision = getRandomAccessFile().readShort(); if (debug1) { log.debug(" type 0 (fixed point): bitOffset= " + bitOffset + " bitPrecision= " + bitPrecision + " unsigned= " + unsigned); @@ -3417,20 +3415,20 @@ void read(String objectName) throws IOException { isOK = (bitOffset == 0) && (bitPrecision % 8 == 0); } else if (type == 1) { // floating point - short bitOffset = raf.readShort(); - short bitPrecision = raf.readShort(); - byte expLocation = raf.readByte(); - byte expSize = raf.readByte(); - byte manLocation = raf.readByte(); - byte manSize = raf.readByte(); - int expBias = raf.readInt(); + short bitOffset = getRandomAccessFile().readShort(); + short bitPrecision = getRandomAccessFile().readShort(); + byte expLocation = getRandomAccessFile().readByte(); + byte expSize = getRandomAccessFile().readByte(); + byte manLocation = getRandomAccessFile().readByte(); + byte manSize = getRandomAccessFile().readByte(); + int expBias = getRandomAccessFile().readInt(); if (debug1) { log.debug(" type 1 (floating point): bitOffset= " + bitOffset + " bitPrecision= " + bitPrecision + " expLocation= " + expLocation + " expSize= " + expSize + " manLocation= " + manLocation + " manSize= " + manSize + " expBias= " + expBias); } } else if (type == 2) { // time - short bitPrecision = raf.readShort(); + short bitPrecision = getRandomAccessFile().readShort(); if (bitPrecision == 16) timeType = DataType.SHORT; else if (bitPrecision == 32) @@ -3449,8 +3447,8 @@ else if (bitPrecision == 64) } } else if (type == 4) { // bit field - short bitOffset = raf.readShort(); - short bitPrecision = raf.readShort(); + short bitOffset = getRandomAccessFile().readShort(); + short bitPrecision = getRandomAccessFile().readShort(); if (debug1) { log.debug(" type 4 (bit field): bitOffset= " + bitOffset + " bitPrecision= " + bitPrecision); } @@ -3458,7 +3456,7 @@ else if (bitPrecision == 64) } else if (type == 5) { // opaque byte len = flags[0]; - opaque_desc = (len > 0) ? readString(raf).trim() : null; + opaque_desc = (len > 0) ? readString(getRandomAccessFile()).trim() : null; if (debug1) { log.debug(" type 5 (opaque): len= " + len + " desc= " + opaque_desc); } @@ -3498,20 +3496,20 @@ else if (bitPrecision == 64) String[] enumName = new String[nmembers]; for (int i = 0; i < nmembers; i++) { if (version < 3) - enumName[i] = readString8(raf); // padding + enumName[i] = readString8(getRandomAccessFile()); // padding else - enumName[i] = readString(raf); // no padding + enumName[i] = readString(getRandomAccessFile()); // no padding } // read the values; must switch to base byte order (!) if (base.endian >= 0) { - raf.order(base.endian); + getRandomAccessFile().order(base.endian); } int[] enumValue = new int[nmembers]; for (int i = 0; i < nmembers; i++) { enumValue[i] = (int) readVariableSizeUnsigned(base.byteSize); // assume size is 1, 2, or 4 } - raf.order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); enumTypeName = objectName; map = new TreeMap<>(); @@ -3540,14 +3538,14 @@ else if (bitPrecision == 64) if (debug1) { debugOut.print(" type 10(array) lengths= "); } - int ndims = (int) raf.readByte(); + int ndims = (int) getRandomAccessFile().readByte(); if (version < 3) { - raf.skipBytes(3); + getRandomAccessFile().skipBytes(3); } dim = new int[ndims]; for (int i = 0; i < ndims; i++) { - dim[i] = raf.readInt(); + dim[i] = getRandomAccessFile().readInt(); if (debug1) { debugOut.print(" " + dim[i]); } @@ -3556,7 +3554,7 @@ else if (bitPrecision == 64) if (version < 3) { // not present in version 3, never used anyway int[] pdim = new int[ndims]; for (int i = 0; i < ndims; i++) - pdim[i] = raf.readInt(); + pdim[i] = getRandomAccessFile().readInt(); } if (debug1) { log.debug(""); @@ -3599,13 +3597,13 @@ private class StructureMember { StructureMember(int version, int byteSize) throws IOException { if (debugPos) { - log.debug(" *StructureMember now at position={}", raf.getFilePointer()); + log.debug(" *StructureMember now at position={}", getRandomAccessFile().getFilePointer()); } - name = readString(raf); + name = readString(getRandomAccessFile()); if (version < 3) { - raf.skipBytes(padding(name.length() + 1, 8)); - offset = raf.readInt(); + getRandomAccessFile().skipBytes(padding(name.length() + 1, 8)); + offset = getRandomAccessFile().readInt(); } else { offset = (int) readVariableSizeMax(byteSize); } @@ -3615,9 +3613,9 @@ private class StructureMember { } if (version == 1) { - dims = raf.readByte(); - raf.skipBytes(3); - raf.skipBytes(24); // ignore dimension info for now + dims = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); + getRandomAccessFile().skipBytes(24); // ignore dimension info for now } // HDFdumpWithCount(buffer, raf.getFilePointer(), 16); @@ -3649,9 +3647,9 @@ private class MessageFillValueOld implements Named { int size; void read() throws IOException { - size = raf.readInt(); + size = getRandomAccessFile().readInt(); value = new byte[size]; - raf.readFully(value); + getRandomAccessFile().readFully(value); if (debug1) { log.debug("{}", this); @@ -3686,25 +3684,25 @@ private class MessageFillValue implements Named { byte flags; void read() throws IOException { - version = raf.readByte(); + version = getRandomAccessFile().readByte(); if (version < 3) { - spaceAllocateTime = raf.readByte(); - fillWriteTime = raf.readByte(); - hasFillValue = raf.readByte() != 0; + spaceAllocateTime = getRandomAccessFile().readByte(); + fillWriteTime = getRandomAccessFile().readByte(); + hasFillValue = getRandomAccessFile().readByte() != 0; } else { - flags = raf.readByte(); + flags = getRandomAccessFile().readByte(); spaceAllocateTime = (byte) (flags & 3); fillWriteTime = (byte) ((flags >> 2) & 3); hasFillValue = (flags & 32) != 0; } if (hasFillValue) { - size = raf.readInt(); + size = getRandomAccessFile().readInt(); if (size > 0) { value = new byte[size]; - raf.readFully(value); + getRandomAccessFile().readFully(value); hasFillValue = true; } else { hasFillValue = false; @@ -3808,41 +3806,41 @@ public String getName() { void read() throws IOException { int ndims; - byte version = raf.readByte(); + byte version = getRandomAccessFile().readByte(); if (version < 3) { - ndims = raf.readByte(); - type = raf.readByte(); - raf.skipBytes(5); // skip 5 bytes + ndims = getRandomAccessFile().readByte(); + type = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(5); // skip 5 bytes boolean isCompact = (type == 0); if (!isCompact) dataAddress = readOffset(); chunkSize = new int[ndims]; for (int i = 0; i < ndims; i++) - chunkSize[i] = raf.readInt(); + chunkSize[i] = getRandomAccessFile().readInt(); if (isCompact) { - dataSize = raf.readInt(); - dataAddress = raf.getFilePointer(); + dataSize = getRandomAccessFile().readInt(); + dataAddress = getRandomAccessFile().getFilePointer(); } } else { - type = raf.readByte(); + type = getRandomAccessFile().readByte(); if (type == 0) { - dataSize = raf.readShort(); - dataAddress = raf.getFilePointer(); + dataSize = getRandomAccessFile().readShort(); + dataAddress = getRandomAccessFile().getFilePointer(); } else if (type == 1) { dataAddress = readOffset(); contiguousSize = readLength(); } else if (type == 2) { - ndims = raf.readByte(); + ndims = getRandomAccessFile().readByte(); dataAddress = readOffset(); chunkSize = new int[ndims]; for (int i = 0; i < ndims; i++) - chunkSize[i] = raf.readInt(); + chunkSize[i] = getRandomAccessFile().readInt(); } } @@ -3857,10 +3855,10 @@ class MessageFilter implements Named { Filter[] filters; void read() throws IOException { - byte version = raf.readByte(); - byte nfilters = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + byte nfilters = getRandomAccessFile().readByte(); if (version == 1) - raf.skipBytes(6); + getRandomAccessFile().skipBytes(6); filters = new Filter[nfilters]; for (int i = 0; i < nfilters; i++) @@ -3902,21 +3900,22 @@ class Filter { int[] data; Filter(byte version) throws IOException { - this.id = raf.readShort(); - short nameSize = ((version > 1) && (id < 256)) ? 0 : raf.readShort(); // if the filter id < 256 then this field is - // not stored - this.flags = raf.readShort(); - nValues = raf.readShort(); + this.id = getRandomAccessFile().readShort(); + // if the filter id < 256 then this field is not stored + short nameSize = ((version > 1) && (id < 256)) ? 0 : getRandomAccessFile().readShort(); + this.flags = getRandomAccessFile().readShort(); + nValues = getRandomAccessFile().readShort(); if (version == 1) - this.name = (nameSize > 0) ? readString8(raf) : getFilterName(id); // null terminated, pad to 8 bytes + this.name = (nameSize > 0) ? readString8(getRandomAccessFile()) : getFilterName(id); // null terminated, pad to + // 8 bytes else this.name = (nameSize > 0) ? readStringFixedLength(nameSize) : getFilterName(id); // non-null terminated data = new int[nValues]; for (int i = 0; i < nValues; i++) - data[i] = raf.readInt(); + data[i] = getRandomAccessFile().readInt(); if ((version == 1) && (nValues & 1) != 0) // check if odd - raf.skipBytes(4); + getRandomAccessFile().skipBytes(4); if (debug1) { log.debug("{}", this); @@ -3987,51 +3986,52 @@ public String getName() { } boolean read(long pos) throws IOException { - raf.seek(pos); + getRandomAccessFile().seek(pos); if (debugPos) { - log.debug(" *MessageAttribute start pos= {}", raf.getFilePointer()); + log.debug(" *MessageAttribute start pos= {}", getRandomAccessFile().getFilePointer()); } short nameSize, typeSize, spaceSize; byte flags = 0; byte encoding = 0; // 0 = ascii, 1 = UTF-8 - version = raf.readByte(); + version = getRandomAccessFile().readByte(); if (version == 1) { - raf.read(); // skip byte - nameSize = raf.readShort(); - typeSize = raf.readShort(); - spaceSize = raf.readShort(); + getRandomAccessFile().read(); // skip byte + nameSize = getRandomAccessFile().readShort(); + typeSize = getRandomAccessFile().readShort(); + spaceSize = getRandomAccessFile().readShort(); } else if ((version == 2) || (version == 3)) { - flags = raf.readByte(); - nameSize = raf.readShort(); - typeSize = raf.readShort(); - spaceSize = raf.readShort(); + flags = getRandomAccessFile().readByte(); + nameSize = getRandomAccessFile().readShort(); + typeSize = getRandomAccessFile().readShort(); + spaceSize = getRandomAccessFile().readShort(); if (version == 3) - encoding = raf.readByte(); + encoding = getRandomAccessFile().readByte(); } else if (version == 72) { - flags = raf.readByte(); - nameSize = raf.readShort(); - typeSize = raf.readShort(); - spaceSize = raf.readShort(); - log.error("HDF5 MessageAttribute found bad version " + version + " at filePos " + raf.getFilePointer()); + flags = getRandomAccessFile().readByte(); + nameSize = getRandomAccessFile().readShort(); + typeSize = getRandomAccessFile().readShort(); + spaceSize = getRandomAccessFile().readShort(); + log.error("HDF5 MessageAttribute found bad version " + version + " at filePos " + + getRandomAccessFile().getFilePointer()); // G:/work/galibert/IMOS_ANMN-NSW_AETVZ_20131127T230000Z_PH100_FV01_PH100-1311-Workhorse-ADCP-109.5_END-20140306T010000Z_C-20140521T053527Z.nc // E:/work/antonio/2014_ch.nc // return false; } else { - log.error("bad version " + version + " at filePos " + raf.getFilePointer()); // buggery, may be HDF5 "more than - // 8 attributes" error + // buggery, maybe HDF5 "more than 8 attributes" error + log.error("bad version " + version + " at filePos " + getRandomAccessFile().getFilePointer()); return false; // throw new IllegalStateException("MessageAttribute unknown version " + version); } // read the attribute name - long filePos = raf.getFilePointer(); - name = readString(raf); // read at current pos + long filePos = getRandomAccessFile().getFilePointer(); + name = readString(getRandomAccessFile()); // read at current pos if (version == 1) nameSize += padding(nameSize, 8); - raf.seek(filePos + nameSize); // make it more robust for errors + getRandomAccessFile().seek(filePos + nameSize); // make it more robust for errors if (debug1) { log.debug(" MessageAttribute version= " + version + " flags = " + Integer.toBinaryString(flags) @@ -4039,7 +4039,7 @@ boolean read(long pos) throws IOException { } // read the datatype - filePos = raf.getFilePointer(); + filePos = getRandomAccessFile().getFilePointer(); if (debugPos) { log.debug(" *MessageAttribute before mdt pos= {}", filePos); } @@ -4054,20 +4054,20 @@ boolean read(long pos) throws IOException { if (version == 1) typeSize += padding(typeSize, 8); } - raf.seek(filePos + typeSize); // make it more robust for errors + getRandomAccessFile().seek(filePos + typeSize); // make it more robust for errors // read the dataspace - filePos = raf.getFilePointer(); + filePos = getRandomAccessFile().getFilePointer(); if (debugPos) { log.debug(" *MessageAttribute before mds = {}", filePos); } mds.read(); if (version == 1) spaceSize += padding(spaceSize, 8); - raf.seek(filePos + spaceSize); // make it more robust for errors + getRandomAccessFile().seek(filePos + spaceSize); // make it more robust for errors // the data starts immediately afterward - ie in the message - dataPos = raf.getFilePointer(); // note this is absolute position (no offset needed) + dataPos = getRandomAccessFile().getFilePointer(); // note this is absolute position (no offset needed) if (debug1) { log.debug(" *MessageAttribute dataPos= {}", dataPos); } @@ -4152,12 +4152,12 @@ String trunc(String s, int max) { void read() throws IOException { if (debugPos) { - log.debug(" *MessageAttributeInfo start pos= {}", raf.getFilePointer()); + log.debug(" *MessageAttributeInfo start pos= {}", getRandomAccessFile().getFilePointer()); } - byte version = raf.readByte(); - byte flags = raf.readByte(); + byte version = getRandomAccessFile().readByte(); + byte flags = getRandomAccessFile().readByte(); if ((flags & 1) != 0) - maxCreationIndex = raf.readShort(); + maxCreationIndex = getRandomAccessFile().readShort(); fractalHeapAddress = readOffset(); v2BtreeAddress = readOffset(); @@ -4176,7 +4176,7 @@ private class MessageComment implements Named { String comment; void read() throws IOException { - comment = readString(raf); + comment = readString(getRandomAccessFile()); } public String toString() { @@ -4195,9 +4195,9 @@ private class MessageLastModified implements Named { int secs; void read() throws IOException { - version = raf.readByte(); - raf.skipBytes(3); // skip byte - secs = raf.readInt(); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); // skip byte + secs = getRandomAccessFile().readInt(); } public String toString() { @@ -4215,7 +4215,7 @@ private class MessageLastModifiedOld implements Named { String datemod; void read() throws IOException { - datemod = raf.readString(14); + datemod = getRandomAccessFile().readString(14); if (debug1) { log.debug(" MessageLastModifiedOld={}", datemod); } @@ -4252,8 +4252,8 @@ private class MessageObjectReferenceCount implements Named { int refCount; void read() throws IOException { - int version = raf.readByte(); - refCount = raf.readInt(); + int version = getRandomAccessFile().readByte(); + refCount = getRandomAccessFile().readInt(); if (debug1) { log.debug(" ObjectReferenceCount={}", refCount); } @@ -4302,7 +4302,7 @@ private void readGroupNew(H5Group group, MessageGroupNew groupNewMessage, DataOb long pos = fractalHeapId.getPos(); if (pos < 0) continue; - raf.seek(pos); + getRandomAccessFile().seek(pos); MessageLink linkMessage = new MessageLink(); linkMessage.read(); if (debugBtree2) { @@ -4413,18 +4413,18 @@ List getSymbolTableEntries() { // recursively read all entries, place them in order in list protected void readAllEntries(long address, List entryList) throws IOException { - raf.seek(getFileOffset(address)); + getRandomAccessFile().seek(getFileOffset(address)); if (debugGroupBtree) { - log.debug("\n--> GroupBTree read tree at position={}", raf.getFilePointer()); + log.debug("\n--> GroupBTree read tree at position={}", getRandomAccessFile().getFilePointer()); } - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("TREE")) throw new IllegalStateException("BtreeGroup doesnt start with TREE"); - int type = raf.readByte(); - int level = raf.readByte(); - int nentries = raf.readShort(); + int type = getRandomAccessFile().readByte(); + int level = getRandomAccessFile().readByte(); + int nentries = getRandomAccessFile().readShort(); if (debugGroupBtree) { log.debug(" type=" + type + " level=" + level + " nentries=" + nentries); } @@ -4485,25 +4485,25 @@ class GroupNode { GroupNode(long address) throws IOException { this.address = address; - raf.seek(getFileOffset(address)); + getRandomAccessFile().seek(getFileOffset(address)); if (debugDetail) { - log.debug("--Group Node position={}", raf.getFilePointer()); + log.debug("--Group Node position={}", getRandomAccessFile().getFilePointer()); } // header - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("SNOD")) { throw new IllegalStateException(magic + " should equal SNOD"); } - version = raf.readByte(); - raf.readByte(); // skip byte - nentries = raf.readShort(); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().readByte(); // skip byte + nentries = getRandomAccessFile().readShort(); if (debugDetail) { log.debug(" version={} nentries={}", version, nentries); } - long posEntry = raf.getFilePointer(); + long posEntry = getRandomAccessFile().getFilePointer(); for (int i = 0; i < nentries; i++) { SymbolTableEntry entry = new SymbolTableEntry(posEntry); posEntry += entry.getSize(); @@ -4519,7 +4519,7 @@ class GroupNode { } } if (debugDetail) { - log.debug("-- Group Node end position={}", raf.getFilePointer()); + log.debug("-- Group Node end position={}", getRandomAccessFile().getFilePointer()); } long size = 8 + nentries * 40; if (debugTracker) @@ -4544,22 +4544,22 @@ private class SymbolTableEntry { boolean isSymbolicLink; SymbolTableEntry(long filePos) throws IOException { - raf.seek(filePos); + getRandomAccessFile().seek(filePos); if (debugSymbolTable) { - log.debug("--> readSymbolTableEntry position={}", raf.getFilePointer()); + log.debug("--> readSymbolTableEntry position={}", getRandomAccessFile().getFilePointer()); } nameOffset = readOffset(); objectHeaderAddress = readOffset(); - cacheType = raf.readInt(); - raf.skipBytes(4); + cacheType = getRandomAccessFile().readInt(); + getRandomAccessFile().skipBytes(4); if (debugSymbolTable) { log.debug(" nameOffset={} objectHeaderAddress={} cacheType={}", nameOffset, objectHeaderAddress, cacheType); } // "scratch pad" - posData = raf.getFilePointer(); + posData = getRandomAccessFile().getFilePointer(); if (debugSymbolTable) dump("Group Entry scratch pad", posData, 16, false); @@ -4573,7 +4573,7 @@ private class SymbolTableEntry { // check for symbolic link if (cacheType == 2) { - linkOffset = raf.readInt(); // offset in local heap + linkOffset = getRandomAccessFile().readInt(); // offset in local heap if (debugSymbolTable) { log.debug("WARNING Symbolic Link linkOffset={}", linkOffset); } @@ -4602,7 +4602,7 @@ private class SymbolTableEntry { */ if (debugSymbolTable) { - log.debug("<-- end readSymbolTableEntry position={}", raf.getFilePointer()); + log.debug("<-- end readSymbolTableEntry position={}", getRandomAccessFile().getFilePointer()); } if (debugTracker) @@ -4662,43 +4662,43 @@ Array getHeapDataArray(HeapIdentifier heapId, DataType dataType, int endian) log.debug(" HeapObject= {}", ho); } if (endian >= 0) { - raf.order(endian); + getRandomAccessFile().order(endian); } if (DataType.FLOAT == dataType) { float[] pa = new float[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readFloat(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readFloat(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (DataType.DOUBLE == dataType) { double[] pa = new double[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readDouble(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readDouble(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == byte.class) { byte[] pa = new byte[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readFully(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readFully(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == short.class) { short[] pa = new short[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readShort(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readShort(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == int.class) { int[] pa = new int[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readInt(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readInt(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } else if (dataType.getPrimitiveClassType() == long.class) { long[] pa = new long[heapId.nelems]; - raf.seek(ho.dataPos); - raf.readLong(pa, 0, pa.length); + getRandomAccessFile().seek(ho.dataPos); + getRandomAccessFile().readLong(pa, 0, pa.length); return Array.factory(dataType, new int[] {pa.length}, pa); } @@ -4722,7 +4722,7 @@ String readHeapString(long heapIdAddress) throws IOException { throw new IllegalStateException("Cant find Heap Object,heapId=" + heapId); if (ho.dataSize > 1000 * 1000) return String.format("Bad HeapObject.dataSize=%s", ho); - raf.seek(ho.dataPos); + getRandomAccessFile().seek(ho.dataPos); return readStringFixedLength((int) ho.dataSize); } @@ -4742,7 +4742,7 @@ String readHeapString(ByteBuffer bb, int pos) throws IOException { H5header.GlobalHeap.HeapObject ho = heapId.getHeapObject(); if (ho == null) throw new IllegalStateException("Cant find Heap Object,heapId=" + heapId); - raf.seek(ho.dataPos); + getRandomAccessFile().seek(ho.dataPos); return readStringFixedLength((int) ho.dataSize); } @@ -4787,11 +4787,11 @@ class HeapIdentifier { // address must be absolute, getFileOffset already added HeapIdentifier(long address) throws IOException { // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(address); - nelems = raf.readInt(); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(address); + nelems = getRandomAccessFile().readInt(); heapAddress = readOffset(); - index = raf.readInt(); + index = getRandomAccessFile().readInt(); if (debugDetail) { log.debug(" read HeapIdentifier address=" + address + this); } @@ -4842,10 +4842,10 @@ private class RegionReference { RegionReference(long filePos) throws IOException { // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(filePos); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(filePos); heapAddress = readOffset(); - index = raf.readInt(); + index = getRandomAccessFile().readInt(); GlobalHeap gheap; if (null == (gheap = heapMap.get(heapAddress))) { @@ -4865,8 +4865,8 @@ private class RegionReference { * src/H5S.c, where foo = {all, hyper, point, none}. * There is _no_ datatype information stored for these kind of selections currently. */ - raf.seek(want.dataPos); - long objId = raf.readLong(); + getRandomAccessFile().seek(want.dataPos); + long objId = getRandomAccessFile().readLong(); DataObject ndo = getDataObject(objId, null); // String what = (ndo == null) ? "none" : ndo.getName(); if (debugRegionReference) { @@ -4886,35 +4886,35 @@ private class GlobalHeap { GlobalHeap(long address) throws IOException { // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(getFileOffset(address)); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(getFileOffset(address)); // header - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("GCOL")) throw new IllegalStateException(magic + " should equal GCOL"); - version = raf.readByte(); - raf.skipBytes(3); - sizeBytes = raf.readInt(); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); + sizeBytes = getRandomAccessFile().readInt(); if (debugDetail) { log.debug("-- readGlobalHeap address=" + address + " version= " + version + " size = " + sizeBytes); // log.debug("-- readGlobalHeap address=" + address + " version= " + version + " size = " + sizeBytes); } - raf.skipBytes(4); // pad to 8 + getRandomAccessFile().skipBytes(4); // pad to 8 int count = 0; int countBytes = 0; while (true) { - long startPos = raf.getFilePointer(); + long startPos = getRandomAccessFile().getFilePointer(); HeapObject o = new HeapObject(); - o.id = raf.readShort(); + o.id = getRandomAccessFile().readShort(); if (o.id == 0) break; // ?? look - o.refCount = raf.readShort(); - raf.skipBytes(4); + o.refCount = getRandomAccessFile().readShort(); + getRandomAccessFile().skipBytes(4); o.dataSize = readLength(); - o.dataPos = raf.getFilePointer(); + o.dataPos = getRandomAccessFile().getFilePointer(); int dsize = ((int) o.dataSize) + padding((int) o.dataSize, 8); countBytes += dsize + 16; @@ -4931,7 +4931,7 @@ private class GlobalHeap { + o.dataSize + " dataPos = " + o.dataPos + " count= " + count + " countBytes= " + countBytes); } - raf.skipBytes(dsize); + getRandomAccessFile().skipBytes(dsize); hos.put(o.id, o); count++; @@ -4940,7 +4940,7 @@ private class GlobalHeap { } if (debugDetail) { - log.debug("-- endGlobalHeap position=" + raf.getFilePointer()); + log.debug("-- endGlobalHeap position=" + getRandomAccessFile().getFilePointer()); } if (debugTracker) memTracker.addByLen("GlobalHeap", address, sizeBytes); @@ -4975,21 +4975,21 @@ private class LocalHeap { this.group = group; // header information is in le byte order - raf.order(RandomAccessFile.LITTLE_ENDIAN); - raf.seek(getFileOffset(address)); + getRandomAccessFile().order(RandomAccessFile.LITTLE_ENDIAN); + getRandomAccessFile().seek(getFileOffset(address)); if (debugDetail) { - log.debug("-- readLocalHeap position={}", raf.getFilePointer()); + log.debug("-- readLocalHeap position={}", getRandomAccessFile().getFilePointer()); } // header - String magic = raf.readString(4); + String magic = getRandomAccessFile().readString(4); if (!magic.equals("HEAP")) { throw new IllegalStateException(magic + " should equal HEAP"); } - version = raf.readByte(); - raf.skipBytes(3); + version = getRandomAccessFile().readByte(); + getRandomAccessFile().skipBytes(3); size = (int) readLength(); freelistOffset = readLength(); dataAddress = readOffset(); @@ -4998,17 +4998,17 @@ private class LocalHeap { + " heap starts at dataAddress=" + dataAddress); } if (debugPos) { - log.debug(" *now at position={}", raf.getFilePointer()); + log.debug(" *now at position={}", getRandomAccessFile().getFilePointer()); } // data - raf.seek(getFileOffset(dataAddress)); + getRandomAccessFile().seek(getFileOffset(dataAddress)); heap = new byte[size]; - raf.readFully(heap); + getRandomAccessFile().readFully(heap); // if (debugHeap) printBytes( out, "heap", heap, size, true); if (debugDetail) { - log.debug("-- endLocalHeap position={}", raf.getFilePointer()); + log.debug("-- endLocalHeap position={}", getRandomAccessFile().getFilePointer()); } int hsize = 8 + 2 * sizeLengths + sizeOffsets; if (debugTracker) @@ -5099,17 +5099,17 @@ private String readString8(RandomAccessFile raf) throws IOException { * @throws java.io.IOException on io error */ private String readStringFixedLength(int size) throws IOException { - return raf.readString(size); + return getRandomAccessFile().readString(size); } @Override public long readLength() throws IOException { - return isLengthLong ? raf.readLong() : (long) raf.readInt(); + return isLengthLong ? getRandomAccessFile().readLong() : (long) getRandomAccessFile().readInt(); } @Override public long readOffset() throws IOException { - return isOffsetLong ? raf.readLong() : (long) raf.readInt(); + return isOffsetLong ? getRandomAccessFile().readLong() : (long) getRandomAccessFile().readInt(); } @Override @@ -5143,17 +5143,17 @@ private long readVariableSizeFactor(int sizeFactor) throws IOException { public long readVariableSizeUnsigned(int size) throws IOException { long vv; if (size == 1) { - vv = DataType.unsignedByteToShort(raf.readByte()); + vv = DataType.unsignedByteToShort(getRandomAccessFile().readByte()); } else if (size == 2) { if (debugPos) { - log.debug("position={}", raf.getFilePointer()); + log.debug("position={}", getRandomAccessFile().getFilePointer()); } - short s = raf.readShort(); + short s = getRandomAccessFile().readShort(); vv = DataType.unsignedShortToInt(s); } else if (size == 4) { - vv = DataType.unsignedIntToLong(raf.readInt()); + vv = DataType.unsignedIntToLong(getRandomAccessFile().readInt()); } else if (size == 8) { - vv = raf.readLong(); + vv = getRandomAccessFile().readLong(); } else { vv = readVariableSizeN(size); } @@ -5163,11 +5163,11 @@ public long readVariableSizeUnsigned(int size) throws IOException { private int readVariableSize(int size) throws IOException { long vv; if (size == 1) { - return raf.readByte(); + return getRandomAccessFile().readByte(); } else if (size == 2) { - return raf.readShort(); + return getRandomAccessFile().readShort(); } else if (size == 4) { - return raf.readInt(); + return getRandomAccessFile().readInt(); } throw new IllegalArgumentException("Dont support int size == " + size); } @@ -5176,7 +5176,7 @@ private int readVariableSize(int size) throws IOException { private long readVariableSizeN(int nbytes) throws IOException { int[] ch = new int[nbytes]; for (int i = 0; i < nbytes; i++) - ch[i] = raf.read(); + ch[i] = getRandomAccessFile().read(); long result = ch[nbytes - 1]; for (int i = nbytes - 2; i >= 0; i--) { @@ -5189,7 +5189,7 @@ private long readVariableSizeN(int nbytes) throws IOException { @Override public RandomAccessFile getRandomAccessFile() { - return raf; + return h5iosp.getRandomAccessFile(); } @Override @@ -5212,13 +5212,13 @@ private int padding(int nbytes, int multipleOf) { void dump(String head, long filePos, int nbytes, boolean count) throws IOException { if (debugOut == null) return; - long savePos = raf.getFilePointer(); + long savePos = getRandomAccessFile().getFilePointer(); if (filePos >= 0) - raf.seek(filePos); + getRandomAccessFile().seek(filePos); byte[] mess = new byte[nbytes]; - raf.readFully(mess); + getRandomAccessFile().readFully(mess); printBytes(head, mess, nbytes, false, debugOut); - raf.seek(savePos); + getRandomAccessFile().seek(savePos); } static void printBytes(String head, byte[] buff, int n, boolean count, java.io.PrintWriter ps) { diff --git a/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5iosp.java b/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5iosp.java index 726e08863f..3963742312 100644 --- a/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5iosp.java +++ b/cdm/core/src/main/java/ucar/nc2/iosp/hdf5/H5iosp.java @@ -90,6 +90,14 @@ public void getEosInfo(Formatter f) throws IOException { } } + RandomAccessFile getRandomAccessFile() { + return raf; + } + + H5header getHeader() { + return headerParser; + } + public static void useHdfEos(boolean val) { useHdfEos = val; } @@ -107,7 +115,7 @@ public static void useHdfEos(boolean val) { public void open(RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile, ucar.nc2.util.CancelTask cancelTask) throws IOException { super.open(raf, ncfile, cancelTask); - headerParser = new H5header(this.raf, ncfile, this); + headerParser = new H5header(ncfile, this); headerParser.read(null); // check if its an HDF5-EOS file @@ -605,7 +613,6 @@ public void close() throws IOException { @Override public void reacquire() throws IOException { super.reacquire(); - headerParser.raf = this.raf; } @Override @@ -626,7 +633,7 @@ public String getDetailInfo() { try { NetcdfFile ncfile = new NetcdfFileSubclass(); - H5header detailParser = new H5header(raf, ncfile, this); + H5header detailParser = new H5header(ncfile, this); detailParser.read(pw); f.format("%s", super.getDetailInfo()); f.format("%s", os.toString(CDM.UTF8)); @@ -650,7 +657,7 @@ public Object sendIospMessage(Object message) { if (message.toString().equals("headerEmpty")) { NetcdfFile ncfile = new NetcdfFileSubclass(); - return new H5header(raf, ncfile, this); + return new H5header(ncfile, this); } if (message.equals(IOSP_MESSAGE_GET_NETCDF_FILE_FORMAT)) { diff --git a/cdm/core/src/main/java/ucar/unidata/io/RandomAccessFile.java b/cdm/core/src/main/java/ucar/unidata/io/RandomAccessFile.java index f3fa58a4c9..6ce48d2a68 100644 --- a/cdm/core/src/main/java/ucar/unidata/io/RandomAccessFile.java +++ b/cdm/core/src/main/java/ucar/unidata/io/RandomAccessFile.java @@ -156,7 +156,7 @@ public FileCacheable open(DatasetUrl durl, int buffer_size, CancelTask cancelTas throws IOException { String location = StringUtil2.replace(durl.getTrueurl(), "\\", "/"); // canonicalize the name RandomAccessFile result = new RandomAccessFile(location, "r", buffer_size); - result.cacheState = 1; // in use + result.cacheState = CacheState.IN_USE; return result; } }; @@ -211,7 +211,16 @@ public static void shutdown() { * File location */ protected String location; - private int cacheState; // 0 - not in cache, 1 = in cache && in use, 2 = in cache but not in use + + enum CacheState { + NOT_IN_CACHE, IN_USE, NOT_IN_USE + } + + private CacheState cacheState = CacheState.NOT_IN_CACHE; + + CacheState getCacheState() { + return cacheState; + } /** * The underlying java.io.RandomAccessFile. @@ -394,16 +403,17 @@ public int getBufferSize() { */ public synchronized void close() throws IOException { if (cache != null) { - if (cacheState > 0) { - if (cacheState == 1) { - cacheState = 2; + switch (cacheState) { + case NOT_IN_CACHE: + break; + case IN_USE: + cacheState = CacheState.NOT_IN_USE; // return true if in the cache, otherwise was opened regular, so must be closed regular if (cache.release(this)) return; - cacheState = 0; // release failed, bail out - } else { + cacheState = CacheState.NOT_IN_CACHE; // release failed, bail out + case NOT_IN_USE: return; // close has been called more than once - ok - } } } @@ -436,14 +446,14 @@ public synchronized void close() throws IOException { @Deprecated @Override public void release() { // one to one with java.io.RandomAccessFile - cacheState = 2; + cacheState = CacheState.NOT_IN_USE; } /** @deprecated do not use */ @Deprecated @Override public void reacquire() { - cacheState = 1; + cacheState = CacheState.IN_USE; } /** @deprecated do not use */ @@ -451,7 +461,7 @@ public void reacquire() { @Override public synchronized void setFileCache(FileCacheIF fileCache) { if (fileCache == null) - cacheState = 0; + cacheState = CacheState.NOT_IN_CACHE; } @Override @@ -730,10 +740,14 @@ public long readToByteChannel(WritableByteChannel dest, long offset, long nbytes * @param b put data into this buffer * @param offset buffer offset * @param len this number of bytes - * @return actual number of bytes read + * @return actual number of bytes read, -1 if underlying random access file was closed * @throws IOException on io error */ protected int read_(long pos, byte[] b, int offset, int len) throws IOException { + if (file == null) { + return -1; + } + file.seek(pos); int n = file.read(b, offset, len); if (debugAccess) { diff --git a/cdm/core/src/test/java/ucar/nc2/internal/iosp/hdf5/TestH5iospNew.java b/cdm/core/src/test/java/ucar/nc2/internal/iosp/hdf5/TestH5iospNew.java new file mode 100644 index 0000000000..f33716a0b5 --- /dev/null +++ b/cdm/core/src/test/java/ucar/nc2/internal/iosp/hdf5/TestH5iospNew.java @@ -0,0 +1,63 @@ +package ucar.nc2.internal.iosp.hdf5; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.iosp.IOServiceProvider; +import ucar.unidata.util.test.TestDir; + +public class TestH5iospNew { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String TEST_FILE = TestDir.cdmLocalTestDataDir + "hdf5/structmetadata_eos.h5"; + + private NetcdfFile netcdfFile; + private H5iospNew h5iospNew; + + @Before + public void open() throws Exception { + netcdfFile = NetcdfFiles.open(TEST_FILE); + final IOServiceProvider iosp = netcdfFile.getIosp(); + assertThat(iosp).isInstanceOf(H5iospNew.class); + h5iospNew = (H5iospNew) iosp; + assertThat(h5iospNew).isNotNull(); + } + + @After + public void close() throws Exception { + netcdfFile.close(); + } + + @Test + public void shouldReleaseRafs() throws IOException { + assertThat(h5iospNew.getRandomAccessFile()).isNotNull(); + assertThat(h5iospNew.getHeader().getRandomAccessFile()).isNotNull(); + assertThat(h5iospNew.getHeader().getH5objects().getRandomAccessFile()).isNotNull(); + + netcdfFile.release(); + + assertThat(h5iospNew.getRandomAccessFile()).isNull(); + assertThat(h5iospNew.getHeader().getRandomAccessFile()).isNull(); + assertThat(h5iospNew.getHeader().getH5objects().getRandomAccessFile()).isNull(); + } + + @Test + public void shouldCloseRafs() throws IOException { + assertThat(h5iospNew.getRandomAccessFile()).isNotNull(); + assertThat(h5iospNew.getHeader().getRandomAccessFile()).isNotNull(); + assertThat(h5iospNew.getHeader().getH5objects().getRandomAccessFile()).isNotNull(); + + netcdfFile.close(); + + assertThat(h5iospNew.getRandomAccessFile()).isNull(); + assertThat(h5iospNew.getHeader().getRandomAccessFile()).isNull(); + assertThat(h5iospNew.getHeader().getH5objects().getRandomAccessFile()).isNull(); + } +} diff --git a/cdm/core/src/test/java/ucar/nc2/iosp/hdf5/TestDataBTree.java b/cdm/core/src/test/java/ucar/nc2/iosp/hdf5/TestDataBTree.java new file mode 100644 index 0000000000..a6b018dac2 --- /dev/null +++ b/cdm/core/src/test/java/ucar/nc2/iosp/hdf5/TestDataBTree.java @@ -0,0 +1,54 @@ +package ucar.nc2.iosp.hdf5; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Variable; +import ucar.nc2.iosp.hdf5.H5header.Vinfo; +import ucar.unidata.util.test.TestDir; + +public class TestDataBTree { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String TEST_FILE = TestDir.cdmLocalTestDataDir + "chunked.h5"; + + private NetcdfFile netcdfFile; + + @Before + public void open() throws Exception { + netcdfFile = NetcdfFile.open(TEST_FILE); + } + + @After + public void close() throws Exception { + netcdfFile.close(); + } + + @Test + public void shouldReleaseRafs() throws IOException { + final Variable variable = netcdfFile.findVariable("data"); + assertThat((Object) variable).isNotNull(); + final DataBTree bTree = ((Vinfo) variable.getSPobject()).btree; + + assertThat(bTree.getRandomAccessFile()).isNotNull(); + netcdfFile.release(); + assertThat(bTree.getRandomAccessFile()).isNull(); + } + + @Test + public void shouldCloseRafs() throws IOException { + final Variable variable = netcdfFile.findVariable("data"); + assertThat((Object) variable).isNotNull(); + final DataBTree bTree = ((Vinfo) variable.getSPobject()).btree; + + assertThat(bTree.getRandomAccessFile()).isNotNull(); + netcdfFile.close(); + assertThat(bTree.getRandomAccessFile()).isNull(); + } +} diff --git a/cdm/core/src/test/java/ucar/nc2/iosp/hdf5/TestH5iosp.java b/cdm/core/src/test/java/ucar/nc2/iosp/hdf5/TestH5iosp.java new file mode 100644 index 0000000000..fdc07ae719 --- /dev/null +++ b/cdm/core/src/test/java/ucar/nc2/iosp/hdf5/TestH5iosp.java @@ -0,0 +1,58 @@ +package ucar.nc2.iosp.hdf5; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ucar.nc2.NetcdfFile; +import ucar.nc2.iosp.IOServiceProvider; +import ucar.unidata.util.test.TestDir; + +public class TestH5iosp { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String TEST_FILE = TestDir.cdmLocalTestDataDir + "hdf5/structmetadata_eos.h5"; + + private NetcdfFile netcdfFile; + private H5iosp h5iosp; + + @Before + public void open() throws Exception { + netcdfFile = NetcdfFile.open(TEST_FILE); + final IOServiceProvider iosp = netcdfFile.getIosp(); + assertThat(iosp).isInstanceOf(H5iosp.class); + h5iosp = (H5iosp) iosp; + assertThat(h5iosp).isNotNull(); + } + + @After + public void close() throws Exception { + netcdfFile.close(); + } + + @Test + public void shouldReleaseRafs() throws IOException { + assertThat(h5iosp.getRandomAccessFile()).isNotNull(); + assertThat(h5iosp.getHeader().getRandomAccessFile()).isNotNull(); + + netcdfFile.release(); + + assertThat(h5iosp.getRandomAccessFile()).isNull(); + assertThat(h5iosp.getHeader().getRandomAccessFile()).isNull(); + } + + @Test + public void shouldCloseRafs() throws IOException { + assertThat(h5iosp.getRandomAccessFile()).isNotNull(); + assertThat(h5iosp.getHeader().getRandomAccessFile()).isNotNull(); + + netcdfFile.close(); + + assertThat(h5iosp.getRandomAccessFile()).isNull(); + assertThat(h5iosp.getHeader().getRandomAccessFile()).isNull(); + } +} diff --git a/cdm/core/src/test/java/ucar/unidata/io/TestRandomAccessFile.java b/cdm/core/src/test/java/ucar/unidata/io/TestRandomAccessFile.java index 34bafbde0a..4e96d4af1a 100644 --- a/cdm/core/src/test/java/ucar/unidata/io/TestRandomAccessFile.java +++ b/cdm/core/src/test/java/ucar/unidata/io/TestRandomAccessFile.java @@ -207,6 +207,22 @@ public void testReadToByteChannel() throws IOException { assertThat(n).isLessThan(nbytes); } + @Test + public void testReadClosedRaf() throws IOException { + RandomAccessFile closedTempFile = new RandomAccessFile(TEST_FILE_PATH, "r", TEST_BUFFER_SIZE); + int n = 1; + byte[] expected = new byte[TEST_BUFFER_SIZE + n]; + System.arraycopy(UTF8_BYTES, 0, expected, 0, TEST_BUFFER_SIZE); + System.arraycopy(new byte[n], 0, expected, TEST_BUFFER_SIZE, n); + + closedTempFile.seek(0); + closedTempFile.close(); + + byte[] actual = new byte[TEST_BUFFER_SIZE + n]; + closedTempFile.read(actual, 0, TEST_BUFFER_SIZE + n); + assertThat(arraysMatch(expected, actual, 0, 0, TEST_BUFFER_SIZE + n)).isTrue(); + } + @Test public void testReadFully() throws IOException { // read fully, buff < file length diff --git a/cdm/core/src/test/java/ucar/unidata/io/TestRandomAccessFileCache.java b/cdm/core/src/test/java/ucar/unidata/io/TestRandomAccessFileCache.java new file mode 100644 index 0000000000..27798f4aed --- /dev/null +++ b/cdm/core/src/test/java/ucar/unidata/io/TestRandomAccessFileCache.java @@ -0,0 +1,91 @@ +package ucar.unidata.io; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import ucar.unidata.io.RandomAccessFile.CacheState; + +public class TestRandomAccessFileCache { + + @ClassRule + public static final TemporaryFolder tempFolder = new TemporaryFolder(); + + private static RandomAccessFile testFile; + private static final String TEST_FILE_PATH = "src/test/data/preserveLineEndings/testUTF8.txt"; + + @BeforeClass + public static void enableCache() { + RandomAccessFile.enableDefaultGlobalFileCache(); + } + + @AfterClass + public static void shutdownCache() { + RandomAccessFile.setGlobalFileCache(null); + } + + @Before + public void setUpTestFile() throws IOException { + testFile = RandomAccessFile.acquire(TEST_FILE_PATH); + } + + @After + public void cleanUpTestFile() throws IOException { + testFile.close(); + } + + @Test + public void shouldReturnInUseWhenFileIsAcquired() { + assertThat(testFile.getCacheState()).isEqualTo(CacheState.IN_USE); + } + + @Test + public void shouldReturnNotInUseWhenFileIsClosed() throws IOException { + testFile.close(); + assertThat(testFile.getCacheState()).isEqualTo(CacheState.NOT_IN_USE); + } + + @Test + public void shouldReturnNotInCacheWhenFileNotInCacheIsClosed() throws IOException { + testFile.setFileCache(null); + testFile.close(); + assertThat(testFile.getCacheState()).isEqualTo(CacheState.NOT_IN_CACHE); + } + + @Test + public void shouldReturnNotInUseWhenFileIsClosedTwice() throws IOException { + testFile.close(); + testFile.close(); + assertThat(testFile.getCacheState()).isEqualTo(CacheState.NOT_IN_USE); + } + + @Test + public void shouldReturnNotInUseWhenFileIsReleased() { + testFile.release(); + assertThat(testFile.getCacheState()).isEqualTo(CacheState.NOT_IN_USE); + } + + @Test + public void shouldReturnInUseWhenFileIsReacquired() { + testFile.reacquire(); + assertThat(testFile.getCacheState()).isEqualTo(CacheState.IN_USE); + } + + @Test + public void shouldReturnNotInCacheWhenFileCacheIsReset() { + testFile.setFileCache(null); + assertThat(testFile.getCacheState()).isEqualTo(CacheState.NOT_IN_CACHE); + } + + @Test + public void shouldReturnNotInCacheForNewRaf() throws IOException { + final RandomAccessFile raf = new RandomAccessFile(tempFolder.newFile().getAbsolutePath(), "r", 0); + assertThat(raf.getCacheState()).isEqualTo(CacheState.NOT_IN_CACHE); + } +} diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribIosp.java b/grib/src/main/java/ucar/nc2/grib/collection/GribIosp.java index 91f6cb951f..38ed08b548 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribIosp.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribIosp.java @@ -785,13 +785,40 @@ private Array makeLazyTime2Darray(Variable coord, Time2Dinfo info) { break; case intvU: - count = 0; + // data[nruns*ntimes] for (int runIdx = 0; runIdx < nruns; runIdx++) { CoordinateTimeIntv timeIntv = (CoordinateTimeIntv) time2D.getTimeCoordinate(runIdx); + double valueBound1 = Double.NaN; + double valueBound2 = Double.NaN; + double twicePrevValueBound2 = Double.NaN; + int timeIndex = 0; for (TimeCoordIntvValue tinv : timeIntv.getTimeIntervals()) { - data[count++] = timeUnit.getValue() * tinv.getCoordValue() + time2D.getOffset(runIdx); + valueBound1 = timeUnit.getValue() * tinv.getBounds1() + time2D.getOffset(runIdx); + valueBound2 = timeUnit.getValue() * tinv.getBounds2() + time2D.getOffset(runIdx); + // Use end-point of current interval as initial guess for current time coordinate value + data[timeIndex] = valueBound2; + if (timeIndex >= 1) { + // Check that time coordinate values are increasing in a strictly-monotonic manner + // (as required by CF conventions). If not strictly-monotonic ... + if (data[timeIndex] <= data[timeIndex - 1]) { + if (timeIndex >= 2 && data[timeIndex - 2] <= valueBound1) { + // Change previous time coordinate value to mid-point between + // current time interval start and end values. + data[timeIndex - 1] = (valueBound2 - valueBound1) / 2.0 + valueBound1; + } else { + // Or change previous time coordinate value to mid-point between + // current time interval end value and the time coord value from two steps back. + twicePrevValueBound2 = timeUnit.getValue() * data[timeIndex - 2] + time2D.getOffset(runIdx); + data[timeIndex - 1] = (valueBound2 - twicePrevValueBound2) / 2.0 + twicePrevValueBound2; + } + } + } + timeIndex++; } } + // The above assumes that sets of intervals are always sorted by + // end point then starting point. That sorting scheme is implemented in + // ucar.nc2.grib.coord.TimeCoordIntvValue.compareTo(o) -- 21 Dec 2022. break; case is1Dtime: diff --git a/grib/src/test/data/GFS_Global_onedeg_20220627_0000.TotalPrecip.Out48hrs.grib2 b/grib/src/test/data/GFS_Global_onedeg_20220627_0000.TotalPrecip.Out48hrs.grib2 new file mode 100644 index 0000000000..82926780de Binary files /dev/null and b/grib/src/test/data/GFS_Global_onedeg_20220627_0000.TotalPrecip.Out48hrs.grib2 differ diff --git a/grib/src/test/java/ucar/nc2/grib/coord/TestDiscontiguousInterval.java b/grib/src/test/java/ucar/nc2/grib/coord/TestDiscontiguousInterval.java new file mode 100644 index 0000000000..cae758c063 --- /dev/null +++ b/grib/src/test/java/ucar/nc2/grib/coord/TestDiscontiguousInterval.java @@ -0,0 +1,60 @@ +package ucar.nc2.grib.coord; + +import org.junit.Assert; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.nc2.*; + +import java.io.IOException; + +public class TestDiscontiguousInterval { + + /** + * Ensure that when creating a time coordinate variable from a series of + * time intervals that are discontiguous and mixed, the time coordinate values + * increase in a strictly-monotonic manner (as required by the CF Conventions + * for netCDF). + * + * @throws IOException + */ + @Test + public void testTimeCoordinate1D_isStrictlyMonotonicallyIncreasing() throws IOException { + String testfile = "../grib/src/test/data/GFS_Global_onedeg_20220627_0000.TotalPrecip.Out48hrs.grib2"; + try (NetcdfFile nc = NetcdfFiles.open(testfile)) { + Variable dataVar = nc.findVariable("Total_precipitation_surface_Mixed_intervals_Accumulation"); + Assert.assertNotNull(dataVar); + + Dimension timeDim = null; + for (Dimension dim : dataVar.getDimensions()) { + if (dim.getShortName().startsWith("time")) { + timeDim = dim; + break; + } + } + Variable timeCoordVar = nc.findVariable(timeDim.getShortName()); + Assert.assertNotNull(timeCoordVar); + + Attribute att = timeCoordVar.findAttribute("bounds"); + Assert.assertNotNull(att); + Assert.assertEquals(timeDim.getShortName() + "_bounds", att.getStringValue()); + + Array timeCoordValues = timeCoordVar.read(); + checkTimeCoordinateVariable1D_IsStrictlyMonotonicallyIncreasing(timeDim.getLength(), timeCoordValues); + } + } + + private void checkTimeCoordinateVariable1D_IsStrictlyMonotonicallyIncreasing(int timeDimLength, + Array timeCoordValues) { + double currentValue = timeCoordValues.getDouble(0); + double prevValue = currentValue; + StringBuilder valuesSoFar = new StringBuilder(); + valuesSoFar.append(currentValue); + for (int i = 1; i < timeDimLength; i++) { + currentValue = timeCoordValues.getDouble(i); + valuesSoFar.append(", ").append(currentValue); + Assert.assertTrue("Not increasing in a strictly-monotonic manner: [" + valuesSoFar + "]", + currentValue > prevValue); + prevValue = currentValue; + } + } +} diff --git a/netcdf-java-platform/build.gradle b/netcdf-java-platform/build.gradle index db66d8fef9..9cbc1a39f6 100644 --- a/netcdf-java-platform/build.gradle +++ b/netcdf-java-platform/build.gradle @@ -28,7 +28,7 @@ dependencies { // netcdf4, dap4:d4lib // Tricky dependency here. We need to make sure that we keep in-line with the version // that chronicle-map uses in the TDS, or else we see bad things happen on the TDS side. - api 'net.java.dev.jna:jna:5.4.0' + api 'net.java.dev.jna:jna:5.12.1' // Annotations: Nullable api 'com.google.code.findbugs:jsr305:3.0.2' diff --git a/project-files/owasp-dependency-check/dependency-check-suppression.xml b/project-files/owasp-dependency-check/dependency-check-suppression.xml index b63f2735d2..e5fe1b31f9 100644 --- a/project-files/owasp-dependency-check/dependency-check-suppression.xml +++ b/project-files/owasp-dependency-check/dependency-check-suppression.xml @@ -6,7 +6,7 @@ reason: Fixed in junit-4.13.1 for applications running JDK 1.7 and later. We require Java 1.8 at a minimum, so we're good with regards to this CVE. See https://nvd.nist.gov/vuln/detail/CVE-2020-15250. - ]]> + ]]> ^pkg:maven/junit/junit@.*$ CVE-2020-15250 @@ -14,8 +14,48 @@ + ]]> ^pkg:maven/org\.jdom/jdom2@.*$ CVE-2021-33813 + + + d539c36ab347d5df35659b174d28b94e6d2536ed + CVE-2019-11358 + + + + d539c36ab347d5df35659b174d28b94e6d2536ed + CVE-2012-6708 + + + + d539c36ab347d5df35659b174d28b94e6d2536ed + CVE-2015-9251 + + + + d539c36ab347d5df35659b174d28b94e6d2536ed + CVE-2020-11022 + + + + d539c36ab347d5df35659b174d28b94e6d2536ed + CVE-2020-11023 +