Merge pull request #99 from jamesmudd/fletcher-checksum-workaround

Fletcher checksum workaround

Author: jamesmudd

jhdf/src/main/java/io/jhdf/filter/FilterManager.java

@@ -40,6 +40,7 @@ public enum FilterManager {; // Enum singleton pattern
 		// Load the built in filters
 		addFilter(new DeflatePipelineFilter());
 		addFilter(new ByteShuffleFilter());
+		addFilter(new FletcherChecksumFilter());
 
 		// Add dynamically loaded filters
 		ServiceLoader<Filter> serviceLoader = ServiceLoader.load(Filter.class);

jhdf/src/main/java/io/jhdf/filter/FilterPipeline.java

@@ -23,15 +23,15 @@ public class FilterPipeline {
 
 	private class PipelineFilterWithData {
 
-		final Filter filter;
-		final int[] filterData;
+		private final Filter filter;
+		private final int[] filterData;
 
-		public PipelineFilterWithData(Filter filter, int[] filterData) {
+		private PipelineFilterWithData(Filter filter, int[] filterData) {
 			this.filter = filter;
 			this.filterData = filterData;
 		}
 
-		byte[] decode(byte[] data) {
+		private byte[] decode(byte[] data) {
 			return filter.decode(data, filterData);
 		}
 	}

jhdf/src/main/java/io/jhdf/filter/FletcherChecksumFilter.java

@@ -0,0 +1,45 @@
+/*******************************************************************************
+ * This file is part of jHDF. A pure Java library for accessing HDF5 files.
+ *
+ * http://jhdf.io
+ *
+ * Copyright 2019 James Mudd
+ *
+ * MIT License see 'LICENSE' file
+ ******************************************************************************/
+package io.jhdf.filter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a placeholder for the checksum filter. Currently a no-op allows datasets with a checksum to be read but the
+ * checksum is not validated.
+ *
+ * @author James Mudd
+ */
+public class FletcherChecksumFilter implements Filter {
+    private static final Logger logger = LoggerFactory.getLogger(FletcherChecksumFilter.class);
+
+    private boolean warningIssued = false;
+
+    @Override
+    public int getId() {
+        return 3;
+    }
+
+    @Override
+    public String getName() {
+        return "fletcher32";
+    }
+
+    @Override
+    public byte[] decode(byte[] encodedData, int[] filterData) {
+        if(!warningIssued) {
+            logger.warn("Fletcher 32 checksum will not be verified");
+            warningIssued = true;
+        }
+
+        return encodedData;
+    }
+}

jhdf/src/test/resources/io/jhdf/fletcher32_datasets_earliest.hdf5

@@ -0,0 +1,41 @@
+#-------------------------------------------------------------------------------
+# This file is part of jHDF. A pure Java library for accessing HDF5 files.
+# 
+# http://jhdf.io
+# 
+# Copyright 2019 James Mudd
+# 
+# MIT License see 'LICENSE' file
+#-------------------------------------------------------------------------------
+import h5py
+
+import numpy as np
+
+def write_chunked_datasets(f):
+    
+    data = np.arange(35).reshape(7,5)
+
+    float_group = f.create_group('float')
+    float_group.create_dataset('float32', data=data, dtype='f4', chunks=(2,1), fletcher32=True)
+    float_group.create_dataset('float64', data=data, dtype='f8', chunks=(3,4), fletcher32=True)
+    
+    int_group = f.create_group('int')
+    int_group.create_dataset('int8', data=data, dtype='i1', chunks=(5,3), fletcher32=True)
+    int_group.create_dataset('int16', data=data, dtype='i2', chunks=(1,1), fletcher32=True)
+    int_group.create_dataset('int32', data=data, dtype='i4', chunks=(1,3), fletcher32=True)
+
+    f.flush()
+    f.close()
+
+if __name__ == '__main__':
+    print('Making compressed fletcher checksum dataset test files...')
+
+    f = h5py.File('fletcher32_datasets_earliest.hdf5', 'w', libver='earliest')
+    write_chunked_datasets(f)
+    print('fletcher32_datasets_earliest.hdf5')
+
+# Start working just on 1.8 files first
+#     f = h5py.File('fletcher32_datasets_latest.hdf5', 'w', libver='latest')
+#     write_chunked_datasets(f)
+#     print('fletcher32_datasets_latest.hdf5')
+