Skip to content

Commit

Permalink
Expand benchmark test to include read performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@chris-allan
chris-allan committed Jul 30, 2020
1 parent 3135da2 commit cd1fbb9
Showing 1 changed file with 101 additions and 12 deletions.
113 changes: 101 additions & 12 deletions src/test/java/org/janelia/saalfeldlab/n5/N5BenchmarkTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
Expand Down Expand Up @@ -56,11 +57,11 @@ public class N5BenchmarkTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();

private static String datasetName = "/dataset";
private static final String datasetName = "/dataset";

private static N5Writer n5;
private N5Writer n5;

private static short[] data;
private short[] data;

private static final Compression[] compressions = {
new RawCompression(),
Expand Down Expand Up @@ -194,10 +195,10 @@ public void benchmarkParallelWritingSpeed() {

final ExecutorService exec = Executors.newFixedThreadPool(i);
final ArrayList<Future<Boolean>> futures = new ArrayList<>();
long t;
long start, read, write;

for (final Compression compression : compressions) {
t = System.currentTimeMillis();
start = System.currentTimeMillis();
try {
final String compressedDatasetName = datasetName + "." + compression.getType();
n5.createDataset(compressedDatasetName, new long[]{64 * nBlocks, 64 * nBlocks, 64 * nBlocks}, new int[]{64, 64, 64}, DataType.UINT16, compression);
Expand All @@ -220,16 +221,38 @@ public void benchmarkParallelWritingSpeed() {
}
for (final Future<Boolean> f : futures)
f.get();
write = System.currentTimeMillis();

System.out.println(String.format("%d : %s : %fs", i, compression.getType(), 0.001 * (System.currentTimeMillis() - t)));
for (int z = 0; z < nBlocks; ++z) {
final int fz = z;
for (int y = 0; y < nBlocks; ++y) {
final int fy = y;
for (int x = 0; x < nBlocks; ++x) {
final int fx = x;
futures.add(
exec.submit(
() -> {
final ShortArrayDataBlock dataBlock = (ShortArrayDataBlock) n5.readBlock(compressedDatasetName, attributes, new long[]{fx, fy, fz});
Assert.assertArrayEquals(new int[]{64, 64, 64}, dataBlock.size);
return true;
}));
}
}
}
for (final Future<Boolean> f : futures)
f.get();
read = System.currentTimeMillis();

System.out.println(String.format("%d : %s write : %fs", i, compression.getType(), 0.001 * (write - start)));
System.out.println(String.format("%d : %s read : %fs", i, compression.getType(), 0.001 * (read - write)));
} catch (final IOException | InterruptedException | ExecutionException e) {
fail(e.getMessage());
}
}

/* TIF blocks */
futures.clear();
t = System.currentTimeMillis();
start = System.currentTimeMillis();
try {
for (int z = 0; z < nBlocks; ++z) {
final int fz = z;
Expand All @@ -249,15 +272,37 @@ public void benchmarkParallelWritingSpeed() {
}
for (final Future<Boolean> f : futures)
f.get();
write = System.currentTimeMillis();

System.out.println(String.format("%d : tif : %fs", i, 0.001 * (System.currentTimeMillis() - t)));
for (int z = 0; z < nBlocks; ++z) {
final int fz = z;
for (int y = 0; y < nBlocks; ++y) {
final int fy = y;
for (int x = 0; x < nBlocks; ++x) {
final int fx = x;
futures.add(
exec.submit(
() -> {
final ImagePlus impBlock = new Opener().openTiff(temp.getRoot().toPath().resolve(fx + "-" + fy + "-" + fz + ".tif").toString(), 1);
Assert.assertArrayEquals(new int[]{64, 64 * 64, 1, 1, 1}, impBlock.getDimensions());
return true;
}));
}
}
}
for (final Future<Boolean> f : futures)
f.get();
read = System.currentTimeMillis();

System.out.println(String.format("%d : tif write : %fs", i, 0.001 * (write - start)));
System.out.println(String.format("%d : tif read : %fs", i, 0.001 * (read - write)));
} catch (final InterruptedException | ExecutionException e) {
fail(e.getMessage());
}

/* HDF5 raw */
futures.clear();
t = System.currentTimeMillis();
start = System.currentTimeMillis();
try {
final String hdf5Name = temp.getRoot().toPath().resolve("dataset.h5").toString();
final IHDF5Writer hdf5Writer = HDF5Factory.open( hdf5Name );
Expand Down Expand Up @@ -285,16 +330,38 @@ public void benchmarkParallelWritingSpeed() {
}
for (final Future<Boolean> f : futures)
f.get();
write = System.currentTimeMillis();

for (int z = 0; z < nBlocks; ++z) {
final int fz = z;
for (int y = 0; y < nBlocks; ++y) {
final int fy = y;
for (int x = 0; x < nBlocks; ++x) {
final int fx = x;
futures.add(
exec.submit(
() -> {
final MDShortArray targetCell = uint16Writer.readMDArrayBlockWithOffset(datasetName, new int[]{64, 64, 64}, new long[]{64 * fz, 64 * fy, 64 * fx});
Assert.assertArrayEquals(new int[]{64, 64, 64}, targetCell.dimensions());
return true;
}));
}
}
}
for (final Future<Boolean> f : futures)
f.get();
read = System.currentTimeMillis();

hdf5Writer.close();
System.out.println(String.format("%d : hdf5 raw : %fs", i, 0.001 * (System.currentTimeMillis() - t)));
System.out.println(String.format("%d : hdf5 raw write : %fs", i, 0.001 * (write - start)));
System.out.println(String.format("%d : hdf5 raw read : %fs", i, 0.001 * (read - write)));
} catch (final InterruptedException | ExecutionException e) {
fail(e.getMessage());
}

/* HDF5 gzip */
futures.clear();
t = System.currentTimeMillis();
start = System.currentTimeMillis();
try {
final String hdf5Name = temp.getRoot().toPath().resolve("dataset.gz.h5").toString();
final IHDF5Writer hdf5Writer = HDF5Factory.open( hdf5Name );
Expand Down Expand Up @@ -322,9 +389,31 @@ public void benchmarkParallelWritingSpeed() {
}
for (final Future<Boolean> f : futures)
f.get();
write = System.currentTimeMillis();

for (int z = 0; z < nBlocks; ++z) {
final int fz = z;
for (int y = 0; y < nBlocks; ++y) {
final int fy = y;
for (int x = 0; x < nBlocks; ++x) {
final int fx = x;
futures.add(
exec.submit(
() -> {
final MDShortArray targetCell = uint16Writer.readMDArrayBlockWithOffset(datasetName, new int[]{64, 64, 64}, new long[]{64 * fz, 64 * fy, 64 * fx});
Assert.assertArrayEquals(new int[]{64, 64, 64}, targetCell.dimensions());
return true;
}));
}
}
}
for (final Future<Boolean> f : futures)
f.get();
read = System.currentTimeMillis();

hdf5Writer.close();
System.out.println(String.format("%d : hdf5 gzip : %fs", i, 0.001 * (System.currentTimeMillis() - t)));
System.out.println(String.format("%d : hdf5 gzip write : %fs", i, 0.001 * (write - start)));
System.out.println(String.format("%d : hdf5 gzip read : %fs", i, 0.001 * (read - write)));
} catch (final InterruptedException | ExecutionException e) {
fail(e.getMessage());
}
Expand Down

0 comments on commit cd1fbb9

Please sign in to comment.