Skip to content

Commit

Permalink
feat!: use readers only unless writers are necessary
Browse files Browse the repository at this point in the history
When writers are necessary, they are attempted to be opened lazily, and cached. H5 backend has slightly different behavior.
  • Loading branch information
cmhulbert committed Aug 31, 2023
1 parent 9914301 commit 4b692d2
Show file tree
Hide file tree
Showing 6 changed files with 33 additions and 59 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -406,10 +406,11 @@ public BooleanProperty discoveryIsActive() {

public FragmentSegmentAssignmentState assignments() throws IOException {

if (getContainer().isReadOnly())
throw new N5ReadOnlyException();

final var writer = getContainer().getWriter();

if (writer == null) {
throw new N5ReadOnlyException();
}
return N5Helpers.assignments(writer, getDatasetPath());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.janelia.saalfeldlab.fx.Tasks;
import org.janelia.saalfeldlab.fx.ui.ObjectField;
import org.janelia.saalfeldlab.n5.N5Reader;
import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader;
import org.janelia.saalfeldlab.paintera.Paintera;
import org.janelia.saalfeldlab.paintera.PainteraConfigYaml;
import org.janelia.saalfeldlab.paintera.state.metadata.N5ContainerState;
Expand Down Expand Up @@ -157,21 +158,6 @@ private Optional<N5Reader> openN5Reader(final String url) {
return Optional.empty();
}

private static boolean isN5Container(final String pathToDirectory) {

try {
final boolean wasCached = Paintera.getN5Factory().getFromCache(pathToDirectory) != null;
final var reader = Paintera.getN5Factory().openReaderOrNull(pathToDirectory);
boolean openableAsN5 = reader != null;
if (!wasCached) {
reader.close();
}
return openableAsN5;
} catch (Exception e) {
return false;
}
}

private void updateFromFileChooser(final File initialDirectory, final Window owner) {

final FileChooser fileChooser = new FileChooser();
Expand Down Expand Up @@ -216,11 +202,12 @@ private void selectionChanged(ObservableValue<? extends String> obs, String oldS
invoke(() -> this.isOpeningContainer.set(true));
final var newContainerState = Optional.ofNullable(n5ContainerStateCache.get(newSelection)).orElseGet(() -> {

var container = Optional.<N5Reader>
ofNullable(Paintera.getN5Factory().openWriterOrNull(newSelection))
.orElseGet(() -> Paintera.getN5Factory().openReaderOrNull(newSelection));

var container = Paintera.getN5Factory().openReaderOrNull(newSelection);
if (container == null) return null;
if (container instanceof N5HDF5Reader) {
container.close();
container = Paintera.getN5Factory().openWriterElseOpenReader(newSelection);
}
return new N5ContainerState(container);
});
if (newContainerState == null)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ class N5FactoryWithCache : N5Factory() {
private val writerCache = HashMap<String, N5Writer>()
private val readerCache = HashMap<String, N5Reader>()
override fun openReader(uri: String): N5Reader {
return getCachedReader(uri) ?: super.openReader(uri).let {
return getFromReaderCache(uri) ?: getFromWriterCache(uri) ?: super.openReader(uri).let {
if (containerIsReadable(it)) {
readerCache[uri] = it
it
Expand All @@ -69,10 +69,10 @@ class N5FactoryWithCache : N5Factory() {
}

override fun openWriter(uri: String): N5Writer {
return getCachedWriter(uri) ?: openAndCacheExistingN5Writer(uri)
return getFromWriterCache(uri) ?: openAndCacheExistingN5Writer(uri)
}
fun createWriter(uri: String): N5Writer {
return getCachedWriter(uri) ?: createAndCacheN5Writer(uri)
return getFromWriterCache(uri) ?: createAndCacheN5Writer(uri)
}

fun openWriterOrNull(uri : String) : N5Writer? = try {
Expand Down Expand Up @@ -126,7 +126,7 @@ class N5FactoryWithCache : N5Factory() {
* @param uri to check the cached reader for
* @return the cached reader, if it exists
*/
private fun getCachedReader(uri: String): N5Reader? {
private fun getFromReaderCache(uri: String): N5Reader? {
synchronized(readerCache) {
readerCache[uri]?.also { reader -> if (!containerIsReadable(reader)) clearKey(uri) }
return readerCache[uri]
Expand All @@ -145,15 +145,15 @@ class N5FactoryWithCache : N5Factory() {
* @param uri to check the cached writer for
* @return the cached writer, if it exists and is valid
*/
private fun getCachedWriter(uri: String): N5Writer? {
private fun getFromWriterCache(uri: String): N5Writer? {
synchronized(writerCache) {
writerCache[uri]?.also { writer -> if (!containerIsWritable(writer)) clearKey(uri) }
return writerCache[uri]
}
}

private fun openAndCacheExistingN5Writer(uri: String): N5Writer {
val readerWasCached = getCachedReader(uri) != null
val readerWasCached = getFromReaderCache(uri) != null

val reader = openReader(uri)
if (!containerIsReadable(reader))
Expand All @@ -171,8 +171,10 @@ class N5FactoryWithCache : N5Factory() {
val n5Writer = super.openWriter(uri)
/* See if we have write permissions before we declare success */
n5Writer.setAttribute("/", N5Reader.VERSION_KEY, n5Writer.version.toString())
if (readerCache[uri] == null) {
readerCache[uri] = n5Writer
}
writerCache[uri] = n5Writer
readerCache[uri] = n5Writer
return n5Writer
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ interface MetadataState {
var unit: String
var reader: N5Reader

var writer: N5Writer?
val writer: N5Writer?
var group: String
val dataset: String
get() = group
Expand Down Expand Up @@ -71,14 +71,12 @@ interface MetadataState {
target.resolution = source.resolution.copyOf()
target.translation = source.translation.copyOf()
target.unit = source.unit
target.reader = source.reader
target.writer = source.writer
target.group = source.group
}
}
}

open class SingleScaleMetadataState constructor(
open class SingleScaleMetadataState(
final override var n5ContainerState: N5ContainerState,
final override val metadata: N5SingleScaleMetadata
) :
Expand All @@ -93,7 +91,9 @@ open class SingleScaleMetadataState constructor(
override var translation = metadata.offset!!
override var unit = metadata.unit()!!
override var reader = n5ContainerState.reader
override var writer = n5ContainerState.writer
override val writer :N5Writer?
get() = n5ContainerState.writer

override var group = metadata.path!!

override fun copy(): SingleScaleMetadataState {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,41 +1,25 @@
package org.janelia.saalfeldlab.paintera.state.metadata

import javafx.beans.property.SimpleObjectProperty
import javafx.beans.value.ObservableValue
import org.apache.commons.lang.builder.HashCodeBuilder
import org.janelia.saalfeldlab.fx.extensions.nonnullVal
import org.janelia.saalfeldlab.fx.extensions.nullableVal
import org.janelia.saalfeldlab.n5.N5Reader
import org.janelia.saalfeldlab.n5.N5Writer
import java.net.URI
import org.janelia.saalfeldlab.paintera.Paintera

data class N5ContainerState(private val n5Container: N5Reader) {
data class N5ContainerState(val reader: N5Reader) {

private val readerProperty: ObservableValue<N5Reader> by lazy { SimpleObjectProperty(n5Container) }
val reader by readerProperty.nonnullVal()

private val writerProperty: ObservableValue<N5Writer?> by lazy { SimpleObjectProperty(n5Container as? N5Writer) }
val writer by writerProperty.nullableVal()

val uri : URI
get() = reader.uri
val writer by lazy {
(reader as? N5Writer) ?: Paintera.n5Factory.openWriterOrNull(uri.toString())
}

val isReadOnly: Boolean
get() = writer == null
val uri by lazy { reader.uri!! }

override fun equals(other: Any?): Boolean {
return if (other is N5ContainerState) {
/* Equal if we are the same url, and we both either have a writer, or have no writer. */
uri == other.uri && ((writer == null) == (other.writer == null))
uri == other.uri
} else {
super.equals(other)
}
}

override fun hashCode(): Int {
val builder = HashCodeBuilder()
.append(reader.uri)
.append(writer?.uri ?: 0)
return builder.toHashCode()
}
override fun hashCode() = HashCodeBuilder().append(uri).toHashCode()
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import java.lang.reflect.Type
class N5BackendRaw<D, T>(@JvmField val metadataState: MetadataState) : AbstractN5BackendRaw<D, T>
where D : NativeType<D>, D : RealType<D>, T : AbstractVolatileRealType<D, T>, T : NativeType<T> {

override val container = metadataState.writer ?: metadataState.reader
override val container = metadataState.reader
override val dataset = metadataState.dataset

override fun createSource(queue: SharedQueue, priority: Int, name: String): DataSource<D, T> {
Expand Down

0 comments on commit 4b692d2

Please sign in to comment.