From ca9dfc6ec10a42244306fa0ee2c47e69ececba8e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 29 Aug 2023 09:06:38 -0400 Subject: [PATCH 1/7] feat: add 1.0 back to default scales --- .../janelia/saalfeldlab/paintera/config/ScreenScalesConfig.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/config/ScreenScalesConfig.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/config/ScreenScalesConfig.kt index 2d81502d0..8af420afb 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/config/ScreenScalesConfig.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/config/ScreenScalesConfig.kt @@ -55,7 +55,7 @@ class ScreenScalesConfig @JvmOverloads constructor(vararg initialScales: Double companion object { - private val DEFAULT_SCREEN_SCALES = doubleArrayOf(0.5, 0.25, 0.125) + private val DEFAULT_SCREEN_SCALES = doubleArrayOf(1.0, 0.5, 0.25, 0.125) @JvmStatic fun defaultScreenScalesCopy(): DoubleArray { From 9329876227fa52e19f17e5efb9f06eb7231b99f0 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 29 Aug 2023 16:22:43 -0400 Subject: [PATCH 2/7] feat: cache by default --- src/main/kotlin/org/janelia/saalfeldlab/paintera/Paintera.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/Paintera.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/Paintera.kt index e39e40d43..1477962cf 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/Paintera.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/Paintera.kt @@ -227,7 +227,7 @@ class Paintera : Application() { companion object { @JvmStatic - val n5Factory = N5FactoryWithCache() + val n5Factory = N5FactoryWithCache().apply { cacheAttributes(true) } private val LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()) From d6bf6df0c0c28d6b4943cbcb6db27366c01df6ae Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 29 Aug 2023 16:22:54 -0400 Subject: [PATCH 3/7] feat: streamline n5 container caching, and separate open vs. create writers where possible --- .../paintera/PainteraCommandLineArgs.java | 4 +- .../opendialog/menu/n5/N5FactoryOpener.java | 93 +------- .../janelia/saalfeldlab/util/n5/N5Data.java | 2 +- .../util/n5/universe/N5FactoryWithCache.java | 116 ---------- .../util/n5/universe/N5FactoryWithCache.kt | 212 ++++++++++++++++++ .../paintera/PainteraMainWindow.kt | 2 +- .../saalfeldlab/paintera/data/n5/N5Adapter.kt | 9 +- .../paintera/data/n5/N5HDF5WriterAdapter.kt | 16 +- .../paintera/state/metadata/MetadataState.kt | 11 +- .../ui/dialogs/create/CreateDataset.kt | 3 +- .../janelia/saalfeldlab/util/n5/N5Helpers.kt | 121 ++-------- 11 files changed, 264 insertions(+), 325 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.java create mode 100644 src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt diff --git a/src/main/java/org/janelia/saalfeldlab/paintera/PainteraCommandLineArgs.java b/src/main/java/org/janelia/saalfeldlab/paintera/PainteraCommandLineArgs.java index bea216693..8f3a907c3 100644 --- a/src/main/java/org/janelia/saalfeldlab/paintera/PainteraCommandLineArgs.java +++ b/src/main/java/org/janelia/saalfeldlab/paintera/PainteraCommandLineArgs.java @@ -76,8 +76,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.janelia.saalfeldlab.util.n5.N5Helpers.getReaderOrGetWriterIfExistsAndWritable; - @Command(name = "Paintera", showDefaultValues = true, resourceBundle = "org.janelia.saalfeldlab.paintera.PainteraCommandLineArgs", usageHelpWidth = 120, parameterListHeading = "%n@|bold,underline Parameters|@:%n", optionListHeading = "%n@|bold,underline Options|@:%n") @@ -649,7 +647,7 @@ private void addToViewer(final PainteraBaseView viewer, final Supplier p for (final String container : containers) { LOG.debug("Adding datasets for container {}", container); - N5Reader n5Container = getReaderOrGetWriterIfExistsAndWritable(container); + N5Reader n5Container = Paintera.getN5Factory().openWriterElseOpenReader(container); final Predicate datasetFilter = options.useDataset(); final ExecutorService es = getDiscoveryExecutorService(); diff --git a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java index 0af2bb1ca..b018ce0f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java +++ b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java @@ -12,9 +12,7 @@ import javafx.beans.value.ObservableValue; import javafx.event.ActionEvent; import javafx.event.EventHandler; -import javafx.scene.control.Alert; import javafx.scene.control.MenuButton; -import javafx.scene.control.TextArea; import javafx.scene.control.TextField; import javafx.scene.control.Tooltip; import javafx.scene.input.KeyCode; @@ -25,12 +23,9 @@ import org.janelia.saalfeldlab.fx.Tasks; import org.janelia.saalfeldlab.fx.ui.ObjectField; import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; import org.janelia.saalfeldlab.paintera.Paintera; import org.janelia.saalfeldlab.paintera.PainteraConfigYaml; import org.janelia.saalfeldlab.paintera.state.metadata.N5ContainerState; -import org.janelia.saalfeldlab.paintera.ui.PainteraAlerts; import org.janelia.saalfeldlab.util.PainteraCache; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -162,31 +157,15 @@ private Optional openN5Reader(final String url) { return Optional.empty(); } - /** - * Open {@code url} as an N5Writer if possible, else empty. - * - * @param url location of the container we wish to open as an N5Writer. - * @return N5Writer of {@code url} if valid N5 container which we can write to; else empty - */ - private Optional openN5Writer(final String url) { - - try { - final var writer = Paintera.getN5Factory().openWriter(url); - LOG.debug("{} was opened as an N5Writer.", url); - return Optional.of(writer); - } catch (Exception e) { - LOG.debug("{} cannot be opened as an N5Writer.", url); - } - return Optional.empty(); - } - private static boolean isN5Container(final String pathToDirectory) { try { - final var reader = Paintera.getN5Factory().openReader(pathToDirectory); + final boolean wasCached = Paintera.getN5Factory().getFromCache(pathToDirectory) != null; + final var reader = Paintera.getN5Factory().openReaderOrNull(pathToDirectory); boolean openableAsN5 = reader != null; - if (reader instanceof N5HDF5Reader) + if (!wasCached) { reader.close(); + } return openableAsN5; } catch (Exception e) { return false; @@ -214,8 +193,7 @@ private void updateFromDirectoryChooser(final File initialDirectory, final Windo .ifPresent(directoryChooser::setInitialDirectory); Optional.ofNullable(directoryChooser.showDialog(ownerWindow)).ifPresent(updatedRoot -> { LOG.debug("Updating root to {} (was {})", updatedRoot, selectionProperty.get()); - - if (fileOpenableAsN5(updatedRoot)) { + if (Paintera.getN5Factory().openReaderOrNull(updatedRoot.getAbsolutePath()) != null) { // set null first to make sure that selectionProperty will be invalidated even if directory is the same String updatedAbsPath = updatedRoot.getAbsolutePath(); if (updatedAbsPath.equals(selectionProperty.get())) { @@ -224,26 +202,6 @@ private void updateFromDirectoryChooser(final File initialDirectory, final Windo selectionProperty.set(updatedAbsPath); } }); - - } - - private boolean fileOpenableAsN5(File updatedRoot) { - - if (updatedRoot == null) { - /* They probably just canceled out of browse; just silently return false; */ - return false; - } else if (!isN5Container(updatedRoot.getAbsolutePath())) { - final Alert alert = PainteraAlerts.alert(Alert.AlertType.INFORMATION); - alert.setHeaderText("Selected path cannot be opened as an N5 container."); - final TextArea ta = new TextArea("The selected path is not a valid N5 container\n\n" + updatedRoot.getAbsolutePath() + "\n\n" + - "A valid N5 container is a directory that contains a file attributes.json with a key \"n5\"."); //FIXME meta need a more accurate message - ta.setEditable(false); - ta.setWrapText(true); - alert.getDialogPane().setContent(ta); - alert.show(); - return false; - } - return true; } private void selectionChanged(ObservableValue obs, String oldSelection, String newSelection) { @@ -258,44 +216,11 @@ private void selectionChanged(ObservableValue obs, String oldS invoke(() -> this.isOpeningContainer.set(true)); final var newContainerState = Optional.ofNullable(n5ContainerStateCache.get(newSelection)).orElseGet(() -> { - /* This particular shortcut is if the N5Reader/Writer is cached, but from - * deserialization, not from open source */ - N5Reader initialReader = Paintera.getN5Factory().getFromCache(newSelection); - if (initialReader != null) return new N5ContainerState(initialReader); - - /* Ok we don't want to do the writer first, even though it means we need to create a separate writer in the case that it can have both. - * This is because if the path provided doesn't currently contain a writer, but it has permissions to create a writer, it will do so. - * This means that if there is no N5 container, it will create one. - * - * In this case, we only want to create a writer if there is already an N5 container. To check, we create a reader first, and see if it - * exists. */ - final var optReader = openN5Reader(newSelection); - if (optReader.isEmpty()) { - return null; - } else { - initialReader = optReader.get(); - } - - /* Another wrinkle though; For HDF5, you can't open a reader and a writer at the same time, even with permission. So - * Now that we know the container actually exists, we need to check if it's HDF5. If it is, we need to close the reader, - * Then try to open a writer. If that isn't possible, we need to re-open a reader. */ - - final N5Reader container; - if (initialReader instanceof N5HDF5Reader) { /* Check if we are an HDF5 container*/ - initialReader.close(); - final var optWriter = openN5Writer(newSelection); - if (optWriter.isEmpty()) { /* if we don't have a writer, re-open the reader */ - container = openN5Reader(newSelection).orElseThrow(() -> - new RuntimeException("HDF5 container at " + newSelection - + " was initially opened as a reader, but failed after attempt to open as a writer") - ); - } else { /* if we have the writer, use it as a reader also */ - container = optWriter.get(); - } - } else { - container = openN5Writer(newSelection).map(N5Reader.class::cast).orElse(initialReader); - } + var container = Optional. + ofNullable(Paintera.getN5Factory().openWriterOrNull(newSelection)) + .orElseGet(() -> Paintera.getN5Factory().openReaderOrNull(newSelection)); + if (container == null) return null; return new N5ContainerState(container); }); if (newContainerState == null) diff --git a/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java b/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java index 51bf4d409..086577340 100644 --- a/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java +++ b/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java @@ -795,7 +795,7 @@ public static void createEmptyLabelDataset( final Map pd = new HashMap<>(); pd.put("type", "label"); - final N5Writer n5 = Paintera.getN5Factory().openWriter(container); + final N5Writer n5 = Paintera.getN5Factory().createWriter(container); final String uniqueLabelsGroup = String.format("%s/unique-labels", group); if (!ignoreExisiting && n5.datasetExists(group)) diff --git a/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.java b/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.java deleted file mode 100644 index f83ac39c3..000000000 --- a/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright (c) 2017-2021, Saalfeld lab, HHMI Janelia - * All rights reserved. - *

- * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

- * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - *

- * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.util.n5.universe; - -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.universe.N5Factory; - -import java.io.IOException; -import java.util.HashMap; - -/** - * Factory for various N5 readers and writers. Implementation specific - * parameters can be provided to the factory instance and will be used when - * such implementations are generated and ignored otherwise. Reasonable - * defaults are provided. - * - * @author Stephan Saalfeld - * @author John Bogovic - * @author Igor Pisarev - */ -public class N5FactoryWithCache extends N5Factory { - - private final HashMap writerCache = new HashMap<>(); - private final HashMap readerCache = new HashMap<>(); - - @Override public N5Reader openReader(String url) { - - /* Check if writer is valid (it may have been closed by someone) */ - final N5Reader cachedContainer = readerCache.get(url); - if (cachedContainer != null) { - try { - cachedContainer.getVersion(); - } catch (Exception e) { - readerCache.remove(url).close(); - } - } else { - final N5Reader reader = super.openReader(url); - if (reader.getAttribute("/", N5Reader.VERSION_KEY, String.class) != null) { - readerCache.put(url, reader); - } - } - return readerCache.get(url); - } - - @Override public N5Writer openWriter(String url) { - - /* Check if writer is valid (it may have been closed by someone) */ - final N5Writer cachedContainer = writerCache.get(url); - if (cachedContainer != null) { - try { - /* See if its open, and we still have write permissions */ - cachedContainer.setAttribute("/", N5Reader.VERSION_KEY, cachedContainer.getVersion().toString()); - } catch (Exception e) { - writerCache.remove(url).close(); - if (readerCache.get(url) == cachedContainer) { - readerCache.remove(url); - } - } - } else { - final N5Writer n5Writer = super.openWriter(url); - /* See if we have write permissions before we declare success */ - n5Writer.setAttribute("/", N5Reader.VERSION_KEY, n5Writer.getVersion()); - writerCache.put(url, n5Writer); - if (readerCache.get(url) != null) { - readerCache.remove(url).close(); - } - readerCache.put(url, n5Writer); - } - return writerCache.get(url); - } - - public void clearKey(String url) { - - final var writer = writerCache.remove(url); - if (writer != null) - writer.close(); - - final var reader = readerCache.remove(url); - if (reader != null) - reader.close(); - } - - public void clearCache() { - writerCache.clear(); - readerCache.clear(); - } - - public N5Reader getFromCache(String url) { - return readerCache.get(url); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt b/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt new file mode 100644 index 000000000..99c60e50d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt @@ -0,0 +1,212 @@ +/** + * Copyright (c) 2017-2021, Saalfeld lab, HHMI Janelia + * All rights reserved. + * + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.util.n5.universe + +import org.janelia.saalfeldlab.n5.N5Exception +import org.janelia.saalfeldlab.n5.N5Reader +import org.janelia.saalfeldlab.n5.N5URI +import org.janelia.saalfeldlab.n5.N5Writer +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader +import org.janelia.saalfeldlab.n5.universe.N5Factory +import org.slf4j.LoggerFactory +import java.lang.invoke.MethodHandles + +/** + * Factory for various N5 readers and writers. Implementation specific + * parameters can be provided to the factory instance and will be used when + * such implementations are generated and ignored otherwise. Reasonable + * defaults are provided. + * + * @author Stephan Saalfeld + * @author John Bogovic + * @author Igor Pisarev + */ +class N5FactoryWithCache : N5Factory() { + + companion object { + private val LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()) + } + + private val writerCache = HashMap() + private val readerCache = HashMap() + override fun openReader(uri: String): N5Reader { + return getCachedReader(uri) ?: super.openReader(uri).let { + if (containerIsReadable(it)) { + readerCache[uri] = it + it + } else { + throw N5ContainerDoesntExist(uri) + } + } + } + + override fun openWriter(uri: String): N5Writer { + return getCachedWriter(uri) ?: openAndCacheExistingN5Writer(uri) + } + fun createWriter(uri: String): N5Writer { + return getCachedWriter(uri) ?: createAndCacheN5Writer(uri) + } + + fun openWriterOrNull(uri : String) : N5Writer? = try { + openWriter(uri) + } catch (e : Exception) { + null + } + + fun openReaderOrNull(uri : String) : N5Reader? = try { + openReader(uri) + } catch (e : Exception) { + null + } + + fun openWriterElseOpenReader(uri : String) = try { + openWriterOrNull(uri) ?: openReader(uri) + } catch (e : N5Exception) { + if (e.message?.startsWith("No container exists at ") == true) + throw N5ContainerDoesntExist(uri, e) + else throw e + } + + private fun containerIsReadable(reader: N5Reader) = try { + reader.getAttribute("/", "/", String::class.java) + true + } catch (e : Exception ) { + false + } + + private fun containerIsWritable(writer: N5Writer) = try { + val version = writer.getAttribute("/", N5Writer.VERSION_KEY, String::class.java) + if (version == null) + false + else { + writer.setAttribute("/", N5Writer.VERSION_KEY, version) + true + } + } catch (e : Exception ) { + false + } + + /** + * If a cached reader is present, and is valid (i.e., can actually read), return the reader. + * + * + * This has the side effect that if the cached reader is not valid (e.g. it has been closed, + * or can otherwise no longer read) then it will be removed from the cache, and null will be returned. + * Note this can case a removal of a writer from the writer cache, in the case that the reader we are + * removing is also a writer that is present in the writer cache. + * + * @param uri to check the cached reader for + * @return the cached reader, if it exists + */ + private fun getCachedReader(uri: String): N5Reader? { + synchronized(readerCache) { + readerCache[uri]?.also { reader -> if (!containerIsReadable(reader)) clearKey(uri) } + return readerCache[uri] + } + } + + /** + * If a cached writer is present, and is valid (i.e., can actually write), return the writer. + * + * + * This has the side effect that if the cached writer is not valid (e.g. it has been closed, + * or can otherwise no longer write) then it will be removed from the cache, and null will be returned. + * Since an N5Writer is a valid N5Reader, when discovering that the cached N5Writer is no longer valid, it + * will also be removed from the reader cache, if it's present. + * + * @param uri to check the cached writer for + * @return the cached writer, if it exists and is valid + */ + private fun getCachedWriter(uri: String): N5Writer? { + synchronized(writerCache) { + writerCache[uri]?.also { writer -> if (!containerIsWritable(writer)) clearKey(uri) } + return writerCache[uri] + } + } + + private fun openAndCacheExistingN5Writer(uri: String): N5Writer { + val readerWasCached = getCachedReader(uri) != null + + val reader = openReader(uri) + if (!containerIsReadable(reader)) + throw N5ContainerDoesntExist(uri) + + /* If we opened explicitly to check, then close now. + * If we are HDF5, we need to close before opening */ + if (!readerWasCached || reader is N5HDF5Reader) + reader.close() + + return createAndCacheN5Writer(uri) + } + + private fun createAndCacheN5Writer(uri: String): N5Writer { + val n5Writer = super.openWriter(uri) + /* See if we have write permissions before we declare success */ + n5Writer.setAttribute("/", N5Reader.VERSION_KEY, n5Writer.version.toString()) + writerCache[uri] = n5Writer + readerCache[uri] = n5Writer + return n5Writer + } + + fun clearKey(uri: String) { + val writer = writerCache.remove(uri) + writer?.close() + val reader = readerCache.remove(uri) + reader?.close() + } + + fun clearCache() { + writerCache.clear() + readerCache.clear() + } + + fun getFromCache(uri: String): N5Reader? { + val cachedWriter = writerCache[uri] + return cachedWriter ?: readerCache[uri] + } +} + +class N5DatasetDoesntExist : N5Exception { + + companion object { + private fun displayDataset(dataset: String) = N5URI.normalizeGroupPath(dataset).ifEmpty { "/" } + } + + constructor(uri : String, dataset: String) : super("Dataset \"${displayDataset(dataset)}\" not found in container $uri") + constructor(uri: String, dataset : String, cause: Throwable) : super("Dataset \"${displayDataset(dataset)}\" not found in container $uri", cause) +} + + +class N5ContainerDoesntExist : N5Exception { + + constructor(location: String) : super("Cannot Open $location") + constructor(location: String, cause: Throwable) : super("Cannot Open $location", cause) +} diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt index 054cca496..c5d272221 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/PainteraMainWindow.kt @@ -174,7 +174,7 @@ class PainteraMainWindow(val gateway: PainteraGateway = PainteraGateway()) { .setPrettyPrinting() Paintera.n5Factory.gsonBuilder(builder) Paintera.n5Factory.clearKey(projectDirectory.actualDirectory.absolutePath) - Paintera.n5Factory.openWriter(projectDirectory.actualDirectory.absolutePath).use { + Paintera.n5Factory.createWriter(projectDirectory.actualDirectory.absolutePath).use { it.setAttribute("/", PAINTERA_KEY, this) } if (notify) { diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5Adapter.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5Adapter.kt index 7c4520b10..39e40918f 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5Adapter.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5Adapter.kt @@ -14,7 +14,6 @@ import org.janelia.saalfeldlab.paintera.Paintera.Companion.n5Factory import org.janelia.saalfeldlab.paintera.serialization.GsonExtensions import org.janelia.saalfeldlab.paintera.serialization.StatefulSerializer import org.janelia.saalfeldlab.paintera.state.SourceState -import org.janelia.saalfeldlab.util.n5.N5Helpers.getWriterIfN5ContainerExists import org.scijava.plugin.Plugin import java.lang.reflect.Type import java.util.function.IntFunction @@ -83,7 +82,7 @@ class N5FSWriterAdapter : StatefulSerializer.SerializerAndDeserializer, dependencyFromIndex: IntFunction>?, ): JsonDeserializer = N5ReaderDeserializer(projectDirectory) { - getWriterIfN5ContainerExists(it) as N5FSWriter + n5Factory.openWriterElseOpenReader(it) as N5FSWriter } override fun getTargetClass() = N5FSWriter::class.java @@ -124,7 +123,7 @@ class N5GoogleCloudWriterAdapter : projectDirectory: Supplier, dependencyFromIndex: IntFunction>?, ): JsonDeserializer = N5ReaderDeserializer(projectDirectory) { - getWriterIfN5ContainerExists(it) as N5GoogleCloudStorageWriter + n5Factory.openWriterElseOpenReader(it) as N5GoogleCloudStorageWriter } override fun getTargetClass() = N5GoogleCloudStorageWriter::class.java @@ -164,7 +163,7 @@ class N5AmazonS3WriterAdapter : projectDirectory: Supplier, dependencyFromIndex: IntFunction>?, ): JsonDeserializer = N5ReaderDeserializer(projectDirectory) { - getWriterIfN5ContainerExists(it) as N5AmazonS3Writer + n5Factory.openWriterElseOpenReader(it) as N5AmazonS3Writer } override fun getTargetClass() = N5AmazonS3Writer::class.java @@ -204,7 +203,7 @@ class N5ZarrWriterAdapter : StatefulSerializer.SerializerAndDeserializer, dependencyFromIndex: IntFunction>?, ): JsonDeserializer = N5ReaderDeserializer(projectDirectory) { - getWriterIfN5ContainerExists(it) as N5ZarrWriter + n5Factory.openWriterElseOpenReader(it) as N5ZarrWriter } override fun getTargetClass() = N5ZarrWriter::class.java diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5HDF5WriterAdapter.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5HDF5WriterAdapter.kt index 953b3b6e0..5c3ebaed7 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5HDF5WriterAdapter.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/data/n5/N5HDF5WriterAdapter.kt @@ -7,8 +7,6 @@ import org.janelia.saalfeldlab.paintera.Paintera.Companion.n5Factory import org.janelia.saalfeldlab.paintera.serialization.GsonExtensions import org.janelia.saalfeldlab.paintera.serialization.StatefulSerializer import org.janelia.saalfeldlab.paintera.state.SourceState -import org.janelia.saalfeldlab.util.n5.N5Helpers.getReaderOrGetWriterIfExistsAndWritable -import org.janelia.saalfeldlab.util.n5.N5Helpers.getWriterIfN5ContainerExists import org.scijava.plugin.Plugin import java.lang.reflect.Type import java.nio.file.Path @@ -84,10 +82,14 @@ class N5HDF5ReaderAdapter : StatefulSerializer.SerializerAndDeserializer, dependencyFromIndex: IntFunction>?, - ): JsonDeserializer = HDF5Deserializer(projectDirectory) { file, overrideBlockSize, defaultBlockSize -> - n5Factory.hdf5OverrideBlockSize(overrideBlockSize) - n5Factory.hdf5DefaultBlockSize(*(defaultBlockSize ?: intArrayOf())) - (getReaderOrGetWriterIfExistsAndWritable(file) as? N5HDF5Reader) ?: throw hdf5OpenError(file) + ): JsonDeserializer = HDF5Deserializer(projectDirectory) { file, overrideBlockSize, blockSize -> + with(n5Factory) { + if (overrideBlockSize && blockSize != null) { + hdf5OverrideBlockSize(true) + hdf5DefaultBlockSize(*blockSize) + } + (openWriterElseOpenReader(file)) as? N5HDF5Reader ?: throw hdf5OpenError(file) + } } override fun getTargetClass() = N5HDF5Reader::class.java @@ -108,7 +110,7 @@ class N5HDF5WriterAdapter : StatefulSerializer.SerializerAndDeserializer = HDF5Deserializer(projectDirectory) { file, _, defaultBlockSize -> //FIXME this should be temporary! we should generify these special adaptors if possible. n5Factory.hdf5DefaultBlockSize(*(defaultBlockSize ?: intArrayOf())) - (getWriterIfN5ContainerExists(file) as? N5HDF5Writer) ?: throw hdf5OpenError(file) + (n5Factory.openWriterElseOpenReader(file) as? N5HDF5Writer) ?: throw hdf5OpenError(file) } override fun getTargetClass() = N5HDF5Writer::class.java diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt index 99613bcee..5c0f829b9 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt @@ -14,11 +14,11 @@ import org.janelia.saalfeldlab.n5.universe.metadata.N5Metadata import org.janelia.saalfeldlab.n5.universe.metadata.N5SingleScaleMetadata import org.janelia.saalfeldlab.n5.universe.metadata.N5SpatialDatasetMetadata import org.janelia.saalfeldlab.n5.universe.metadata.SpatialMultiscaleMetadata +import org.janelia.saalfeldlab.paintera.Paintera import org.janelia.saalfeldlab.paintera.state.metadata.MetadataState.Companion.isLabel import org.janelia.saalfeldlab.util.n5.ImagesWithTransform import org.janelia.saalfeldlab.util.n5.N5Data import org.janelia.saalfeldlab.util.n5.N5Helpers -import org.janelia.saalfeldlab.util.n5.N5Helpers.getReaderOrGetWriterIfExistsAndWritable import org.janelia.saalfeldlab.util.n5.metadata.N5PainteraDataMultiScaleGroup import java.util.Optional @@ -229,7 +229,10 @@ class MetadataUtils { @JvmStatic fun createMetadataState(n5containerAndDataset: String): Optional { - val reader = getReaderOrGetWriterIfExistsAndWritable(n5containerAndDataset) ?: return Optional.empty() + + val reader = with(Paintera.n5Factory) { + openWriterOrNull(n5containerAndDataset) ?: openReaderOrNull(n5containerAndDataset)?: return Optional.empty() + } val n5ContainerState = N5ContainerState(reader) return N5Helpers.parseMetadata(reader).map { treeNode -> @@ -243,7 +246,9 @@ class MetadataUtils { @JvmStatic fun createMetadataState(n5container: String, dataset: String?): Optional { - val reader = getReaderOrGetWriterIfExistsAndWritable(n5container) ?: return Optional.empty() + val reader = with(Paintera.n5Factory) { + openWriterOrNull(n5container) ?: openReaderOrNull(n5container)?: return Optional.empty() + } val n5ContainerState = N5ContainerState(reader) val metadataRoot = N5Helpers.parseMetadata(reader) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt index 563bcf72e..9c81c701d 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/ui/dialogs/create/CreateDataset.kt @@ -210,8 +210,7 @@ class CreateDataset(private val currentSource: Source<*>?, vararg allSources: So scaleLevels.stream().mapToInt { it.maxNumEntries() }.toArray() ) - val path = Path.of(container).toFile().canonicalPath - val writer = n5Factory.openWriter(path) + val writer = n5Factory.openWriter(container) N5Helpers.parseMetadata(writer, true).ifPresent { _ -> val containerState = N5ContainerState(writer) createMetadataState(containerState, dataset).ifPresent { metadataStateProp.set(it) } diff --git a/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt b/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt index f61ff7c12..c8ffc351f 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt @@ -6,11 +6,7 @@ import com.google.gson.JsonObject import javafx.beans.property.BooleanProperty import javafx.beans.value.ChangeListener import javafx.event.EventHandler -import javafx.scene.control.Button -import javafx.scene.control.ButtonType -import javafx.scene.control.Label -import javafx.scene.control.TextArea -import javafx.scene.control.TextField +import javafx.scene.control.* import javafx.scene.layout.HBox import javafx.scene.layout.Priority import javafx.scene.layout.VBox @@ -24,8 +20,6 @@ import org.janelia.saalfeldlab.labels.blocks.LabelBlockLookup import org.janelia.saalfeldlab.labels.blocks.LabelBlockLookupAdapter import org.janelia.saalfeldlab.labels.blocks.n5.LabelBlockLookupFromN5Relative import org.janelia.saalfeldlab.n5.DatasetAttributes -import org.janelia.saalfeldlab.n5.N5Exception -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException import org.janelia.saalfeldlab.n5.N5Reader import org.janelia.saalfeldlab.n5.N5Writer import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader @@ -51,6 +45,7 @@ import org.janelia.saalfeldlab.util.NamedThreadFactory import org.janelia.saalfeldlab.util.n5.metadata.N5PainteraDataMultiScaleMetadata.PainteraDataMultiScaleParser import org.janelia.saalfeldlab.util.n5.metadata.N5PainteraLabelMultiScaleGroup.PainteraLabelMultiScaleParser import org.janelia.saalfeldlab.util.n5.metadata.N5PainteraRawMultiScaleGroup.PainteraRawMultiScaleParser +import org.janelia.saalfeldlab.util.n5.universe.N5ContainerDoesntExist import org.slf4j.LoggerFactory import java.io.IOException import java.lang.invoke.MethodHandles @@ -788,82 +783,6 @@ object N5Helpers { class MaxIDNotSpecified(message: String) : PainteraException(message) class NotAPainteraDataset(val container: N5Reader, val group: String) : PainteraException(String.format("Group %s in container %s is not a Paintera dataset.", group, container)) - /** - * If an n5 container exists at [uri], return it as [N5Writer] if possible, and [N5Reader] if not. - * - * @param uri the location of the n5 container - * @return [N5Writer] or [N5Reader] if container exists - */ - @JvmStatic - fun getReaderOrGetWriterIfExistsAndWritable(uri: String): N5Reader? { - val cachedContainer = getReaderOrWriterIfCached(uri) - return cachedContainer ?: openReaderOrGetWriterIfExistsAndWritable(uri) - } - - /** - * If an n5 container exists at [uri], return it as [N5Writer] if possible. - * - * @param uri the location of the n5 container - * @return [N5Writer] if container exists and is openable as a writer. - */ - @JvmStatic - fun getWriterIfN5ContainerExists(uri: String): N5Writer? { - return getReaderOrGetWriterIfExistsAndWritable(uri) as? N5Writer - } - - /** - * Retrieves a reader or writer for the given container if it exists. - * - * If the reader is successfully opened, the container must exist, so we try to open a writer. - * This is to ensure that an N5 container isn't created by opening as a writer if one doesn't exist. - * If opening a writer is not possible it falls back to again as a reader. - * - * @param container the path to the container - * @return a reader or writer as N5Reader, or null if the container does not exist - */ - private fun openReaderOrGetWriterIfExistsAndWritable(container: String) = try { - n5Factory.openReader(container)?.let { - var reader: N5Reader? = it - if (it is N5HDF5Reader) { - it.close() - reader = null - } - try { - n5Factory.openWriter(container) - } catch (_: Exception) { - reader ?: n5Factory.openReader(container) - } - } - } catch (e: N5IOException) { - if (e.message?.startsWith("No container exists at") == true) { - throw N5ContainerDoesntExist(container, e) - } else { - LOG.debug("Cannot Open Reader", e) - null - } - } catch (e: Exception) { - LOG.debug("Cannot Open Container At: $container", e) - null - } - - /** - * If there is a cached N5Reader for [container] than try to open a writer (also may be cached), - * or fallback to the cached reader - * - * @param container The path to the N5 container. - * @return The N5Reader instance. - */ - private fun getReaderOrWriterIfCached(container: String): N5Reader? { - val reader: N5Reader? = n5Factory.getFromCache(container)?.let { - try { - n5Factory.openWriter(container) - } catch (_: Exception) { - it - } - } - return reader - } - private const val URI = "uri" internal fun N5Reader.serializeTo(json: JsonObject) { if (!uri.equals(paintera.projectDirectory.actualDirectory.toURI())) { @@ -881,26 +800,28 @@ object N5Helpers { return getN5ContainerWithRetryPrompt(uri) } - internal fun getN5ContainerWithRetryPrompt(uri : String) : N5Reader { + internal fun getN5ContainerWithRetryPrompt(uri: String): N5Reader { return try { - getReaderOrGetWriterIfExistsAndWritable(uri)!! - } catch (e : N5ContainerDoesntExist) { - promptForNewLocationOrRemove(uri, e) + n5Factory.openWriterElseOpenReader(uri) + } catch (e: N5ContainerDoesntExist) { + promptForNewLocationOrRemove(uri, e, "Container Not Found", + """ + N5 container does not exist at + $uri + + If the container has moved, specify it's new location. + If the container no longer exists, you can attempt to remove this source. + """.trimIndent() + ) } } - internal fun promptForNewLocationOrRemove(uri: String, cause : Throwable) : N5Reader { - var exception : () -> Throwable = { cause } + internal fun promptForNewLocationOrRemove(uri: String, cause: Throwable, header: String? = null, contentText: String? = null): N5Reader { + var exception: () -> Throwable = { cause } val n5Container = AtomicReference() InvokeOnJavaFXApplicationThread.invokeAndWait { PainteraAlerts.confirmation("Accept", "Quit", true).also { alert -> - alert.headerText = "Container Not Found" - alert.contentText = """ - N5 container does not exist at $uri - If the container has moved, specify it's new location. - If the container no longer exists, you can remove this source. - """.trimIndent() - + alert.headerText = header ?: "Error Opening N5 Container" alert.buttonTypes.add(ButtonType.FINISH) (alert.dialogPane.lookupButton(ButtonType.FINISH) as Button).apply { text = "Remove Source" @@ -924,7 +845,7 @@ object N5Helpers { alert.dialogPane.content = VBox().apply { children += HBox().apply { - children += TextArea("Invalid Location:\n\t$uri").also { it.editableProperty().set(false) } + children += TextArea( contentText ?: "Error accessing container at $uri" ).also { it.editableProperty().set(false) } } children += HBox().apply { children += Label("New Location ").also { HBox.setHgrow(it, Priority.NEVER) } @@ -951,10 +872,4 @@ object N5Helpers { constructor(location: String, cause: Throwable) : super("Source expected at:\n$location\nshould be removed", cause) } - - class N5ContainerDoesntExist : N5Exception { - - constructor(location: String) : super("Cannot Open $location") - constructor(location: String, cause: Throwable) : super("Cannot Open $location", cause) - } } From 991430176732641c81d694e3b3375a53b18e926b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 29 Aug 2023 16:22:57 -0400 Subject: [PATCH 4/7] feat: prompt for location if dataset not found --- .../state/label/n5/N5BackendPainteraDataset.kt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/label/n5/N5BackendPainteraDataset.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/label/n5/N5BackendPainteraDataset.kt index a9b675f15..fea0679ee 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/label/n5/N5BackendPainteraDataset.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/label/n5/N5BackendPainteraDataset.kt @@ -28,6 +28,7 @@ import org.janelia.saalfeldlab.paintera.state.metadata.N5ContainerState import org.janelia.saalfeldlab.util.grids.LabelBlockLookupNoBlocks import org.janelia.saalfeldlab.util.n5.N5Helpers import org.janelia.saalfeldlab.util.n5.N5Helpers.serializeTo +import org.janelia.saalfeldlab.util.n5.universe.N5DatasetDoesntExist import org.scijava.plugin.Plugin import org.slf4j.LoggerFactory import java.io.File @@ -230,10 +231,21 @@ class N5BackendPainteraDataset( ): N5BackendPainteraDataset { return with(SerializationKeys) { with(GsonExtensions) { - val container = N5Helpers.deserializeFrom(json.asJsonObject) + var container = N5Helpers.deserializeFrom(json.asJsonObject) val dataset: String = json[DATASET]!! val n5ContainerState = N5ContainerState(container) - val metadataState = MetadataUtils.createMetadataState(n5ContainerState, dataset).nullable!! + var metadataState = MetadataUtils.createMetadataState(n5ContainerState, dataset).nullable + while (metadataState == null) { + container = N5Helpers.promptForNewLocationOrRemove(container.uri.toString(), N5DatasetDoesntExist(container.uri.toString(), dataset), + "Dataset not found", + """ + Expected dataset "${dataset.ifEmpty { "/" }}" not found at + ${container.uri} + """.trimIndent() + ) + val n5ContainerState = N5ContainerState(container) + metadataState = MetadataUtils.createMetadataState(n5ContainerState, dataset).nullable + } N5BackendPainteraDataset( metadataState, projectDirectory, From 4b692d25a12e539977d7e6dfd76822746206746f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 31 Aug 2023 11:13:05 -0400 Subject: [PATCH 5/7] feat!: use readers only unless writers are necessary When writers are necessary, they are attempted to be opened lazily, and cached. H5 backend has slightly different behavior. --- .../menu/n5/GenericBackendDialogN5.java | 7 ++-- .../opendialog/menu/n5/N5FactoryOpener.java | 25 ++++----------- .../util/n5/universe/N5FactoryWithCache.kt | 16 ++++++---- .../paintera/state/metadata/MetadataState.kt | 10 +++--- .../state/metadata/N5ContainerState.kt | 32 +++++-------------- .../paintera/state/raw/n5/N5BackendRaw.kt | 2 +- 6 files changed, 33 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/GenericBackendDialogN5.java b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/GenericBackendDialogN5.java index 0463fb225..15bee0f91 100644 --- a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/GenericBackendDialogN5.java +++ b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/GenericBackendDialogN5.java @@ -406,10 +406,11 @@ public BooleanProperty discoveryIsActive() { public FragmentSegmentAssignmentState assignments() throws IOException { - if (getContainer().isReadOnly()) - throw new N5ReadOnlyException(); - final var writer = getContainer().getWriter(); + + if (writer == null) { + throw new N5ReadOnlyException(); + } return N5Helpers.assignments(writer, getDatasetPath()); } diff --git a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java index b018ce0f8..6767e9794 100644 --- a/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java +++ b/src/main/java/org/janelia/saalfeldlab/paintera/ui/dialogs/opendialog/menu/n5/N5FactoryOpener.java @@ -23,6 +23,7 @@ import org.janelia.saalfeldlab.fx.Tasks; import org.janelia.saalfeldlab.fx.ui.ObjectField; import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader; import org.janelia.saalfeldlab.paintera.Paintera; import org.janelia.saalfeldlab.paintera.PainteraConfigYaml; import org.janelia.saalfeldlab.paintera.state.metadata.N5ContainerState; @@ -157,21 +158,6 @@ private Optional openN5Reader(final String url) { return Optional.empty(); } - private static boolean isN5Container(final String pathToDirectory) { - - try { - final boolean wasCached = Paintera.getN5Factory().getFromCache(pathToDirectory) != null; - final var reader = Paintera.getN5Factory().openReaderOrNull(pathToDirectory); - boolean openableAsN5 = reader != null; - if (!wasCached) { - reader.close(); - } - return openableAsN5; - } catch (Exception e) { - return false; - } - } - private void updateFromFileChooser(final File initialDirectory, final Window owner) { final FileChooser fileChooser = new FileChooser(); @@ -216,11 +202,12 @@ private void selectionChanged(ObservableValue obs, String oldS invoke(() -> this.isOpeningContainer.set(true)); final var newContainerState = Optional.ofNullable(n5ContainerStateCache.get(newSelection)).orElseGet(() -> { - var container = Optional. - ofNullable(Paintera.getN5Factory().openWriterOrNull(newSelection)) - .orElseGet(() -> Paintera.getN5Factory().openReaderOrNull(newSelection)); - + var container = Paintera.getN5Factory().openReaderOrNull(newSelection); if (container == null) return null; + if (container instanceof N5HDF5Reader) { + container.close(); + container = Paintera.getN5Factory().openWriterElseOpenReader(newSelection); + } return new N5ContainerState(container); }); if (newContainerState == null) diff --git a/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt b/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt index 99c60e50d..cf42f2a88 100644 --- a/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt +++ b/src/main/java/org/janelia/saalfeldlab/util/n5/universe/N5FactoryWithCache.kt @@ -58,7 +58,7 @@ class N5FactoryWithCache : N5Factory() { private val writerCache = HashMap() private val readerCache = HashMap() override fun openReader(uri: String): N5Reader { - return getCachedReader(uri) ?: super.openReader(uri).let { + return getFromReaderCache(uri) ?: getFromWriterCache(uri) ?: super.openReader(uri).let { if (containerIsReadable(it)) { readerCache[uri] = it it @@ -69,10 +69,10 @@ class N5FactoryWithCache : N5Factory() { } override fun openWriter(uri: String): N5Writer { - return getCachedWriter(uri) ?: openAndCacheExistingN5Writer(uri) + return getFromWriterCache(uri) ?: openAndCacheExistingN5Writer(uri) } fun createWriter(uri: String): N5Writer { - return getCachedWriter(uri) ?: createAndCacheN5Writer(uri) + return getFromWriterCache(uri) ?: createAndCacheN5Writer(uri) } fun openWriterOrNull(uri : String) : N5Writer? = try { @@ -126,7 +126,7 @@ class N5FactoryWithCache : N5Factory() { * @param uri to check the cached reader for * @return the cached reader, if it exists */ - private fun getCachedReader(uri: String): N5Reader? { + private fun getFromReaderCache(uri: String): N5Reader? { synchronized(readerCache) { readerCache[uri]?.also { reader -> if (!containerIsReadable(reader)) clearKey(uri) } return readerCache[uri] @@ -145,7 +145,7 @@ class N5FactoryWithCache : N5Factory() { * @param uri to check the cached writer for * @return the cached writer, if it exists and is valid */ - private fun getCachedWriter(uri: String): N5Writer? { + private fun getFromWriterCache(uri: String): N5Writer? { synchronized(writerCache) { writerCache[uri]?.also { writer -> if (!containerIsWritable(writer)) clearKey(uri) } return writerCache[uri] @@ -153,7 +153,7 @@ class N5FactoryWithCache : N5Factory() { } private fun openAndCacheExistingN5Writer(uri: String): N5Writer { - val readerWasCached = getCachedReader(uri) != null + val readerWasCached = getFromReaderCache(uri) != null val reader = openReader(uri) if (!containerIsReadable(reader)) @@ -171,8 +171,10 @@ class N5FactoryWithCache : N5Factory() { val n5Writer = super.openWriter(uri) /* See if we have write permissions before we declare success */ n5Writer.setAttribute("/", N5Reader.VERSION_KEY, n5Writer.version.toString()) + if (readerCache[uri] == null) { + readerCache[uri] = n5Writer + } writerCache[uri] = n5Writer - readerCache[uri] = n5Writer return n5Writer } diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt index 5c0f829b9..167112e16 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/MetadataState.kt @@ -38,7 +38,7 @@ interface MetadataState { var unit: String var reader: N5Reader - var writer: N5Writer? + val writer: N5Writer? var group: String val dataset: String get() = group @@ -71,14 +71,12 @@ interface MetadataState { target.resolution = source.resolution.copyOf() target.translation = source.translation.copyOf() target.unit = source.unit - target.reader = source.reader - target.writer = source.writer target.group = source.group } } } -open class SingleScaleMetadataState constructor( +open class SingleScaleMetadataState( final override var n5ContainerState: N5ContainerState, final override val metadata: N5SingleScaleMetadata ) : @@ -93,7 +91,9 @@ open class SingleScaleMetadataState constructor( override var translation = metadata.offset!! override var unit = metadata.unit()!! override var reader = n5ContainerState.reader - override var writer = n5ContainerState.writer + override val writer :N5Writer? + get() = n5ContainerState.writer + override var group = metadata.path!! override fun copy(): SingleScaleMetadataState { diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/N5ContainerState.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/N5ContainerState.kt index ad4938a06..4cde6ba18 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/N5ContainerState.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/metadata/N5ContainerState.kt @@ -1,41 +1,25 @@ package org.janelia.saalfeldlab.paintera.state.metadata -import javafx.beans.property.SimpleObjectProperty -import javafx.beans.value.ObservableValue import org.apache.commons.lang.builder.HashCodeBuilder -import org.janelia.saalfeldlab.fx.extensions.nonnullVal -import org.janelia.saalfeldlab.fx.extensions.nullableVal import org.janelia.saalfeldlab.n5.N5Reader import org.janelia.saalfeldlab.n5.N5Writer -import java.net.URI +import org.janelia.saalfeldlab.paintera.Paintera -data class N5ContainerState(private val n5Container: N5Reader) { +data class N5ContainerState(val reader: N5Reader) { - private val readerProperty: ObservableValue by lazy { SimpleObjectProperty(n5Container) } - val reader by readerProperty.nonnullVal() - - private val writerProperty: ObservableValue by lazy { SimpleObjectProperty(n5Container as? N5Writer) } - val writer by writerProperty.nullableVal() - - val uri : URI - get() = reader.uri + val writer by lazy { + (reader as? N5Writer) ?: Paintera.n5Factory.openWriterOrNull(uri.toString()) + } - val isReadOnly: Boolean - get() = writer == null + val uri by lazy { reader.uri!! } override fun equals(other: Any?): Boolean { return if (other is N5ContainerState) { - /* Equal if we are the same url, and we both either have a writer, or have no writer. */ - uri == other.uri && ((writer == null) == (other.writer == null)) + uri == other.uri } else { super.equals(other) } } - override fun hashCode(): Int { - val builder = HashCodeBuilder() - .append(reader.uri) - .append(writer?.uri ?: 0) - return builder.toHashCode() - } + override fun hashCode() = HashCodeBuilder().append(uri).toHashCode() } diff --git a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/raw/n5/N5BackendRaw.kt b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/raw/n5/N5BackendRaw.kt index d80edc341..9f30258a3 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/raw/n5/N5BackendRaw.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/paintera/state/raw/n5/N5BackendRaw.kt @@ -24,7 +24,7 @@ import java.lang.reflect.Type class N5BackendRaw(@JvmField val metadataState: MetadataState) : AbstractN5BackendRaw where D : NativeType, D : RealType, T : AbstractVolatileRealType, T : NativeType { - override val container = metadataState.writer ?: metadataState.reader + override val container = metadataState.reader override val dataset = metadataState.dataset override fun createSource(queue: SharedQueue, priority: Int, name: String): DataSource { From bc0da638d88c8f95baac344cb5667d54f3fed686 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 31 Aug 2023 16:37:13 -0400 Subject: [PATCH 6/7] fix: test and opening on windows --- .../kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt | 6 ++++-- .../saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java | 4 ---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt b/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt index c8ffc351f..e9918d403 100644 --- a/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt +++ b/src/main/kotlin/org/janelia/saalfeldlab/util/n5/N5Helpers.kt @@ -21,6 +21,7 @@ import org.janelia.saalfeldlab.labels.blocks.LabelBlockLookupAdapter import org.janelia.saalfeldlab.labels.blocks.n5.LabelBlockLookupFromN5Relative import org.janelia.saalfeldlab.n5.DatasetAttributes import org.janelia.saalfeldlab.n5.N5Reader +import org.janelia.saalfeldlab.n5.N5URI import org.janelia.saalfeldlab.n5.N5Writer import org.janelia.saalfeldlab.n5.hdf5.N5HDF5Reader import org.janelia.saalfeldlab.n5.universe.N5DatasetDiscoverer @@ -721,8 +722,9 @@ object N5Helpers { ?.takeIf { it.isJsonObject } ?.let { gson.fromJson(it, LabelBlockLookup::class.java) as LabelBlockLookup } ?: let { - val labelToBlockDataset = Paths.get(group, "label-to-block-mapping").toString() - val relativeLookup = LabelBlockLookupFromN5Relative("label-to-block-mapping/s%d") + val labelToBlockDataset = N5URI.normalizeGroupPath(group + reader.groupSeparator + "label-to-block-mapping"); + val scaleDatasetPattern = N5URI.normalizeGroupPath("label-to-block-mapping" + reader.groupSeparator + "s%d") + val relativeLookup = LabelBlockLookupFromN5Relative(scaleDatasetPattern) val numScales = if (metadataState is MultiScaleMetadataState) metadataState.scaleTransforms.size else 1 val labelBlockLookupMetadata = LabelBlockLookupGroup(labelToBlockDataset, numScales) labelBlockLookupMetadata.write(metadataState.writer!!) diff --git a/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java b/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java index 536d1fcbe..78b9e3029 100644 --- a/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java @@ -603,10 +603,6 @@ public DummyMetadataState(String labelsDataset, N5Writer container) { } - @Override public void setWriter(@Nullable N5Writer writer) { - - } - @Override public void setGroup(String group) { } From 606731c1e1bd296aad01edb47baf139af0bf4a30 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 1 Sep 2023 11:36:41 -0400 Subject: [PATCH 7/7] fix: don't use platform paths for groups On windows, `\` is interpreted as an escape character when normalizing the group path, and it is removed. --- .../paintera/data/n5/CommitCanvasN5.java | 15 +++++---------- .../org/janelia/saalfeldlab/util/n5/N5Data.java | 8 ++++---- .../paintera/data/n5/CommitCanvasN5Test.java | 7 ++++--- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5.java b/src/main/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5.java index 0ed3b9007..be5e1743b 100644 --- a/src/main/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5.java +++ b/src/main/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5.java @@ -37,12 +37,7 @@ import org.janelia.saalfeldlab.labels.blocks.LabelBlockLookupKey; import org.janelia.saalfeldlab.labels.blocks.n5.IsRelativeToContainer; import org.janelia.saalfeldlab.labels.downsample.WinnerTakesAll; -import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.*; import org.janelia.saalfeldlab.n5.imglib2.N5LabelMultisets; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.paintera.data.mask.persist.PersistCanvas; @@ -57,7 +52,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -136,7 +130,8 @@ public void updateLabelBlockLookup(final List> blockDi LOG.debug("Found scale datasets {}", (Object)scaleUniqueLabels); for (int level = 0; level < scaleUniqueLabels.length; ++level) { - final DatasetSpec datasetUniqueLabels = DatasetSpec.of(n5Writer, Paths.get(uniqueLabelsPath, scaleUniqueLabels[level]).toString()); + final String uniqueLabelScalePath = N5URI.normalizeGroupPath(uniqueLabelsPath + n5Writer.getGroupSeparator() + scaleUniqueLabels[level]); + final DatasetSpec datasetUniqueLabels = DatasetSpec.of(n5Writer, uniqueLabelScalePath); final TLongObjectMap removedById = new TLongObjectHashMap<>(); final TLongObjectMap addedById = new TLongObjectHashMap<>(); final TLongObjectMap blockDiffs = blockDiffsByLevel.get(level); @@ -262,8 +257,8 @@ public List> persistCanvas(final CachedCellImg blockDiffsAt = new TLongObjectHashMap<>(); blockDiffs.add(blockDiffsAt); - final DatasetSpec targetDataset = DatasetSpec.of(n5Writer, Paths.get(dataset, scaleDatasets[level]).toString()); - final DatasetSpec previousDataset = DatasetSpec.of(n5Writer, Paths.get(dataset, scaleDatasets[level - 1]).toString()); + final DatasetSpec targetDataset = DatasetSpec.of(n5Writer, N5URI.normalizeGroupPath(dataset + n5Writer.getGroupSeparator() + scaleDatasets[level])); + final DatasetSpec previousDataset = DatasetSpec.of(n5Writer, N5URI.normalizeGroupPath(dataset + n5Writer.getGroupSeparator() + scaleDatasets[level - 1])); final double[] targetDownsamplingFactors = N5Helpers.getDownsamplingFactors(n5Writer, targetDataset.dataset); final double[] previousDownsamplingFactors = N5Helpers.getDownsamplingFactors(n5Writer, previousDataset.dataset); diff --git a/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java b/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java index 086577340..37749fbdb 100644 --- a/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java +++ b/src/main/java/org/janelia/saalfeldlab/util/n5/N5Data.java @@ -380,7 +380,7 @@ ImagesWithTransform[] openRawMultiscale( final double[] initialDonwsamplingFactors = N5Helpers.getDownsamplingFactors( reader, - Paths.get(dataset, scaleDatasets[0]).toString() + N5URI.normalizeGroupPath(dataset + reader.getGroupSeparator() + scaleDatasets[0]) ); LOG.debug("Initial transform={}", transform); final ExecutorService es = Executors.newFixedThreadPool( @@ -393,7 +393,7 @@ ImagesWithTransform[] openRawMultiscale( final int fScale = scale; futures.add(es.submit(ThrowingSupplier.unchecked(() -> { LOG.debug("Populating scale level {}", fScale); - final String scaleDataset = Paths.get(dataset, scaleDatasets[fScale]).toString(); + final String scaleDataset = N5URI.normalizeGroupPath(dataset + reader.getGroupSeparator() + scaleDatasets[fScale]); imagesWithInvalidate[fScale] = openRaw(reader, scaleDataset, transform.copy(), queue, priority); final double[] downsamplingFactors = N5Helpers.getDownsamplingFactors(reader, scaleDataset); LOG.debug("Read downsampling factors: {}", Arrays.toString(downsamplingFactors)); @@ -685,7 +685,7 @@ public static ImagesWithTransform[ final double[] initialDonwsamplingFactors = N5Helpers.getDownsamplingFactors( reader, - Paths.get(dataset, scaleDatasets[0]).toString()); + N5URI.normalizeGroupPath(dataset + reader.getGroupSeparator() + scaleDatasets[0])); final ExecutorService es = Executors.newFixedThreadPool( scaleDatasets.length, new NamedThreadFactory("populate-mipmap-scales-%d", true)); @@ -695,7 +695,7 @@ public static ImagesWithTransform[ final int fScale = scale; futures.add(es.submit(ThrowingSupplier.unchecked(() -> { LOG.debug("Populating scale level {}", fScale); - final String scaleDataset = Paths.get(dataset, scaleDatasets[fScale]).toString(); + final String scaleDataset = N5URI.normalizeGroupPath(dataset + reader.getGroupSeparator() + scaleDatasets[fScale]); imagesWithInvalidate[fScale] = openLabelMultiset(reader, scaleDataset, transform.copy(), queue, priority); final double[] downsamplingFactors = N5Helpers.getDownsamplingFactors(reader, scaleDataset); LOG.debug("Read downsampling factors: {}", Arrays.toString(downsamplingFactors)); diff --git a/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java b/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java index 78b9e3029..05e73c8a7 100644 --- a/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5Test.java @@ -352,10 +352,11 @@ private static void testPainteraData( testCanvasPersistance(container, dataset, s0, canvas, openLabels, asserts); // test highest level block lookups - final String uniqueBlock0 = String.join("/", dataset, "unique-labels", "s0"); + final String groupSeparator = container.getReader().getGroupSeparator(); + final String uniqueBlock0Group = N5URI.normalizeGroupPath(String.join(groupSeparator, dataset, "unique-labels","s0")); final Path mappingPattern = Paths.get(container.getUri().getPath(), dataset, "label-to-block-mapping", "s%d", "%d"); final Path mapping0 = Paths.get(container.getUri().getPath(), dataset, "label-to-block-mapping", "s0"); - final DatasetAttributes uniqueBlockAttributes = writer.getDatasetAttributes(uniqueBlock0); + final DatasetAttributes uniqueBlockAttributes = writer.getDatasetAttributes(uniqueBlock0Group); final List blocks = Grids.collectAllContainedIntervals(dims, blockSize); final TLongObjectMap labelToBLockMapping = new TLongObjectHashMap<>(); for (final Interval block : blocks) { @@ -375,7 +376,7 @@ private static void testPainteraData( } }); - final DataBlock uniqueBlock = writer.readBlock(uniqueBlock0, uniqueBlockAttributes, blockPos); + final DataBlock uniqueBlock = writer.readBlock(uniqueBlock0Group, uniqueBlockAttributes, blockPos); Assert.assertEquals(labels, new TLongHashSet((long[])uniqueBlock.getData())); }