Merge pull request #528 from saalfeldlab/feat/samAutoInterpolation

Feat/sam auto interpolation

README.md

@@ -308,7 +308,7 @@ Tutorial videos:
 | `Ctrl` + `Shift` + `D`   | Reset all view windows to default position (3 orthogonal views in one window, with a 3D view as well)                                                             |
 | `F2`                     | Toggle menu bar visibility                                                                                                                                        |
 | `Shift` + `F2`           | Toggle menu bar mode (overlay views, or above views)                                                                                                              |
-| `F3`                     | Toggle statuc bar visibility                                                                                                                                      |
+| `F3`                     | Toggle status bar visibility                                                                                                                                      |
 | `Shift` + `F3`           | Toggle status bar mode (overlay views, or below views)                                                                                                            | `P`                      | Toggle visibility of side panel menu on right hand side                                                                                                           |
 | `F11`                    | Toggle fullscreen                                                                                                                                                 |
 | **Project Controls**     |                                                                                                                                                                   |
@@ -399,14 +399,20 @@ Tutorial videos:
 
 | Action                       | Description                                                                                |
 |------------------------------|--------------------------------------------------------------------------------------------|
-| `S`                          | Enter shape interpolation mode                                                             |
-| `Esc`                        | Exit shape interpolation mode                                                              |
-| `1` / `0`                    | Move to first/last slice                                                                   |
+| `S`                          | Enter Shape Interpolation mode                                                             |
+| `Esc`                        | Exit current tool, or exit Shape Interpolation mode if no tool is active                   |
+| `Up Arrow` / `Down Arrow`    | Move to first/last slice                                                                   |
 | `Left Arrow` / `Right Arrow` | Move to the previous/next slice                                                            |
 | `Enter`                      | Commit interpolated shape into canvas                                                      |
 | `Ctrl` + `P`                 | Toggle interpolation preview                                                               |
 | Left Click                   | Exclusively select the label under the cursor, **removing** all other labels at this slice |
 | Right Click                  | Inclusively select the label under the cursor, **keeping** all other labels at this slice  |
+| `F`                          | Activate 2D Flood Fill Tool                                                                |
+| Hold `SPACE`                 | Activate PAINT Tool                                                                        |
+| `A`                          | Activate SAM Tool                                                                          |
+| `P`                          | Bisect all current slices with new SAM auto-segmentation slices                            |
+| `[`                          | Add a new SAM auto-segmentation slices to the beginning of the current slices              |
+| `]`                          | Add a new SAM auto-segmentation slices to the end of the current slices                    |
 
 #### Automatic Labelling: Segment Anything
 - Integrates Segment Anything to predict automatic segmentations, based on the underlying image
@@ -417,17 +423,18 @@ Tutorial videos:
 - See [technical notes](#automatic-labelling-with-segment-anything) for more information
 See [Technical Notes](#)
 
-| Action               | Description                                                             |
-|----------------------|-------------------------------------------------------------------------|
-| `A`                  | Start automatic labelling mode                                          |
-| Left Click / `Enter` | Paint current automatic segmentation to the canvas                      |
-| `Ctrl` + Left Click  | Add point which should be **inside** of the automatic segmentation      |
-| `Ctrl` + Right Click | Add point which should be **outside** of the automatic segmentation     |
-| `Ctrl` + Scroll      | Increase or decreses the threshold to accept the automatic segmentation |
+| Action               | Description                                                           |
+|----------------------|-----------------------------------------------------------------------|
+| `A`                  | Start automatic labelling mode                                        |
+| Left Click / `Enter` | Paint current automatic segmentation to the canvas                    |
+| `Ctrl` + Left Click  | Add point which should be **inside** of the automatic segmentation    |
+| `Ctrl` + Right Click | Add point which should be **outside** of the automatic segmentation   |
+| Left Click Drag      | Create box prompt for automatic segmentation                          |
+| `Ctrl` + Scroll      | Increase or decrease the prediction threshold to use for segmentation |
 
 ## Supported Data
 
-Paintera supports single and multi-channel raw data and label data from N5, HDF5, Zarr, AWS, and Google Cloud storage. The preferred format is the Paintera data format but regular single or multi-scale datasets can be imported as well. Any N5-like format can be converted into the preferred Paintera format with the [Paintera Conversion Helper](https://github.com/saalfeldlab/paintera-conversion-helper) that is automatically installed with Paintera from [conda](#conda) or [pip](#pip). For example, to convert raw and neuron_ids of the [padded sample A](https://cremi.org/static/data/sample_A_padded_20160501.hdf) of the [CREMI](https://cremi.org) challenge, simply run (assuming the data was downloaded into the current directory):
+Paintera supports single and multi-channel raw data and label data from N5, Zarr, and HDF5 data, stored locally and on AWS or Google Cloud storage. The preferred format is the Paintera data format but regular single or multi-scale datasets can be imported as well. Any N5-like format can be converted into the preferred Paintera format with the [Paintera Conversion Helper](https://github.com/saalfeldlab/paintera-conversion-helper) that is automatically installed with Paintera from [conda](#conda) or [pip](#pip). For example, to convert raw and neuron_ids of the [padded sample A](https://cremi.org/static/data/sample_A_padded_20160501.hdf) of the [CREMI](https://cremi.org) challenge, simply run (assuming the data was downloaded into the current directory):
 ```sh
 paintera-convert to-paintera \
   --scale 2,2,1 2,2,1 2,2,1 2 2 \
@@ -441,7 +448,7 @@ paintera-convert to-paintera \
     -d volumes/labels/neuron_ids
 ```
 
-Here,
+In this example:
 
 - `--scale` specifies the number of downsampled mipmap levels, where each comma-separated triple specifies a downsampling factor relative to the previous level. The total number of levels in the mipmap pyramid is the number of specified factors plus one (the data at original resolution)
 - `--revert-array-attributes` reverses array attributes like `"resolution"` and `"offset"` that may be available in the source datasets
@@ -463,8 +470,8 @@ Through the N5 API, Paintera supports multiple data container types:
 - N5
 - HDF5
 - Zarr
-- N5 over AWS S3
-- N5 over Google Cloud
+- N5/Zarr over AWS S3
+- N5/Zarr over Google Cloud
 #### Metadata
 Paintera also can understand multiple metadata variants:
 - Paintera Metadata [#61](https://github.com/saalfeldlab/paintera/issues/61)

pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>org.scijava</groupId>
 		<artifactId>pom-scijava</artifactId>
-		<version>36.0.0</version>
+		<version>37.0.0</version>
 	</parent>
 
 	<groupId>org.janelia.saalfeldlab</groupId>
@@ -53,7 +53,7 @@
 		<javadoc.skip>true</javadoc.skip>
 		<maven.javadoc.skip>${javadoc.skip}</maven.javadoc.skip>
 
-		<saalfx.version>1.1.0</saalfx.version>
+		<saalfx.version>1.2.0</saalfx.version>
 
 		<janino.version>3.0.7</janino.version>
 		<logback.version>1.4.0</logback.version>
@@ -75,13 +75,14 @@
 		<macos.sign />
 		<macos.sign.identity />
 
-		<n5.version>3.0.2</n5.version>
-		<n5-hdf5.version>2.0.0</n5-hdf5.version>
-		<n5-google-cloud.version>4.0.0</n5-google-cloud.version>
-		<n5-aws-s3.version>4.0.1</n5-aws-s3.version>
-		<n5-zarr.version>1.0.1</n5-zarr.version>
+		<!--N5 versions. Should match with versions pinned by n5-universe -->
+		<n5.version>3.2.0</n5.version>
+		<n5-hdf5.version>2.2.0</n5-hdf5.version>
+		<n5-google-cloud.version>4.1.0</n5-google-cloud.version>
+		<n5-aws-s3.version>4.1.2</n5-aws-s3.version>
+		<n5-zarr.version>1.3.1</n5-zarr.version>
 		<n5-imglib2.version>7.0.0</n5-imglib2.version>
-		<n5-universe.version>1.1.0</n5-universe.version>
+		<n5-universe.version>1.4.0</n5-universe.version>
 
 		<enforcer.skip>true</enforcer.skip>
 	</properties>
@@ -156,6 +157,10 @@
 			<groupId>org.jetbrains.kotlin</groupId>
 			<artifactId>kotlin-stdlib</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.janelia.saalfeldlab</groupId>
+			<artifactId>n5-aws-s3</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>io.github.oshai</groupId>
 			<artifactId>kotlin-logging-jvm</artifactId>
@@ -194,7 +199,7 @@
 		<dependency>
 			<groupId>net.imglib2</groupId>
 			<artifactId>imglib2-label-multisets</artifactId>
-			<version>0.12.0</version>
+			<version>0.13.1</version>
 		</dependency>
 		<dependency>
 			<groupId>sc.fiji</groupId>

src/main/java/bdv/fx/viewer/ViewerPanelFX.java

@@ -30,7 +30,7 @@
 package bdv.fx.viewer;
 
 import bdv.cache.CacheControl;
-import bdv.fx.viewer.render.RenderUnit;
+import bdv.fx.viewer.render.ViewerRenderUnit;
 import bdv.viewer.Interpolation;
 import bdv.viewer.RequestRepaint;
 import bdv.viewer.Source;
@@ -82,7 +82,7 @@ public class ViewerPanelFX
 
 	private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-	private final RenderUnit renderUnit;
+	private final ViewerRenderUnit renderUnit;
 
 	private final CanvasPane canvasPane = new CanvasPane(1, 1);
 
@@ -175,7 +175,7 @@ public ViewerPanelFX(
 
 		ActionSet.installActionSet(this, mouseTracker.getActions());
 
-		this.renderUnit = new RenderUnit(
+		this.renderUnit = new ViewerRenderUnit(
 				threadGroup,
 				this::getState,
 				interpolation,
@@ -556,7 +556,7 @@ public OverlayPane<?> getDisplay() {
 		return this.overlayPane;
 	}
 
-	public RenderUnit getRenderUnit() {
+	public ViewerRenderUnit getRenderUnit() {
 
 		return renderUnit;
 	}
@@ -565,7 +565,7 @@ private void startRenderAnimator() {
 
 		new AnimationTimer() {
 
-			private RenderUnit.RenderResult renderResult = null;
+			private ViewerRenderUnit.RenderResult renderResult = null;
 
 			@Override
 			public void handle(long now) {

src/main/java/bdv/fx/viewer/ViewerState.java

@@ -38,11 +38,19 @@ protected void setViewerTransform(final AffineTransform3D to) {
 		stateChanged();
 	}
 
+	/**
+	 * set {@param to} to the global to viewer transform for this ViewerState
+	 * @param to the target transform object
+	 */
 	public synchronized void getViewerTransform(final AffineTransform3D to) {
 
 		to.set(this.viewerTransform);
 	}
 
+	public synchronized double[] getDimensions() {
+		return new double[]{viewer.getWidth(), viewer.getHeight()};
+	}
+
 	public void setTimepoint(final int timepoint) {
 
 		synchronized (this) {

src/main/java/bdv/fx/viewer/render/MultiResolutionRendererFX.java

@@ -37,8 +37,6 @@
 import net.imglib2.parallel.TaskExecutor;
 import net.imglib2.type.numeric.ARGBType;
 
-import java.util.concurrent.ExecutorService;
-
 /**
  * A renderer that uses a coarse-to-fine rendering scheme. First, a small {@link ArrayImg} at a fraction of
  * the canvas resolution is rendered. Then, increasingly larger images are rendered, until the full canvas resolution is
@@ -108,7 +106,7 @@ public PixelBufferWritableImage create(final int width, final int height, final
 
 	public MultiResolutionRendererFX(
 			final TransformAwareRenderTargetGeneric<PixelBufferWritableImage> display,
-			final PainterThread painterThread,
+			final PainterThreadFx painterThread,
 			final double[] screenScales,
 			final long targetRenderNanos,
 			final boolean doubleBuffered,

src/main/java/bdv/fx/viewer/render/MultiResolutionRendererGeneric.java

@@ -110,7 +110,7 @@ public interface ImageGenerator<T> {
 	/**
 	 * Thread that triggers repainting of the display. Requests for repainting are send there.
 	 */
-	private final PainterThread painterThread;
+	private final PainterThreadFx painterThread;
 
 	/**
 	 * Currently active projector, used to re-paint the display. It maps the source data to {@link #screenImages}.
@@ -251,6 +251,8 @@ public interface ImageGenerator<T> {
 
 	private final AffineTransform3D currentProjectorTransform = new AffineTransform3D();
 
+	final AnimationTimer animation;
+
 	/**
 	 * @param display                    The canvas that will display the images we render.
 	 * @param painterThread              Thread that triggers repainting of the display. Requests for repainting are send there.
@@ -268,7 +270,7 @@ public interface ImageGenerator<T> {
 	 */
 	MultiResolutionRendererGeneric(
 			final TransformAwareRenderTargetGeneric<T> display,
-			final PainterThread painterThread,
+			final PainterThreadFx painterThread,
 			final double[] screenScales,
 			final long targetRenderNanos,
 			final boolean doubleBuffered,
@@ -303,15 +305,16 @@ public interface ImageGenerator<T> {
 		newFrameRequest = false;
 		previousTimepoint = -1;
 
-		new AnimationTimer() {
+		this.animation = new AnimationTimer() {
 
 			@Override
 			public void handle(long now) {
 				if (requestedScreenScaleIndex >= 0 && requestedScreenScaleIndex < screenScales.length && pendingRepaintRequests[requestedScreenScaleIndex] != null) {
 					painterThread.requestRepaint();
 				}
 			}
-		}.start();
+		};
+		this.animation.start();
 	}
 
 	/**
@@ -457,7 +460,7 @@ public int paint(
 			createProjector = newFrameRequest || resized || requestedScreenScaleIndex != currentScreenScaleIndex || !sameAsLastRenderedInterval;
 			newFrameRequest = false;
 
-			final List<SourceAndConverter<?>> sacs = sources;
+			final List<SourceAndConverter<?>> sacs = List.copyOf(sources);
 
 			if (createProjector) {
 				currentScreenScaleIndex = requestedScreenScaleIndex;
@@ -576,7 +579,7 @@ else if (!p.isValid()) {
 				}
 			} else {
 				// FIXME: there is a race condition that sometimes may cause an ArrayIndexOutOfBounds exception:
-				// Screen scales are first initialized with the default setting (see RenderUnit),
+				// Screen scales are first initialized with the default setting (see ViewerRenderUnit),
 				// then the project metadata is loaded, and the screen scales are changed to the saved configuration.
 				// If the project screen scales are [1.0], sometimes the renderer receives a request to re-render the screen at screen scale 1, which results in the exception.
 				if (currentScreenScaleIndex >= pendingRepaintRequests.length)

src/main/java/bdv/fx/viewer/render/PainterThread.java → src/main/java/bdv/fx/viewer/render/PainterThreadFx.java

@@ -6,11 +6,11 @@
 import java.lang.invoke.MethodHandles;
 import java.util.concurrent.RejectedExecutionException;
 
-public final class PainterThread extends Thread {
+public final class PainterThreadFx extends Thread {
 
 	private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-	private final PainterThread.Paintable paintable;
+	private final PainterThreadFx.Paintable paintable;
 
 	private boolean pleaseRepaint;
 
@@ -19,17 +19,17 @@ public final class PainterThread extends Thread {
 	private long lastUpdate = -1;
 	private long targetFrameRateMs = 1000 / 60;
 
-	public PainterThread(PainterThread.Paintable paintable) {
+	public PainterThreadFx(PainterThreadFx.Paintable paintable) {
 
 		this(null, "PainterThread", paintable);
 	}
 
-	public PainterThread(ThreadGroup group, PainterThread.Paintable paintable) {
+	public PainterThreadFx(ThreadGroup group, PainterThreadFx.Paintable paintable) {
 
 		this(group, "PainterThread", paintable);
 	}
 
-	public PainterThread(ThreadGroup group, String name, PainterThread.Paintable paintable) {
+	public PainterThreadFx(ThreadGroup group, String name, PainterThreadFx.Paintable paintable) {
 
 		super(group, name);
 		this.paintable = paintable;