Skip to content

Commit bb88768

Browse files
Anna Grimclaude
andcommitted
feat: add JointProofreader, ArboristSplitDataset, fov_merge, and misc fixes
- models/joint_model.py: add JointProofreader (joint split+merge model with shared CNN3D backbone, Arborist merge head, and 3-layer HGAT split head) - models/new_gnn_models.py: add gat3 (3rd HGAT layer), bump gnn_hidden_dim and img_embed_dim defaults 64→96, add ProposalTreeEncoder - split_proofreading/new_split_feature_extraction.py: add ArboristSplitDataset (prefetched iterator with Arborist tree features alongside image patches) - split_proofreading/split_inference.py: route to ArboristSplitDataset when use_tree_features=True (required by JointProofreader split head) - configs.py: add fov_merge to ImageConfig (load at fov, resize to patch_shape for merge detection); add modality to MergeInferenceConfig; add model_config_path and use_tree_features to SplitInferenceConfig; add keys/__getitem__ to Config base class - machine_learning/image_dataloader.py: DetectionPatchLoader loads at fov_merge when set, then resizes to patch_shape before returning - machine_learning/train.py: add n_mistake_mips reservoir-sampled mistake MIP saving per val epoch - utils/img_util.py: cast img to float32 before np.percentile to fix NaN on float16 arrays (all-white MIP bug) - merge_proofreading/search_datasets.py: fix subgraph_to_tree_sample root arg - proofreading_pipeline.py: pass modality and use_tree_features from configs - proposal_graph.py: accept **kwargs in __init__ for forward-compat - utils/geometry_util.py: guard is_double against empty query_nodes Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 8f83fc7 commit bb88768

12 files changed

Lines changed: 924 additions & 90 deletions

File tree

‎src/neuron_proofreader/configs.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@
2323

2424
class Config(ABC):
2525

26+
def keys(self):
27+
return asdict(self).keys()
28+
29+
def __getitem__(self, key):
30+
return asdict(self)[key]
31+
2632
def to_dict(self):
2733
"""
2834
Converts configuration attributes to a dictionary.
@@ -90,15 +96,19 @@ class ImageConfig(Config):
9096
----------
9197
brightness_clip : int
9298
Intensity value that voxel brightness is clipped to.
99+
fov_merge : Tuple[int], optional
100+
Patch shape loaded from the image for merge detection. Resized to
101+
patch_shape before the model. None loads at patch_shape directly.
93102
percentiles : Tuple[float], optional
94103
Percentiles used to normalize patches.
95104
patch_shape : Tuple[int]
96-
Shape of patch to be read from image.
105+
Shape of patch expected by the model.
97106
transform : bool
98107
Indication of whether to use image augmentation.
99108
"""
100109

101110
brightness_clip: int = 400
111+
fov_merge: Tuple[int, int, int] = None
102112
img_path: str = None
103113
name: str = "image_config"
104114
percentiles: Tuple[float, float] = (1, 99.5)
@@ -127,6 +137,10 @@ class MergeInferenceConfig(Config):
127137
memory; must be set before running learned merge detection.
128138
min_search_size : float
129139
Minimum fragment cable length (in microns) to include in the search.
140+
modality : str
141+
Input modality expected by the merge model. Options are "image" for
142+
image-only models and "image_graph" for models that also consume a
143+
skeleton TreeSample per site (e.g. JointProofreader).
130144
model_config_path : str, optional
131145
Path to the merge model config JSON file.
132146
model_path : str, optional
@@ -145,6 +159,7 @@ class MergeInferenceConfig(Config):
145159

146160
batch_size: int = None
147161
min_search_size: float = 0
162+
modality: str = "image"
148163
model_config_path: str = None
149164
model_path: str = None
150165
name: str = "merge_inference_config"
@@ -167,6 +182,8 @@ class SplitInferenceConfig(Config):
167182
Increment that acceptance threshold is lowered by each round.
168183
min_threshold : float
169184
Minimum confidence threshold for accepting a proposal.
185+
model_config_path : str, optional
186+
Path to the split model config JSON file.
170187
patch_shape : Tuple[int], optional
171188
Patch shape for image sampling, overrides ImageConfig.patch_shape if
172189
set.
@@ -176,16 +193,22 @@ class SplitInferenceConfig(Config):
176193
split model was trained with.
177194
removal_threshold : float
178195
Proposals with model predictions below this value are removed.
196+
use_tree_features : bool
197+
If True, each proposal carries an Arborist TreeSample alongside its
198+
image patch. Required by models whose split head encodes skeleton
199+
geometry (e.g. JointProofreader).
179200
"""
180201

181202
batch_size: int = None
182203
dt: float = 0.05
183204
min_threshold: float = 0.8
205+
model_config_path: str = None
184206
model_path: str = None
185207
name: str = "split_inference_config"
186208
patch_shape: Tuple[int, int, int] = None
187209
percentiles: Tuple[float, float] = (1, 99.9)
188210
removal_threshold: float = 0.3
211+
use_tree_features: bool = False
189212

190213

191214
@dataclass

‎src/neuron_proofreader/machine_learning/image_dataloader.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -276,21 +276,23 @@ class DetectionPatchLoader(PatchLoader):
276276

277277
# --- Implementation of Abstract Inferface ---
278278
def __call__(self, node):
279-
# Get patches
280279
center, shape = self.compute_patch_specs(node)
281280
img = self.read_image(center, shape)
282281
mask = self.create_mask(center, shape, node)
283-
patches = self.stack(img, mask)
284282

285-
# Check whether to apply image augmentation
283+
if self.fov_merge is not None:
284+
img = img_util.resize(img, self.patch_shape)
285+
mask = img_util.resize_nearest(mask, self.patch_shape)
286+
287+
patches = self.stack(img, mask)
286288
if self.transform:
287289
patches = self.transform(patches)
288290
return node, patches
289291

290292
def compute_patch_specs(self, node):
291293
voxel = self.graph.node_voxel(node)
292294
voxel = self.adjust_voxel(voxel)
293-
return voxel, self.patch_shape
295+
return voxel, self.fov_merge or self.patch_shape
294296

295297

296298
class DetectionBatchLoader(PatchLoader):

‎src/neuron_proofreader/machine_learning/train.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ def __init__(
7373
max_epochs=200,
7474
pos_weight=None,
7575
n_mips=0,
76+
n_mistake_mips=64,
7677
use_amp=True,
7778
):
7879
"""
@@ -102,6 +103,10 @@ def __init__(
102103
Number of randomly sampled validation predictions to save as MIPs
103104
each epoch, named by outcome (e.g. "false_positive{idx}.png").
104105
Default is 0 (disabled).
106+
n_mistake_mips : int, optional
107+
Number of mistake MIPs (false positives and false negatives) to
108+
save per epoch via reservoir sampling. Saved under
109+
mistake_mips/epoch_NNN/ in the log directory. Default is 64.
105110
use_amp : bool, optional
106111
If True, enables automatic mixed precision (float16) training.
107112
Default is True.
@@ -119,8 +124,10 @@ def __init__(
119124
self.log_dir = log_dir
120125
self.max_epochs = max_epochs
121126
self.mips_dir = os.path.join(log_dir, "mips")
127+
self.mistake_mips_dir = os.path.join(log_dir, "mistake_mips")
122128
self.model_name = model_name
123129
self.n_mips = n_mips
130+
self.n_mistake_mips = n_mistake_mips
124131

125132
pw = (
126133
torch.tensor([pos_weight], device=device)
@@ -202,6 +209,9 @@ def _run_epoch(self, dataloader, epoch, train):
202209
util.mkdir(self.mips_dir, True)
203210
n = len(dataloader.dataset)
204211
self._mip_idxs = set(random.sample(range(n), min(self.n_mips, n)))
212+
if self.n_mistake_mips > 0:
213+
self._mistake_buf = []
214+
self._mistake_seen = 0
205215

206216
metrics = ml_util.BinaryMetricAccumulator()
207217
for x, y in dataloader:
@@ -222,10 +232,13 @@ def _run_epoch(self, dataloader, epoch, train):
222232

223233
if not train:
224234
self._save_mips(x, y, y_pred, idx_offset)
235+
self._collect_mistakes(x, y, y_pred)
225236
idx_offset += len(y)
226237

227238
stats = metrics.compute(min_recall=self.enforced_recall)
228239
self.update_tensorboard(stats, epoch, prefix)
240+
if not train and self.n_mistake_mips > 0:
241+
self._flush_mistake_mips(epoch)
229242
return stats
230243

231244
def forward_pass(self, x, y):
@@ -348,6 +361,37 @@ def _save_mips(self, x, y, y_pred, idx_offset):
348361
x[i, 0], 2 * x[i, 1], output_path
349362
)
350363

364+
def _collect_mistakes(self, x, y, y_pred):
365+
"""
366+
Adds false positives and false negatives from this batch to a
367+
reservoir-sampled buffer capped at n_mistake_mips images.
368+
"""
369+
if self.n_mistake_mips == 0:
370+
return
371+
imgs = ml_util.to_cpu(x["img"] if isinstance(x, dict) else x, True)
372+
for i in range(len(y)):
373+
outcome = classify_prediction(y[i], y_pred[i])
374+
if outcome not in ("false_positive", "false_negative"):
375+
continue
376+
self._mistake_seen += 1
377+
item = (imgs[i], outcome)
378+
if len(self._mistake_buf) < self.n_mistake_mips:
379+
self._mistake_buf.append(item)
380+
else:
381+
j = random.randint(0, self._mistake_seen - 1)
382+
if j < self.n_mistake_mips:
383+
self._mistake_buf[j] = item
384+
385+
def _flush_mistake_mips(self, epoch):
386+
"""
387+
Saves the buffered mistake MIPs for this epoch to disk.
388+
"""
389+
epoch_dir = os.path.join(self.mistake_mips_dir, f"epoch_{epoch:03d}")
390+
util.mkdir(epoch_dir)
391+
for i, (img, outcome) in enumerate(self._mistake_buf):
392+
path = os.path.join(epoch_dir, f"{outcome}_{i:02d}.png")
393+
img_util.plot_image_and_segmentation_mips(img[0], 2 * img[1], path)
394+
351395
def save_config(self):
352396
"""
353397
Saves trainer configuration to a JSON file in the log directory.
@@ -414,6 +458,7 @@ def __init__(
414458
max_epochs=200,
415459
pos_weight=None,
416460
n_mips=0,
461+
n_mistake_mips=64,
417462
):
418463
"""
419464
Instantiates a DistributedTrainer object.
@@ -433,6 +478,8 @@ def __init__(
433478
pos_weight : float or None, optional
434479
Weight applied to the positive class in BCEWithLogitsLoss. Default
435480
is None (no reweighting).
481+
n_mistake_mips : int, optional
482+
Number of mistake MIPs to save per epoch. Default is 64.
436483
"""
437484
# Call parent class
438485
super().__init__(
@@ -444,6 +491,7 @@ def __init__(
444491
max_epochs=max_epochs,
445492
pos_weight=pos_weight,
446493
n_mips=n_mips,
494+
n_mistake_mips=n_mistake_mips,
447495
)
448496

449497
# Check that multiple GPUs are available

‎src/neuron_proofreader/merge_proofreading/search_datasets.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ def get_patch_and_arborist(self, nodes, img, offset):
290290
s = img_util.get_slices(center, self.patch_shape)
291291
patch = img[(slice(0, 2), *s)]
292292
subgraph = self.graph.rooted_subgraph(node, self.subgraph_radius)
293-
tree_sample = subgraph_to_tree_sample(subgraph, node)
293+
tree_sample = subgraph_to_tree_sample(subgraph, 0)
294294
yield node, patch, tree_sample
295295

296296
# --- Helpers ---
@@ -353,5 +353,5 @@ def get_patch(self, node, img):
353353
def get_patch_and_arborist(self, node, img):
354354
patch = torch.from_numpy(img).float()
355355
subgraph = self.graph.rooted_subgraph(node, self.subgraph_radius)
356-
tree_sample = subgraph_to_tree_sample(subgraph, node)
356+
tree_sample = subgraph_to_tree_sample(subgraph, 0)
357357
yield node, patch, tree_sample

0 commit comments

Comments
 (0)