Add scatterplots to C pages

pyproject.toml

@@ -49,6 +49,7 @@ version = "neuview --version"
 setup-env = "cp .env.example .env && echo 'Created .env file. Please edit it and add your NEUPRINT_TOKEN.'"
 fill-all = "neuview fill-queue --all"
 create-list = "neuview create-list"
+create-scatter = "neuview create-scatter"
 
 [tool.pytest.ini_options]
 testpaths = [ "test",]
@@ -77,6 +78,9 @@ task = "pop-all"
 [[tool.pixi.tasks.create-all-pages.depends-on]]
 task = "create-list"
 
+[[tool.pixi.tasks.create-all-pages.depends-on]]
+task = "create-scatter"
+
 [[tool.pixi.tasks.create-all-pages.depends-on]]
 task = "increment-version"
 
@@ -104,13 +108,19 @@ args = [ "",]
 [[tool.pixi.tasks.subset-medium-no-index.depends-on]]
 task = "pop-all"
 
+[[tool.pixi.tasks.subset-medium-no-index.depends-on]]
+task = "create-scatter"
+
 [tool.pixi.tasks.subset-medium]
 [[tool.pixi.tasks.subset-medium.depends-on]]
 task = "subset-medium-no-index"
 
 [[tool.pixi.tasks.subset-medium.depends-on]]
 task = "create-list"
 
+[[tool.pixi.tasks.subset-medium.depends-on]]
+task = "create-scatter"
+
 [tool.pixi.tasks.subset-small-no-index]
 [[tool.pixi.tasks.subset-small-no-index.depends-on]]
 task = "clean-output"
@@ -122,13 +132,19 @@ args = [ "config.yaml", "subset-small",]
 [[tool.pixi.tasks.subset-small-no-index.depends-on]]
 task = "pop-all"
 
+[[tool.pixi.tasks.subset-small-no-index.depends-on]]
+task = "create-scatter"
+
 [tool.pixi.tasks.subset-small]
 [[tool.pixi.tasks.subset-small.depends-on]]
 task = "subset-small-no-index"
 
 [[tool.pixi.tasks.subset-small.depends-on]]
 task = "create-list"
 
+[[tool.pixi.tasks.subset-small.depends-on]]
+task = "create-scatter"
+
 [tool.pixi.tasks.increment-version]
 cmd = "python scripts/increment_version.py"
 

src/neuview/cli.py

@@ -10,13 +10,15 @@
 import sys
 from typing import Optional
 import logging
+from pathlib import Path
 
 from .commands import (
     GeneratePageCommand,
     TestConnectionCommand,
     FillQueueCommand,
     PopCommand,
     CreateListCommand,
+    CreateScatterCommand,
 )
 from .services import ServiceContainer
 from .services.neuron_discovery_service import InspectNeuronTypeCommand
@@ -375,5 +377,30 @@ async def run_create_list():
     asyncio.run(run_create_list())
 
 
+@main.command("create-scatter")
+@click.pass_context
+def create_scatter(ctx):
+    """Generate three SVG scatterplots of spatial metrics for optic lobe types."""
+    services = setup_services(ctx.obj["config_path"], ctx.obj["verbose"])
+
+    async def run_create_scatter():
+
+        await services.scatter_service.create_scatterplots()
+
+        # Print the three scatterplot files that should have been created
+        scfg = services.scatter_service.scatter_config
+        scatter_dir = Path(scfg.scatter_dir)
+        fname = scfg.scatter_fname
+
+        for region in ("ME", "LO", "LOP"):
+            file_path = scatter_dir / f"{region}_{fname}"
+            if file_path.exists():
+                click.echo(f"✅ Created: {file_path}")
+            else:
+                click.echo(f"⚠️ Expected but not found: {file_path}", err=True)
+
+    asyncio.run(run_create_scatter())
+
+
 if __name__ == "__main__":
     main()

src/neuview/commands.py

@@ -94,6 +94,17 @@ def __post_init__(self):
             self.requested_at = datetime.now()
 
 
+@dataclass
+class CreateScatterCommand:
+    """Command to create svg scatterplots of spatial metrics."""
+
+    requested_at: Optional[datetime] = None
+
+    def __post_init__(self):
+        if self.requested_at is None:
+            self.requested_at = datetime.now()
+
+
 @dataclass
 class DatasetInfo:
     """Information about the dataset."""

src/neuview/services/cache_service.py

@@ -171,6 +171,9 @@ async def save_neuron_data_to_cache(
 
                         # Filter ROIs by threshold and clean names (same logic as IndexService)
                         threshold = self.threshold_service.get_roi_filtering_threshold()
+                        threshold_high = self.threshold_service.get_roi_filtering_threshold(
+                            profile_name="roi_filtering_strict"
+                            )
                         cleaned_roi_summary = []
                         seen_names = set()
 
@@ -189,16 +192,22 @@ async def save_neuron_data_to_cache(
                                 )
                                 if clean_name not in seen_names:
                                     seen_names.add(clean_name)
-                                    cleaned_roi_summary.append(
-                                        {
+                                    entry = {
                                             "name": clean_name,
                                             "pre_percentage": roi["pre_percentage"],
                                             "post_percentage": roi["post_percentage"],
                                             "total_synapses": roi["pre"] + roi["post"],
                                             "pre_synapses": roi["pre"],
                                             "post_synapses": roi["post"],
                                         }
-                                    )
+                                    if clean_name in ["ME", "LO", "LOP"]:
+                                        if (roi["pre_percentage"] >= threshold_high and (roi["pre"]+roi["post"])>50
+                                            or roi["post_percentage"] >= threshold_high and (roi["pre"]+roi["post"])>50):
+                                            include_in_scatter = 1
+                                        else:
+                                            include_in_scatter = 0
+                                        entry["incl_scatter"] = include_in_scatter
+                                    cleaned_roi_summary.append(entry)
 
                         roi_summary = cleaned_roi_summary
 
@@ -228,8 +237,9 @@ async def save_neuron_data_to_cache(
 
                         parent_rois = sorted(list(parent_rois_set))
 
-                        # Calculate spatial metrics for columns if column ROIs are present
-                        # Currently these are calculated from both L and R instances
+                        # Calculate spatial metrics for columns if column ROIs are present.
+                        # These metrics are calculated using synapses within the ROI from
+                        # both L and R instances.
                         for side in ["L", "R"]:
                             for region in ["ME", "LO", "LOP"]:
                                 str_pattern = f"{region}_{side}_col_"

src/neuview/services/index_service.py

@@ -56,17 +56,17 @@ async def create_index(self, command) -> Result[str, str]:
                 return Err(f"Output directory does not exist: {output_dir}")
 
             # Discover neuron types from cache or file scanning
-            neuron_types, scan_time = self._discover_neuron_types(output_dir)
+            neuron_types, scan_time = self.discover_neuron_types(output_dir)
             if not neuron_types:
                 return Err("No neuron type HTML files found in output directory")
 
             # Initialize connector if needed for database lookups
-            connector = await self._initialize_connector_if_needed(
+            connector = await self.initialize_connector_if_needed(
                 neuron_types, output_dir
             )
 
             # Correct neuron names (convert filenames back to original names)
-            corrected_neuron_types, cache_performance = self._correct_neuron_names(
+            corrected_neuron_types, cache_performance = self.correct_neuron_names(
                 neuron_types, connector
             )
 
@@ -95,7 +95,7 @@ async def create_index(self, command) -> Result[str, str]:
             logger.error(f"Failed to create optimized index: {e}")
             return Err(f"Failed to create index: {str(e)}")
 
-    def _discover_neuron_types(self, output_dir: Path) -> tuple:
+    def discover_neuron_types(self, output_dir: Path) -> tuple:
         """Discover neuron types from queue file to ensure all are included."""
         neuron_types = defaultdict(set)
 
@@ -209,7 +209,7 @@ def _discover_neuron_types(self, output_dir: Path) -> tuple:
                 )
                 return neuron_types, 0.0
 
-    async def _initialize_connector_if_needed(self, neuron_types, output_dir):
+    async def initialize_connector_if_needed(self, neuron_types, output_dir):
         """Initialize database connector only if needed for lookups."""
         # Pre-load ROI hierarchy from cache (no database queries if cached)
         roi_hierarchy_loaded = False
@@ -278,7 +278,7 @@ async def _initialize_connector_if_needed(self, neuron_types, output_dir):
 
         return connector
 
-    def _correct_neuron_names(self, neuron_types, connector):
+    def correct_neuron_names(self, neuron_types, connector):
         """Correct neuron names by converting filenames back to original names."""
         cached_data_lazy = (
             self.cache_manager.get_cached_data_lazy() if self.cache_manager else None