Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add spectrum plotting of data in the Live Viewer via Dask #2336

Open
wants to merge 46 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
a410321
Create DaskImageDataStack to hold delayed array of all images in LV path
MikeSullivan7 Aug 7, 2024
37088ad
add dask and dask-image to dependancies
MikeSullivan7 Aug 7, 2024
fd92b8d
live viewer model unit test fixes
MikeSullivan7 Aug 8, 2024
0890fee
yapf ruff fixes
MikeSullivan7 Aug 8, 2024
85a4695
cleanup
MikeSullivan7 Aug 8, 2024
85f699a
impliment toggle to create delayed dask array
MikeSullivan7 Aug 8, 2024
a09127b
added ability to read fits files via delayed dask functions
MikeSullivan7 Aug 8, 2024
71bb53f
remove astropy import and yapf fix
MikeSullivan7 Aug 8, 2024
09927e5
live viewer eyes tests fixes
MikeSullivan7 Aug 9, 2024
c7079b1
move all delayed array creation into DaskImageDataStack
MikeSullivan7 Aug 12, 2024
9be8efe
eyes test fixes
MikeSullivan7 Aug 13, 2024
93a146d
DaskImageDataStackTest created
MikeSullivan7 Aug 13, 2024
ed192d1
test_WHEN_create_delayed_array_THEN_delayed_array_created
MikeSullivan7 Aug 13, 2024
22a3534
create _get_fake_data() to test tif and fits files
MikeSullivan7 Aug 14, 2024
96ed759
test tif files handled correctly
MikeSullivan7 Aug 14, 2024
ff47f2d
testing unsupported files
MikeSullivan7 Aug 14, 2024
aeafeb9
testing supported files
MikeSullivan7 Aug 14, 2024
ccbe595
minor refactoring and using NotImplimentedError to check supported files
MikeSullivan7 Aug 14, 2024
ab1aa55
yapf and ruff fixes
MikeSullivan7 Aug 14, 2024
47acbf3
release note
MikeSullivan7 Aug 15, 2024
161dd1a
setting create_delayed_array to false falls back to loading into memory
MikeSullivan7 Aug 16, 2024
5099710
ruff fix
MikeSullivan7 Aug 16, 2024
b80b905
mypy fixes
MikeSullivan7 Aug 16, 2024
b0876c9
eyes_tests_fix
MikeSullivan7 Aug 16, 2024
b013797
Dask array now handles current image being deleted
MikeSullivan7 Aug 19, 2024
4c01e01
yapf fix
MikeSullivan7 Aug 19, 2024
7593ea5
Dask Stack now copes with file deletion
MikeSullivan7 Aug 20, 2024
64f3883
DaskImageDataStack allows images to be added and removed dynamically
MikeSullivan7 Aug 21, 2024
3c18f43
fixes to LV Model test
MikeSullivan7 Aug 22, 2024
23ce67c
fix get_computed_image
MikeSullivan7 Aug 22, 2024
3d4b575
mean is calculated as each image is added to DaskImageStack
MikeSullivan7 Aug 23, 2024
1785d74
mypy fix
MikeSullivan7 Aug 23, 2024
7eaf611
added split spectrum plot to LV window
MikeSullivan7 Sep 9, 2024
c08fb5a
live viewer spectrum plot toggles via right click menu
MikeSullivan7 Sep 10, 2024
54c185b
Spectrum calculation now toggled on and off by right click menu
MikeSullivan7 Sep 12, 2024
427ff4b
Live Viewer behaves correctly when spectrum is turned on with no data
MikeSullivan7 Sep 13, 2024
26737ed
dask slicing is optimised
MikeSullivan7 Sep 17, 2024
056f304
Using z slider is more fluid with Dask
MikeSullivan7 Sep 17, 2024
beacb41
dask mean calculation optimised
MikeSullivan7 Sep 18, 2024
7ec86f0
refactor and optimise how DaskImageDataStack updates its image lis
MikeSullivan7 Sep 19, 2024
7b9c47d
change how image_list updates and some benchmarking
MikeSullivan7 Sep 23, 2024
708344b
small optimization of mean calc and mypy fix
MikeSullivan7 Sep 27, 2024
94888a2
release note
MikeSullivan7 Sep 27, 2024
a14efd5
100 most recent images are stored in LRU cache
MikeSullivan7 Nov 22, 2024
c85406a
Images stored in LRUCache and new means append correctly (with debug …
MikeSullivan7 Nov 26, 2024
67a58c0
Mean buffer loading and prints removed
MikeSullivan7 Nov 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor and optimise how DaskImageDataStack updates its image lis
MikeSullivan7 committed Nov 20, 2024
commit 7ec86f0f2d72b6572dc011bf6a58f495b4a09f93
38 changes: 25 additions & 13 deletions mantidimaging/gui/windows/live_viewer/model.py
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ class DaskImageDataStack:
_selected_index: int
mean: np.ndarray = np.array([])
roi: SensibleROI | None = None
param_to_calc: list[str] = []

def __init__(self, image_list: list[Image_Data], create_delayed_array: bool = False):
self.image_list = image_list
@@ -123,12 +124,19 @@ def create_delayed_stack_from_image_data(self, image_list: list[Image_Data]) ->
f"{image_list[0].image_path.suffix.lower()}")
return delayed_stack

def update_delayed_stack(self, param_to_calc=None) -> None:
if param_to_calc is None:
param_to_calc = []
self.delayed_stack = self.create_delayed_stack_from_image_data(self.image_list)
#self.delayed_stack.visualize(filename=f'dask-update_delayed_stack-{self._selected_index}', format='png')
if 'mean' in param_to_calc:
def update_delayed_stack(self, new_image_list) -> None:
if self.delayed_stack is None:
self.delayed_stack = self.create_delayed_stack_from_image_data(new_image_list)
else:
new_images = [
image for image in new_image_list
if image.image_path not in [image.image_path for image in self.image_list]
]
#new_images = [image for image in new_image_list if image not in self.image_list]
self.delayed_stack = dask.optimize(
dask.array.concatenate([self.delayed_stack,
self.create_delayed_stack_from_image_data(new_images)]))[0]
if 'mean' in self.param_to_calc:
if len(self.mean) == len(self.image_list) - 1:
self.add_last_mean()
else:
@@ -137,15 +145,17 @@ def update_delayed_stack(self, param_to_calc=None) -> None:
else:
self.calc_mean_fully()

def update_image_list(self, new_image_list: list) -> None:
def update_image_list(self, new_image_list: list, update_stack: bool = True) -> None:
if update_stack and self.create_delayed_array:
self.update_delayed_stack(new_image_list)
self.image_list = new_image_list

def add_last_mean(self) -> None:
if self.delayed_stack is not None:
if self.roi:
left, top, right, bottom = self.roi
#mean_visual = dask.optimize(dask.array.mean(self.delayed_stack[-1, top:bottom, left:right]))[0].visualize(filename=f'dask-mean-add-task-opt-{self._selected_index}.png', format='png')
mean_to_add = dask.optimize(dask.array.mean(self.delayed_stack[-1, top:bottom, left:right]))[0].compute()
mean_to_add = dask.optimize(dask.array.mean(self.delayed_stack[-1, top:bottom,
left:right]))[0].compute()
self.mean = np.append(self.mean, mean_to_add)
else:
mean_to_add = dask.optimize(dask.array.mean(self.delayed_stack[-1]))[0].compute()
@@ -158,7 +168,6 @@ def calc_mean_fully(self) -> None:
def calc_mean_fully_roi(self):
if self.delayed_stack is not None:
left, top, right, bottom = self.roi
#mean_visual = dask.array.mean(self.delayed_stack[:, top:bottom, left:right], axis=(1, 2)).visualize(filename=f'dask-mean-full-task-{self._selected_index}.png', format='png')
self.mean = dask.array.mean(self.delayed_stack[:, top:bottom, left:right], axis=(1, 2)).compute()

def set_roi(self, roi: SensibleROI):
@@ -172,6 +181,9 @@ def delete_all_data(self):
def create_and_set_delayed_stack(self):
self.delayed_stack = self.create_delayed_stack_from_image_data(self.image_list)

def add_param_to_calc(self, param_name: str):
self.param_to_calc.append(param_name)


class Image_Data:
"""
@@ -367,6 +379,8 @@ def __init__(self, directory: Path):
self.sub_directories: dict[Path, SubDirectory] = {}
self.add_sub_directory(SubDirectory(self.directory))

self.image_stack.add_param_to_calc('mean')

def find_images(self, directory: Path) -> list[Image_Data]:
"""
Find all the images in the directory.
@@ -451,9 +465,7 @@ def _handle_directory_change(self) -> None:

self.image_stack.update_image_list(images)

if self.image_stack.create_delayed_array:
self.image_stack.update_delayed_stack(['mean'])
#self.image_stack.update_delayed_stack()
if 'mean' in self.image_stack.param_to_calc:
self.update_spectrum.emit(self.image_stack.mean)

self.update_recent_watcher(images[-1:])