Skip to content

Commit 152d79d

Browse files
author
Daisuke
committed
pipeline for mightex data
Proven to work from office PC for susanoo/2024-11-22_1/exp5&6 Also mightex data transfer script (uploadMtx.m) proven at timeline pc
1 parent 6f4290a commit 152d79d

File tree

8 files changed

+299
-3
lines changed

8 files changed

+299
-3
lines changed

‎setPath_analysisImaging.m

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
addpath(genpath('C:\Users\dshi0006\git\visbox')); %addpath(genpath('C:\Users\Experiment\Documents\MATLAB\visbox'));
2+
addpath(genpath('C:\Users\dshi0006\git\analysisImaging')); %%addpath(genpath('\\ad.monash.edu\home\User006\dshi0006\Documents\MATLAB\master\Analysis\wfAnalysis\widefield'));
3+
addpath(genpath('C:\Users\dshi0006\git\npy-matlab')); %addpath(genpath('C:\npy-matlab'));
4+
addpath(genpath('C:\Users\dshi0006\git\dsbox'));%addpath(genpath('\\ad.monash.edu\home\User006\dshi0006\Documents\MATLAB\dsbox')); %screen2png
5+
%addpath('\\ad.monash.edu\home\User006\dshi0006\Documents\MATLAB\master\Analysis\wfAnalysis');

‎wfAnalysis/daily_pipeline/amberOps.mat

785 Bytes
Binary file not shown.

‎wfAnalysis/daily_pipeline/runSVDKTmtx.m

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
%%
2+
% This script requires:
3+
% Timeline data uploaded to master server
4+
%
5+
% This script does:
6+
%
7+
% apply SVD
8+
% visualize SVD results (inspectSVDresult.m)
9+
% check if #frames are identical that recorded in TL (in saveSVD.m)
10+
% split V into each experiment (in saveSVD.m)
11+
% save U and V to the market server (in saveSVD.m)
12+
13+
% things to do before running this script:
14+
% 1, upload raw data to Market
15+
% 2, download the raw data under E:\Subjects
16+
17+
clear all
18+
setPath_analysisImaging;
19+
20+
%cd('\\ad.monash.edu\home\User006\dshi0006\Documents\MATLAB\master\Analysis\wfAnalysis\daily_pipeline');
21+
cd('C:\Users\dshi0006\git\analysisImaging\wfAnalysis\daily_pipeline');
22+
load('amberOps.mat');
23+
% For blue/purple alternate
24+
%load('bluePurpleOps.mat');
25+
% For purple only
26+
% load('C:\Users\Experiment\Documents\MATLAB\purpleOps.mat')
27+
mouseName = 'susanoo';
28+
thisDate = '2024-11-22'; %[datestr(now,'yyyy-mm-dd')];
29+
thisSeries = 1;
30+
expNums = 5;%[1:4];
31+
hwbinning = 1; %automatically retrieve this from thorcam header??
32+
magnification = .5;
33+
makeROI = false; %if false, use already saved ROI from the save subject (thisROI.mat)
34+
doRegistration = 1; %15/10/20
35+
36+
37+
%where vidXraw.dat and vidXreg.dat are created (subsequently moved to the data server)
38+
%rootDrive = 'C:\svdinput'; %NG ... too small
39+
rootDrive = 'E:\svdinput';
40+
%will be used in;
41+
%ops.localSavePath
42+
43+
%where raw data is temporally downloaded must be under
44+
%rawDataDir/(animal)/(session)/(expNum)
45+
%rawDataDir = '\\vault-v2.erc.monash.edu.au\MNHS-dshi0006\Subjects';%if the raw data is already uploaded to the server
46+
%rawDataDir = 'X:\Subjects'; %market server ... too slow to load
47+
rawDataDir = 'E:\Subjects'; %local temporary storage
48+
49+
50+
%where SVD and summary data is saved
51+
%dat.expFilePath(ops.mouseName, thisDate, thisSeries, expNums, 'widefield','master')
52+
53+
%where the raw and processed data is saved
54+
%dat.expFilePath(ops.mouseName, thisDate, thisSeries, expNums, 'widefield_raw','master')
55+
56+
57+
%% modify ops for this experiment
58+
%hack for 9/6/20 phpL
59+
% ops.vids(1).frameMod = [2 0];
60+
% ops.vids(2).frameMod = [2 1];
61+
%ops.vids = ops.vids(1);%3/7/20
62+
63+
thisDateSeries = [thisDate, '_' num2str(thisSeries)];
64+
if ~makeROI
65+
if exist(fullfile(rawDataDir, mouseName,'thisROI.mat'),'file')
66+
load(fullfile(rawDataDir, mouseName,'thisROI.mat'),'thisROI');
67+
ops.roi = thisROI;
68+
else
69+
ops.roi = [];
70+
end
71+
end
72+
73+
ops.mouseName = mouseName;
74+
ops.thisDate = thisDateSeries;
75+
%if using inclExpList, specify ops.fileBase
76+
ops.inclExpList = expNums;
77+
ops.fileBase = fullfile(rawDataDir, mouseName, thisDateSeries);
78+
ops.statusDestination = fopen('test.txt','w');
79+
ops.userName = 'Daisuke';
80+
ops = rmfield(ops,'emailAddress');
81+
ops.rigName = 'alloptrig';%'wfrig'; %used in determineTimelineAlignments
82+
ops.doRegistration = doRegistration;
83+
ops.useGPU = 1; %used only for registration?
84+
ops.objectiveType = num2str(magnification); %0.5 / 0.8
85+
ops.pixelSizeUM = 5/str2num(ops.objectiveType)*hwbinning; %added 15/5/20
86+
ops.hasASCIIstamp = 0; %18/5/20
87+
88+
89+
for e = 1:length(expNums)
90+
expRefs{e} = dat.constructExpRef(mouseName, thisDate, thisSeries, expNums(e));
91+
end
92+
ops.expRefs = expRefs;
93+
94+
% retrieve camera sampling rate and exposure duration from Timeline
95+
timelinePath = dat.expFilePath(expRefs{1}, 'timeline', 'master');
96+
load(timelinePath);
97+
[strobeOnTimes, ~, strobeDurs] = getStrobeTimes(Timeline, ops.rigName);
98+
exposureDur = median(strobeDurs); %[s]
99+
FsPerColor = 1/median(diff(strobeOnTimes))/length(ops.vids);
100+
ops.Fs = FsPerColor;%16/10/20
101+
clear Timeline
102+
103+
104+
for i = 1:length(ops.vids)
105+
%where video data is stored
106+
ops.vids(i).fileBase = fullfile(rawDataDir, mouseName, thisDateSeries);
107+
ops.vids(i).exposureDur = exposureDur;
108+
ops.vids(i).Fs = FsPerColor;
109+
end
110+
%% for single color imaging
111+
% ops.vids = ops.vids(1);
112+
% ops.vids.frameMod = [1 0];
113+
114+
115+
% create ROI
116+
if makeROI
117+
theseFiles = generateFileList(ops, 1);
118+
119+
imageForROI = mean(loadTiffStack(theseFiles{1}, 'tiffobj',0),3); %make purple and blue image separately??
120+
imagesc(imageForROI);
121+
axis equal tight;
122+
colormap(gray);
123+
caxis(prctile(imageForROI(:),[1 99]));
124+
%colormouse;%change the colormap range interactively using the mouse
125+
title('left click to put anchor points., double click to finish');
126+
roiAhand = images.roi.AssistedFreehand;
127+
draw(roiAhand);
128+
ops.roi = createMask(roiAhand);
129+
close;
130+
131+
%22/7/20 for use later
132+
thisROI=ops.roi;
133+
save(fullfile(rawDataDir, mouseName,'thisROI.mat'),'thisROI');
134+
else %reuse ROI created previously.
135+
load(fullfile(rawDataDir, mouseName,'thisROI.mat'));
136+
ops.roi = thisROI;
137+
end
138+
139+
140+
%where SVD data is saved
141+
ops.localSavePath = fullfile(rootDrive, 'data', mouseName, thisDate);
142+
143+
if ~exist(ops.localSavePath,'dir')
144+
mkdir(ops.localSavePath); %7/5/20
145+
end
146+
cd(ops.localSavePath)
147+
save ops.mat ops
148+
149+
150+
%% run SVD, save results to server, erase original data
151+
pipelineHereKT();
152+
153+
154+
155+
156+
%%
157+
% mouseName = 'Erlanger';
158+
% thisDate = datestr(now, 'yyyy-mm-dd');
159+
% expNums = [2 3];
160+
%
161+
%
162+
% addpath(genpath('C:\Users\Experiment\Documents\GitHub\widefield'))
163+
% addpath(genpath('\\zserver.cortexlab.net\Code\Rigging\main'));
164+
% addpath(genpath('\\zserver.cortexlab.net\Code\Rigging\cb-tools'));
165+
%
166+
% s = svdVid.listen;
167+
%
168+
% s.ops.mouseName = mouseName;
169+
% s.ops.thisDate = thisDate;
170+
%
171+
% s.wizard
172+
%
173+
% s.ops.vids(1).fileBase = fullfile('J:\data', mouseName, thisDate);
174+
% s.ops.vids(2).fileBase = fullfile('J:\data', mouseName, thisDate);
175+
% s.ops.vids(1).rigName = 'kilotrode';
176+
% s.ops.vids(2).rigName = 'kilotrode';
177+
% s.ops.useGPU = true;
178+
%
179+
% for e = 1:length(expNums)
180+
% clear ed
181+
% ed.expRef = dat.constructExpRef(mouseName, thisDate, expNums(e));
182+
% s.addExpDat(ed)
183+
% end
184+
%
185+
% s.ops.localSavePath = fullfile('J:\data', mouseName, thisDate);
186+
%
187+
% cd(fullfile('J:\data', mouseName, thisDate))
188+
% ops = s.ops;
189+
% save ops.mat ops
190+
% pipelineHereKT

‎wfAnalysis/daily_pipeline/uploadMtx.m

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
function [result, mtxDir, tlDir] = uploadMtx(animal, session, expNumber, mightexDirectory)
2+
% copty mightex experiment data (camera and DMD) to the directory where
3+
% timeline data is saved
4+
% then rename tifstack so the pipeline can run
5+
6+
p = dat.paths;
7+
8+
if nargin < 4
9+
mightexDirectory = 'C:\Users\experiment\Documents\Mightex\PolyScan3\User Data\Daisuke_Shimaoka\DMDcamera\20241029_Daisuke_Shimaoka';
10+
end
11+
12+
if isnumeric(expNumber)
13+
expNumber = num2str(expNumber);
14+
end
15+
16+
%% detect date & time of a specified experiment number from Timeline data
17+
%timeline data:
18+
%C:\LocalExpData\susanoo\2024-11-22_1\6\2024-11-22_1_6_susanoo_Timeline.mat
19+
% mtx data: C:\Users\experiment\Documents\Mightex\PolyScan3\User Data\Daisuke_Shimaoka\DMDcamera\20241029_Daisuke_Shimaoka\20241122_191702
20+
tlDir = fullfile(p.localRepository, animal, session, expNumber);
21+
%tlInfo = dir(fullfile(tlDir, [session '_' expNumber '_' animal '_Timeline.mat']));
22+
%tlTimeStamp = tlInfo.datenum;
23+
dtl = System.IO.File.GetCreationTime(tlDir);
24+
tlTimeStamp = datenum(double([dtl.Year, dtl.Month, dtl.Day, dtl.Hour, dtl.Minute, dtl.Second]));
25+
26+
mtxCandidateInfo = dir(mightexDirectory);
27+
mtxCandidateInfo = mtxCandidateInfo([mtxCandidateInfo.isdir]);
28+
mtxCandidateInfo = mtxCandidateInfo( ~ismember({mtxCandidateInfo.name}, {'.','..'}));
29+
%mtxCandidateTimeStamp = [mtxCandidateInfo.datenum]; %NG last access date, not creation date
30+
mtxCandidateTimeStamp = nan(numel(mtxCandidateInfo),1);
31+
for idir = 1:numel(mtxCandidateInfo)
32+
d = System.IO.File.GetCreationTime(fullfile(mtxCandidateInfo(idir).folder, ...
33+
mtxCandidateInfo(idir).name, 'SessionEvents.dat'));
34+
mtxCandidateTimeStamp(idir) = datenum(double([d.Year, d.Month, d.Day, d.Hour, d.Minute, d.Second]));
35+
end
36+
37+
[~, tgtDirIdx] = min(abs(mtxCandidateTimeStamp - tlTimeStamp)); %may not be robust
38+
if isempty(tgtDirIdx)
39+
error('Could not find directory where mightex data has been saved');
40+
end
41+
42+
%% identify the mightex directory corresponding to the experiment
43+
mtxInfo = mtxCandidateInfo(tgtDirIdx);
44+
mtxDir = fullfile(mtxInfo.folder, mtxInfo.name);
45+
disp(['Mightex data @: ' mtxDir]);
46+
47+
%% check if a tifstack has been created (if not, create here?)
48+
tifStackInfo = dir(fullfile(mtxDir, '*.tif'));
49+
tifStackInfo = tifStackInfo([~tifStackInfo.isdir]);
50+
if isempty(tifStackInfo)
51+
error('Could not find a tifstack');
52+
else
53+
originalTif = fullfile(tifStackInfo.folder, tifStackInfo.name);
54+
end
55+
56+
%% rename the tiffstack to (expnumber).tif
57+
%renamedTif = fullfile(tlPath, [expNumber '.tif']);
58+
renamedTif = fullfile(tifStackInfo.folder, [expNumber '.tif']);
59+
if ~isequal(originalTif, renamedTif)
60+
result = movefile(originalTif, renamedTif);
61+
if ~result
62+
error(['Could not rename the tifstack to ' [expNumber '.tif']]);
63+
end
64+
end
65+
66+
%% copy the tifstack and other mightx data
67+
result = copyfile(mtxDir, tlDir);
68+
if result
69+
disp(['Mightex data successfully copied to ' tlDir]);
70+
elseif ~result
71+
error(['Could not copy mightex data to ' tlDir]);
72+
end
73+

‎widefield/core/loadRawToDat.m

+3-2
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,9 @@
195195
if ops.verbose && extraVerbose
196196
fprintf(1, ' computing mean image\n');
197197
end
198-
sumImage = sumImage+sum(double(imstack),3);
199-
198+
% sumImage = sumImage+sum(double(imstack),3);
199+
sumImage = sumImage+sum(imstack,3); %9/1/25
200+
200201
if ops.verbose
201202
fprintf(1, ' saving to dat\n');
202203
end

‎widefield/generalUtils/fileHandling/readOneMtx.m

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
2+
3+
function thisFrame = readOneMtx(filename)
4+
5+
fid = fopen(filename, 'r');
6+
if fid == -1, error(['Unable to open file: ' filename]); end
7+
nRows = fread(fid, 1, 'uint32'); % spatial resolution, n rows
8+
nCols = fread(fid, 1, 'uint32'); % spatial resolution, n columns
9+
bitDepth = fread(fid, 1, 'uint32');
10+
nFrames = 1;%fread(fid, 1, 'uint32');
11+
fclose(fid);
12+
13+
format = { ...
14+
'uint32' 1 'nRows';... % Number of rows.
15+
'uint32' 1 'nCows';... % Number of columns.
16+
'uint32' 1 'nFrames';... % Number of frames.
17+
'double' 1 'startTime';... % Time of first frame in UNIX format.
18+
'uint16' [nRows nCols nFrames] 'imagedata';... % Image data.
19+
'double', [nFrames 1], 't'}; % Time of each frame in seconds.
20+
m = memmapfile(filename, 'Format', format);
21+
22+
thisFrame = m.Data(1).imagedata;
23+
24+
end

‎widefield/generalUtils/getStrobeTimes.m

+3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@
2727
case 'wfrig' %8/5/20
2828
strobeName = 'camExposure';
2929
strobeThresh = 2;
30+
case 'alloptrig' %09/01/25
31+
strobeName = 'PCObusy';
32+
strobeThresh = 2;
3033
otherwise
3134
error('getStrobeTimes doesn''t recognize rig name %s', rigName);
3235
end

‎widefield/scripts/pipelineHereKT.m

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ function pipelineHereKT()
1717
close all;
1818

1919
showFig = 1;
20-
saveDat = 0; %if 1, save dat files into the Vault server, else delete them
20+
saveDat = 1; %if 1, save dat files into the Vault server, else delete them
2121

2222
%serverDir = '\\ad.monash.edu\home\User006\dshi0006\Documents\tempDataServer'; %7/5/20
2323
%'\\lugaro.cortexlab.net\bigdrive\staging\';

0 commit comments

Comments
 (0)