diff --git a/+deepinterp/+Net/ImageTimeSeries.m b/+deepinterp/+Net/ImageTimeSeries.m deleted file mode 100644 index b966a47..0000000 --- a/+deepinterp/+Net/ImageTimeSeries.m +++ /dev/null @@ -1,137 +0,0 @@ -classdef ImageTimeSeries < deepinterp.Net - - properties (GetAccess=public, SetAccess=protected) - N % number of rows of inputs to the network - M % number of columns of inputs to the network - Npre % Number of input frames needed before the predicted frame - Npost % Number of input frames needed after the predicted frame - end - - - methods - - function obj = ImageTimeSeries(command, options) - % OBJ = NET(COMMAND,...) - % - % Create a new deepinterp.Net.ImageTimeSeries object. - % - % deepinterp.Net.ImageTimeSeries objects remove independent - % frame-by-frame noise in image sequences that are usually time - % series. The network takes Npre video frames _before_ the frame - % to be denoised, and Npost video frames _after_ the frame to be - % denoised, to make a computational prediction of the denoised - % frame. - % - % COMMAND can be: - % - % 'New' : create a new empty network - % 'KerasFile': open and import a Keras file - % - % The function also accepts additional arguments as name/value pairs: - % -------------------------------------------------------------------- - % | Parameter (default) | Description | - % |-------------------------|----------------------------------------| - % | file ('') | Filename to open if reading a file | - % | N (512) | Number of rows in images if new network| - % | M (512) | Number of columns in images for new net| - % | Npre (30) | Number of frames before prediction | - % | Npost (30) | Number of frames after prediction | - % |-------------------------|----------------------------------------| - % - % Example: - % sampleNet = deepinterp.sampleFile(... - % '2019_09_11_23_32_unet_single_1024_mean_absolute_error_Ai93-0450.h5'); - % n = deepinterp.Net.ImageTimeSeries('KerasFile','file',sampleNet,... - % 'Npre',30,'Npost',30); - % - arguments - command (1,:) char {mustBeTextScalar} - options.file (1,:) char {mustBeFile} - options.N (1,1) double {mustBeInteger} = 512; - options.M (1,1) double {mustBeInteger} = 512; - options.Npre (1,1) double {mustBeInteger} = 30; - options.Npost (1,1) double {mustBeInteger} = 30; - end - - superOptions = rmfield(options,{'N','M','Npre','Npost'}); - superInputs = cat(2,{command},... - deepinterp.internal.struct2namevaluepair(superOptions)); - obj = obj@deepinterp.Net(superInputs{:}); - obj.Npre = options.Npre; - obj.Npost = options.Npost; - - if strcmp(command,'New'), - disp(['In future we will build a net network here.']); - end; - - obj = obj.SetInputSize(); - - end; % ImageTimeSeries - constructor - - function obj = SetInputSize(obj) - % SETINPUTSIZE - set the input size properties - % - % OBJ = SETINPUTSIZE(OBJ) - % - % Sets the input size properties N and M, and verifies that - % the number of image sequences required in each input add up to - % obj.Npre and obj.Npost. - % - % The values can be examined by OBJ.N, OBJ.M. - % - if isempty(obj.network), - obj.N = NaN; - obj.M = NaN; - else, - inSz = obj.network.Layers(1).InputSize; - obj.N = inSz(1); - obj.M = inSz(2); - assert(inSz(3)==obj.Npre+obj.Npost,... - ['Npre + Npost must add up to the total ' ... - 'number of image inputs to ' ... - 'deepinterp.net.ImageTimeSeries.']); - end; - end; % SetInputSize - - function output = interp(obj, input, options) - % INTERP - apply DeepInterpolation to inputs - % - % OUTPUT = INTERP(DEEPINTERP_IMAGETIMESERIES_OBJ, INPUT) - % - % Apply the DeepInterpolation network to the INPUT data. - % INPUT should be an NxMxT matrix, where N and M match - % the size of the DEEPINTERP_IMAGETIMESERIES_OBJ.N and .M parameters, - % and T must be greater than the Npre + Npost property values of - % DEEPINTERP_IMAGETIMESERIES_OBJ. Interpolation will only be performed - % for frames greater than Npre and less than numel(T) - Npost. The first - % Npre frames and last Npost frames will be equal to the input. - % - arguments - obj (1,1) - input - options.progbar (1,1) logical = true; - end - - output = input; - offsets = [-obj.Npre:-1 1:obj.Npost]; - if options.progbar, - deepinterp.internal.progressbar(); - end; - totalWork = size(input,3)-(obj.Npost+obj.Npre); - for t = obj.Npre+1 : size(input,3) - obj.Npost, - if options.progbar, - deepinterp.internal.progressbar((t-(obj.Npre+1))/totalWork); - end; - output(:,:,t) = predict(obj.network,input(:,:,offsets+t)); - end; - if options.progbar, - progressbar(0.9999999999); - end; - end; % interp() - - % INTERP - apply DeepInterpolation - - - end; % methods - -end % classdef diff --git a/internal/fmriDenoiserNetwork.m b/+deepinterp/+internal/fmriDenoiserNetwork.m similarity index 100% rename from internal/fmriDenoiserNetwork.m rename to +deepinterp/+internal/fmriDenoiserNetwork.m diff --git a/+deepinterp/+internal/getPretrainedModelFilename.m b/+deepinterp/+internal/getPretrainedModelFilename.m new file mode 100644 index 0000000..09e9b12 --- /dev/null +++ b/+deepinterp/+internal/getPretrainedModelFilename.m @@ -0,0 +1,48 @@ +function [filename,parameters] = getPretrainedModelFilename(modelName) +% GETPRETRAINEDMODELFILENAME - get the file name and parameters for a pretrained model +% +% [FILENAME, PARAMETERS] = GETPRETRAINEDMODELFILENAME(MODELNAME) +% +% Returns the FILENAME and the pretrained model PARAMETERS for a given MODELNAME. +% +% Alternatively, one may call the function with no input arguments for a list +% of valid MODELNAME entries: +% +% [ALLMODELNAMES] = GETPRETRAINEDMODELFILENAME() +% +% Example: +% [filename,parameters] = deepinterp.internal.getPretrainedModelFilenam('TP-Ai93-450'); +% + +filename = []; +parameters = []; + +[allModelNames,allModelParameters] = deepinterp.getPretrainedModels(); + +if nargin<1, + filename = allModelNames; + return; +end; + +idx = find(strcmpi(modelName,allModelNames)); + +if isempty(idx), + error(['No match for ' modelName ' in known models.']); +end; + +parameters = allModelParameters(idx); + +filename = fullfile(deepinterp.toolboxpath,'pretrainedModels',parameters.filename); + +if ~isfile(filename), + % try to download + FileURL = parameters.url; + disp(['Downloading pretrained model...']); + try, + websave(filename,FileURL); + catch, + error(['Error downloading model.']); + throw(ex); + end; +end; + diff --git a/+deepinterp/+internal/importKerasMAE.m b/+deepinterp/+internal/importKerasMAE.m index 71f916c..f56aaa7 100644 --- a/+deepinterp/+internal/importKerasMAE.m +++ b/+deepinterp/+internal/importKerasMAE.m @@ -11,9 +11,10 @@ importednet = importKerasLayers(kerasFile,'ImportWeights',true); placeholders = findPlaceholderLayers(importednet); +if ~isempty(placeholders), + importednet = replaceLayer(importednet, placeholders.Name , ... + deepinterp.internal.maeRegressionLayer); +end; -regressionnet = replaceLayer(importednet, placeholders.Name , ... - deepinterp.maeRegressionLayer); - -net = assembleNetwork(regressionnet); +net = assembleNetwork(importednet); diff --git a/+deepinterp/+internal/textfile2char.m b/+deepinterp/+internal/textfile2char.m new file mode 100644 index 0000000..8c8a074 --- /dev/null +++ b/+deepinterp/+internal/textfile2char.m @@ -0,0 +1,16 @@ +function str = textfile2char(filename) +% TEXTFILE2CHAR - Read a text file into a character string +% +% STR = TEXTFILE2CHAR(FILENAME) +% +% This function reads the entire contents of the file FILENAME into +% the character string STR. +% + +fid = fopen(filename,'rt'); +if fid>0, + str = char(fread(fid,Inf))'; + fclose(fid); +else, + error(['Could not open text file named ' filename '.']); +end; diff --git a/+deepinterp/Net.m b/+deepinterp/Net.m index 46ca67f..8c3a30b 100644 --- a/+deepinterp/Net.m +++ b/+deepinterp/Net.m @@ -2,6 +2,13 @@ properties (SetAccess=protected) network % Deep learning network + network_type % The type of network to create + N % number of rows of inputs to the network + M % number of columns of inputs to the network + Npre % Number of input frames needed before the predicted frame + Npost % Number of input frames needed after the predicted frame + Nomit % Number of omitted frames for the prediction + model_parameters % Parameters of the model, if applicable end @@ -12,35 +19,76 @@ % OBJ = NET(COMMAND,...) % % Create a new deepinterp.Net object according to instructions. - % COMMAND can be: + % COMMAND can be any of the following: % - % 'KerasFile': open and import a Keras file - % 'New' : create a new empty network + % 'New' : create a new network (under development) + % 'KerasFile' : open and import a Keras file + % 'Pretrained' : Open a pretrained model in DeepInterpolation % % The function also accepts additional arguments as name/value pairs: % -------------------------------------------------------------------- % | Parameter (default) | Description | % |-------------------------|----------------------------------------| % | file ('') | Filename to open if reading a file | - % |-------------------------|----------------------------------------| + % | type ('two-photon') | Type of data the network operates on | + % | | options are 'two-photon', 'ephys', | + % | | 'fMRI', 'other'. | + % | model ('TP-Ai93-450') | When a pretrained model is requested, | + % | | use this named model. For a list of | + % | | available models, use | + % | | deepinterp.getPretrainedModels() | + % | N (512) | N (first dimension of input) | + % | M (512) | M (second dimension of input) | + % | Nomit (1) | Number of frames to omit in prediction | + % | Npre (30) | Number of frames before the predicted | + % | | frame that are used to generate the | + % | | prediction | + % | Npost (30) | Number of frames after the predicted | + % | | frame that are used to generate the | + % | | prediction | + % |-------------------------|----------------------------------------| % - % Example: - % n = deepinterp.Net('KerasFile','file','myKerasFile.H5'); + % Examples: + % n1 = deepinterp.net('Pretrained','model','TP-Ai93-450'); + % n2 = deepinterp.Net('KerasFile','file','myKerasFile.H5'); % arguments command (1,:) char {mustBeTextScalar} options.file (1,:) char {mustBeFile} - end + options.type (1,:) char {mustBeMember(options.type,{'two-photon','ephys','fMRI','other'})} = 'two-photon' + options.model (1,:) char {mustBeTextScalar} = 'TP-Ai93-450' + options.N (1,1) double {mustBeInteger} = 512 + options.M (1,1) double {mustBeInteger} = 512 + options.Npre (1,1) double {mustBeInteger} = 30 + options.Npost (1,1) double {mustBeInteger} = 30 + options.Nomit (1,1) double {mustBeInteger} = 1 + end + + network_type = options.type; - switch (command), - case 'KerasFile', + switch (lower(command)), + case 'new', + % nothing to do yet + error(['NEW option still under development.']); + case 'kerasfile', obj.network=deepinterp.internal.importKerasMAE(options.file); - case 'New', - % nothing to do + case 'pretrained', + [modelfilename, modelparams] = deepinterp.internal.getPretrainedModelFilename(options.model); + obj = deepinterp.Net(modelparams.format,'file',modelfilename,... + 'N',modelparams.dim(1),'M',modelparams.dim(2),... + 'Npre',modelparams.pre,'Npost',modelparams.post,'Nomit',modelparams.omit,... + 'type',modelparams.type); + return; otherwise, error(['Unknown command: ' command]); end; + obj.network_type = network_type; + obj.Npre = options.Npre; + obj.Npost = options.Npost; + obj.Nomit = options.Nomit; + obj = obj.SetInputSize(); + end; % Net constructor function net_obj = train(net_obj, inputs, outputs) @@ -54,22 +102,74 @@ % Subclasses should describe the form of these inputs % and outputs. % - + error('Training function in the object framework is not yet developed.'); end; % train() - function output = interp(net_obj, inputs) - % INTERP - apply DeepInterpolation + function output = interp(obj, input, options) + % INTERP - apply DeepInterpolation to inputs % - % OUTPUT = INTERP(NET_OBJ, INPUT) + % OUTPUT = INTERP(DEEPINTERP_NET_OBJ, INPUT) % - % Apply the DeepInterpolation network to the - % INPUT data. - % - % Subclasses should describe the form of the input - % and output. + % Apply the DeepInterpolation network to the INPUT data. + % INPUT should be an NxMxT matrix, where N and M match + % the size of the DEEPINTERP_NET_OBJ.N and .M parameters, + % and T must be greater than the Npre + Npost property values of + % DEEPINTERP_IMAGETIMESERIES_OBJ. Interpolation will only be performed + % for frames greater than Npre and less than numel(T) - Npost. The first + % Npre frames and last Npost frames will be equal to the input. % + arguments + obj (1,1) + input + options.progbar (1,1) logical = true; + end + output = input; + Nomit_pre = (obj.Nomit+1)/2; + Nomit_post = (obj.Nomit+1)/2; + offsets = [ fliplr(-[Nomit_pre:obj.Npre+Nomit_pre-1]) Nomit_post:obj.Npost+Nomit_post-1]; + if options.progbar, + deepinterp.internal.progressbar(); + end; + totalWork = size(input,3)-(obj.Npost+obj.Npre); + for t = obj.Npre+1 : size(input,3) - obj.Npost, + if options.progbar, + deepinterp.internal.progressbar((t-(obj.Npre+1))/totalWork); + end; + output(:,:,t) = predict(obj.network,input(:,:,offsets+t)); + end; + if options.progbar, + progressbar(0.9999999999); + end; end; % interp() + + function obj = SetInputSize(obj) + % SETINPUTSIZE - set the input size properties + % + % OBJ = SETINPUTSIZE(OBJ) + % + % Sets the input size properties N and M, and verifies that + % the number of image sequences required in each input add up to + % obj.Npre and obj.Npost. + % + % The values can be examined by OBJ.N, OBJ.M. + % + if isempty(obj.network), + obj.N = NaN; + obj.M = NaN; + else, + inSz = obj.network.Layers(1).InputSize; + obj.N = inSz(1); + obj.M = inSz(2); + assert(inSz(3)==obj.Npre+obj.Npost,... + ['Npre + Npost must add up to the total ' ... + 'number of image inputs to ' ... + 'deepinterp.net.ImageTimeSeries.']); + end; + end; % SetInputSize + + + end; end % classdef diff --git a/+deepinterp/getPretrainedModels.m b/+deepinterp/getPretrainedModels.m new file mode 100644 index 0000000..715c140 --- /dev/null +++ b/+deepinterp/getPretrainedModels.m @@ -0,0 +1,17 @@ +function [modelNames,modelParams] = getPretrainedModels() +% GETPRETRAINEDMODELS - return known, built-in pretrained models +% +% [MODELNAMES, MODELPARAMS] = GETPRETRAINEDMODELS() +% +% Return a list of known MODELNAMES and full parameter information +% about these models in MODELPARAMS. +% + +filename = fullfile(deepinterp.toolboxpath,'pretrainedModels','pretrained.json'); + +the_text = deepinterp.internal.textfile2char(filename); + +modelParams = jsondecode(the_text); + +modelNames = {modelParams.name}; + diff --git a/+deepinterp/setup.m b/+deepinterp/setup.m index aaf4c2c..43294b7 100644 --- a/+deepinterp/setup.m +++ b/+deepinterp/setup.m @@ -9,9 +9,8 @@ function setup() tbd = deepinterp.toolboxpath(); addpath(tbd); -addpath(fullfile(tbd, "network_layers")) addpath(fullfile(tbd, "sampleData")) +addpath(fullfile(tbd, "pretrainedModels")) addpath(fullfile(tbd, "examples")) -addpath(fullfile(tbd, "internal")) addpath(fullfile(tbd, "datastore")) diff --git a/examples/customdatastore_example.mlx b/examples/customdatastore_example.mlx index c8dedfe..8a8e889 100644 Binary files a/examples/customdatastore_example.mlx and b/examples/customdatastore_example.mlx differ diff --git a/examples/small_ophys_inference.mlx b/examples/small_ophys_inference.mlx deleted file mode 100644 index 4f1ad81..0000000 Binary files a/examples/small_ophys_inference.mlx and /dev/null differ diff --git a/examples/tiny_ephys_inference.mlx b/examples/tiny_ephys_inference.mlx index 5f38fbc..2390b44 100644 Binary files a/examples/tiny_ephys_inference.mlx and b/examples/tiny_ephys_inference.mlx differ diff --git a/examples/tiny_ephys_training.mlx b/examples/tiny_ephys_training.mlx index cd5720f..4ab664d 100644 Binary files a/examples/tiny_ephys_training.mlx and b/examples/tiny_ephys_training.mlx differ diff --git a/examples/tiny_fMRI_inference.mlx b/examples/tiny_fMRI_inference.mlx index 056c3ef..d0b6f72 100644 Binary files a/examples/tiny_fMRI_inference.mlx and b/examples/tiny_fMRI_inference.mlx differ diff --git a/examples/tiny_ophys_inference.mlx b/examples/tiny_ophys_inference.mlx new file mode 100644 index 0000000..947c06d Binary files /dev/null and b/examples/tiny_ophys_inference.mlx differ diff --git a/sampleData/2020_02_29_15_28_unet_single_ephys_1024_mean_squared_error-1050.h5 b/pretrainedModels/2020_02_29_15_28_unet_single_ephys_1024_mean_squared_error-1050.h5 similarity index 100% rename from sampleData/2020_02_29_15_28_unet_single_ephys_1024_mean_squared_error-1050.h5 rename to pretrainedModels/2020_02_29_15_28_unet_single_ephys_1024_mean_squared_error-1050.h5 diff --git a/sampleData/2021_07_31_09_49_38_095550_unet_1024_search_mean_squared_error_pre_30_post_30_feat_32_power_1_depth_4_unet_True-0125-0.5732.h5 b/pretrainedModels/2021_07_31_09_49_38_095550_unet_1024_search_mean_squared_error_pre_30_post_30_feat_32_power_1_depth_4_unet_True-0125-0.5732.h5 similarity index 100% rename from sampleData/2021_07_31_09_49_38_095550_unet_1024_search_mean_squared_error_pre_30_post_30_feat_32_power_1_depth_4_unet_True-0125-0.5732.h5 rename to pretrainedModels/2021_07_31_09_49_38_095550_unet_1024_search_mean_squared_error_pre_30_post_30_feat_32_power_1_depth_4_unet_True-0125-0.5732.h5 diff --git a/pretrainedModels/fmri_pretrained_matlab_compatible.mat b/pretrainedModels/fmri_pretrained_matlab_compatible.mat new file mode 100644 index 0000000..9e53870 Binary files /dev/null and b/pretrainedModels/fmri_pretrained_matlab_compatible.mat differ diff --git a/pretrainedModels/pretrained.json b/pretrainedModels/pretrained.json new file mode 100644 index 0000000..b7ff6b3 --- /dev/null +++ b/pretrainedModels/pretrained.json @@ -0,0 +1,67 @@ +[ + { + "name":"TP-Ai93-450", + "filename":"2019_09_11_23_32_unet_single_1024_mean_absolute_error_Ai93-0450.h5", + "url":"https://wds-matlab-community-toolboxes.s3.amazonaws.com/DeepInterpolation-MATLAB/Trained_models/ophys/2019_09_11_23_32_unet_single_1024_mean_absolute_error_Ai93-0450.h5", + "dim":[512,512], + "pre":30, + "post":30, + "omit":1, + "type":"two-photon", + "format": "KerasFile", + "parameters": + { + "framerate":30, + "objective_power":16, + "objective_NA":0.8, + "indicator":"GCaMP6f", + "strain":"Ai148", + "training_data":"https://github.com/AllenInstitute/deepinterpolation/blob/cfb42bc2c5bd8cc08ee870a163230e8c89c676ba/examples/paper_generation_code/json_data/2019-09-05-train-very-large-single-plane-Ai148-norm.json", + "description":"Pre-processing: Individual movies were motion corrected. Each movie recording was mean-centered and normalized with a single pair of value for all pixels", + "field_of_view_um":[400,400], + "excitation_wavelength":910, + "excitation_wavelength_nm":910 + } + }, + { + "name":"TP-Ai148", + "filename":"", + "url":"", + "dim":[512,512], + "pre":30, + "post":30, + "omit":1, + "type":"two-photon", + "format": "KerasFile", + "parameters": + { + "framerate":30, + "objective_power":16, + "objective_NA":0.8, + "indicator":"GCaMP6f", + "strain":"Ai148", + "training_data":"https://github.com/AllenInstitute/deepinterpolation/blob/cfb42bc2c5bd8cc08ee870a163230e8c89c676ba/examples/paper_generation_code/json_data/2019-09-05-train-very-large-single-plane-Ai148-norm.json", + "description":"Pre-processing: Individual movies were motion corrected. Each movie recording was mean-centered and normalized with a single pair of value for all pixels", + "field_of_view_um":[400,400], + "excitation_wavelength_nm":910 + } + }, + { + "name":"Ephys-Neuropixels_Phase_3a_1050", + "filename":"2020_02_29_15_28_unet_single_ephys_1024_mean_squared_error-1050.h5", + "url":"https://wds-matlab-community-toolboxes.s3.amazonaws.com/DeepInterpolation-MATLAB/Trained_models/fMRI/2020_08_28_00_25_fmri_unet_denoiser_mean_absolute_error_2020_08_28_00_25_model.h5", + "dim":[192,2], + "pre":30, + "post":30, + "omit":3, + "type":"ephys", + "format": "KerasFile", + "parameters": + { + "samplerate":30000, + "electrode": "Neuropixels Phase 3a", + "training_data":"https://github.com/AllenInstitute/deepinterpolation/blob/cfb42bc2c5bd8cc08ee870a163230e8c89c676ba/examples/paper_generation_code/json_data/2019-09-05-train-very-large-single-plane-Ai148-norm.json", + "description":"Pre-processing: Median subtraction was applied to individual probes to remove signals that were common across all recording sites. Each probe recording was mean-centered and normalized with a single pair of value for all nodes on the probe." + } + } +] diff --git a/sampleData/pretrainedNetwork.mat b/pretrainedModels/pretrainedNetwork.mat similarity index 100% rename from sampleData/pretrainedNetwork.mat rename to pretrainedModels/pretrainedNetwork.mat diff --git a/sampleData/retrainedNetwork_20230305_195921.mat b/sampleData/retrainedNetwork_20230305_195921.mat deleted file mode 100644 index d7d67e7..0000000 Binary files a/sampleData/retrainedNetwork_20230305_195921.mat and /dev/null differ diff --git a/setup.m b/setup.m index 25e28b5..6599986 100644 --- a/setup.m +++ b/setup.m @@ -1,10 +1,5 @@ function base_folder = setup() % Add current directory and sample_data, layers and examples to path - base_folder = fileparts(mfilename('fullpath')); - addpath(base_folder); - addpath(fullfile(base_folder, "network_layers")) - addpath(fullfile(base_folder, "sample_data")) - addpath(fullfile(base_folder, "examples")) - addpath(fullfile(base_folder, "internal")) - addpath(fullfile(base_folder, "datastore")) + base_folder = deepinterp.toolboxpath(); + deepinterp.setup(); end