-
Notifications
You must be signed in to change notification settings - Fork 8
/
modelMLP.lua
146 lines (135 loc) · 4.83 KB
/
modelMLP.lua
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
----------------------------------------------------------------------
--
-- Deep time series learning: Analysis of Torch
--
-- Multi-layer perceptron
--
----------------------------------------------------------------------
----------------------------------------------------------------------
-- Imports
require 'nn'
require 'torch'
require 'modelClass'
local nninit = require 'nninit'
local modelMLP, parent = torch.class('modelMLP', 'modelClass')
function modelMLP:defineModel(structure, options)
-- Container
local model = nn.Sequential();
-- Hidden layers
for i = 1,structure.nLayers do
-- Linear transform
if i == 1 then
model:add(nn.Linear(structure.nInputs,structure.layers[i]));
else
model:add(nn.Linear(structure.layers[i-1],structure.layers[i]));
end
-- Batch normalization
if self.batchNormalize then model:add(nn.BatchNormalization(structure.layers[i])); end
-- Non-linearity
model:add(self.nonLinearity());
-- Dropout
if self.dropout then model:add(nn.Dropout(self.dropout)); end
end
-- Final regression layer
model:add(nn.Linear(structure.layers[structure.nLayers],structure.nOutputs))
return model;
end
function modelMLP:definePSD(inS, outS, options)
--[[ Encoder part ]]--
encoder = nn.Sequential();
-- Linear transform
encoder:add(nn.Linear(inS, outS));
-- Batch normalization
if self.batchNormalize then encoder:add(nn.BatchNormalization(outS)); end
-- Non-linearity
encoder:add(self.nonLinearity());
-- Dropout
if self.dropout then encoder:add(nn.Dropout(self.dropout)) end
--encoder:add(nn.Diag(outS));
--[[ Decoder is L1 solution ]]--
decoder = unsup.LinearFistaL1(inS, outS, options.lambda)
-- PSD autoencoder
model = unsup.PSD(encoder, decoder, options.beta)
return model;
end
function modelMLP:defineAE(inS, outS, options)
--[[ Encoder part ]]--
encoder = nn.Sequential();
-- Linear transform
encoder:add(nn.Linear(inS, outS));
-- Batch normalization
if self.batchNormalize then encoder:add(nn.BatchNormalization(outS)); end
-- Non-linearity
encoder:add(self.nonLinearity());
-- Dropout
if self.dropout then encoder:add(nn.Dropout(self.dropout)) end
--encoder:add(nn.Diag(outS));
-- decoder
decoder = nn.Sequential();
decoder:add(nn.Linear(outS,inS));
-- impose weight sharing
decoder:get(1).weight = encoder:get(1).weight:t();
decoder:get(1).gradWeight = encoder:get(1).gradWeight:t();
-- complete model
return unsup.AutoEncoder(encoder, decoder, options.beta);
end
function modelMLP:definePretraining(structure, l, options)
-- Prepare the layer properties
if l == 1 then inS = structure.nInputs; else inS = structure.layers[l - 1]; end
outS = structure.layers[l];
-- Define layer as an auto-encoder
if (self.pretrainType == 'ae') then model = self:defineAE(inS, outS, options); end
-- Define layer as predictive sparse coding
if (self.pretrainType == 'psd') then model = self:definePSD(inS, outS, options); end
return model;
end
function modelMLP:retrieveEncodingLayer(model)
-- Here simply return the encoder
encoder = model.encoder
encoder:remove();
return model.encoder;
end
function modelMLP:weightsInitialize(model)
-- Find only the linear modules
linearNodes = model:findModules('nn.Linear')
for l = 1,#linearNodes do
module = linearNodes[l];
module:init('weight', self.initialize);
module:init('bias', self.initialize);
end
return model;
end
function modelMLP:weightsTransfer(model, trainedLayers)
-- Find only the linear modules
linearNodes = model:findModules('nn.Linear')
for l = 1,#trainedLayers do
-- Find equivalent in pre-trained layer
preTrained = trainedLayers[l].encoder:findModules('nn.Linear');
linearNodes[l].weight = preTrained[1].weight;
linearNodes[l].bias = preTrained[1].bias;
end
-- Initialize the batch normalization layers
for k,v in pairs(model:findModules('nn.BatchNormalization')) do
v.weight:fill(1)
v.bias:zero()
end
return model;
end
function modelMLP:parametersDefault()
self.initialize = nninit.xavier;
self.nonLinearity = nn.ReLU;
self.batchNormalize = true;
self.pretrainType = 'ae';
self.pretrain = true;
self.dropout = 0.5;
end
function modelMLP:parametersRandom()
-- All possible non-linearities
self.distributions = {};
self.distributions.nonLinearity = {nn.HardTanh, nn.HardShrink, nn.SoftShrink, nn.SoftMax, nn.SoftMin, nn.SoftPlus, nn.SoftSign, nn.LogSigmoid, nn.LogSoftMax, nn.Sigmoid, nn.Tanh, nn.ReLU, nn.PReLU, nn.RReLU, nn.ELU, nn.LeakyReLU};
self.distributions.initialize = {nninit.normal, nninit.uniform, nninit.xavier, nninit.kaiming, nninit.orthogonal, nninit.sparse};
self.distributions.batchNormalize = {true, false};
self.distributions.pretrainType = {'ae', 'psd'};
self.distributions.pretrain = {true, false};
self.distributions.dropout = {0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9};
end