-
Notifications
You must be signed in to change notification settings - Fork 0
/
my_answers.py
143 lines (116 loc) · 5.77 KB
/
my_answers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
import numpy as np
class NeuralNetwork(object):
def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
# set number of nodes in input, hidden and output layers.
self.input_nodes = input_nodes
self.hidden_nodes = hidden_nodes
self.output_nodes = output_nodes
# initialize weights and learning rate
self.weights_input_to_hidden = np.random.normal(0.0, self.input_nodes**-0.5,
(self.input_nodes, self.hidden_nodes))
self.weights_hidden_to_output = np.random.normal(0.0, self.hidden_nodes**-0.5,
(self.hidden_nodes, self.output_nodes))
self.lr = learning_rate
# define activation function
self.activation_function = lambda x : 1 / (1 + np.exp(-x))
def train(self, features, targets):
''' Train the network on batch of features and targets.
Arguments
---------
features: 2D array, each row is one data record, each column is a feature
targets: 1D array of target values
epoch: number of times passed through the data
'''
# number of records
n_records = features.shape[0]
# initialize weight steps
delta_weights_i_h = np.zeros(self.weights_input_to_hidden.shape)
delta_weights_h_o = np.zeros(self.weights_hidden_to_output.shape)
# save a copy for each weight step for momentum
delta_weights_i_h_old, delta_weights_h_o_old = delta_weights_i_h.copy(), delta_weights_h_o.copy()
### Gradient descent with momentum ###
for X, y in zip(features, targets):
final_outputs, hidden_outputs = self.forward_pass_train(X)
delta_weights_i_h, delta_weights_h_o = self.backpropagation(final_outputs, hidden_outputs, X, y,
delta_weights_i_h, delta_weights_h_o)
self.update_weights(delta_weights_i_h + 0.9 * delta_weights_i_h_old,
delta_weights_h_o + 0.9 * delta_weights_h_o_old, n_records)
def forward_pass_train(self, X):
'''
Arguments
---------
X: 1D array of feature values
'''
### Forward pass ###
## matmul note
# inner dimension is the number of input nodes
# outer dimensions are the number of hidden nodes and 1
hidden_inputs = np.matmul(self.weights_input_to_hidden.T, X[:, None])
hidden_outputs = self.activation_function(hidden_inputs)
## matmul note
# inner dimension is the number of hidden nodes
# outer dimensions are 1 and 1 (effectively a dot product)
final_inputs = np.matmul(self.weights_hidden_to_output.T, hidden_outputs)
final_outputs = final_inputs
return final_outputs, hidden_outputs
def backpropagation(self, final_outputs, hidden_outputs, X, y, delta_weights_i_h, delta_weights_h_o):
''' Implement backpropagation
Arguments
---------
final_outputs: output from forward pass
y: target (i.e. label) batch
delta_weights_i_h: change in weights from input to hidden layers
delta_weights_h_o: change in weights from hidden to output layers
lr: learning rate
'''
### Backward pass ###
# identity function has gradient 1
output_error_term = y - final_outputs
## matmul note
# inner dimension is the number of outputs
# outer dimensions are the number of hidden nodes and 1
hidden_error_term = (np.matmul(self.weights_hidden_to_output, output_error_term)
* hidden_outputs * (1 - hidden_outputs))
## matmul note
# inner dimension is 1
# outer dimensions are the number of input nodes and the number of hidden nodes
delta_weights_i_h += self.lr * np.matmul(X[:, None], hidden_error_term.T)
# inner dimension is 1
# outer dimensions are the number of hidden nodes and the number of output nodes
delta_weights_h_o += self.lr * np.matmul(hidden_outputs, output_error_term.T)
return delta_weights_i_h, delta_weights_h_o
def update_weights(self, delta_weights_i_h, delta_weights_h_o, n_records):
''' Update weights on gradient descent step
Arguments
---------
delta_weights_i_h: change in weights from input to hidden layers
delta_weights_h_o: change in weights from hidden to output layers
n_records: number of records
'''
self.weights_hidden_to_output += delta_weights_h_o / n_records
self.weights_input_to_hidden += delta_weights_i_h / n_records
def run(self, features):
''' Run a forward pass through the network with input features
Arguments
---------
features: 2D array of feature values
'''
#### Forward pass ####
## matmul note
# inner dimension is the number of input nodes
# outer dimensions are the number of hidden nodes and the number of records
hidden_inputs = np.matmul(self.weights_input_to_hidden.T, features.T)
hidden_outputs = self.activation_function(hidden_inputs)
## matmul note
# inner dimension is the number of hidden nodes
# outer dimensions are the number of output nodes and the number of records
final_inputs = np.matmul(self.weights_hidden_to_output.T, hidden_outputs)
final_outputs = final_inputs.T
return final_outputs
#########################################################
# Set your hyperparameters here
##########################################################
iterations = 10000
learning_rate = 0.2
hidden_nodes = 14
output_nodes = 1