diff --git a/optimization/adam.py b/optimization/adam.py new file mode 100644 index 0000000..8b824d9 --- /dev/null +++ b/optimization/adam.py @@ -0,0 +1,68 @@ +import tensorflow as tf +from typing import Callable +class Adam: + def __init__(self, + learning_rate:float, + beta1:float, + beta2:float, + objective_function:Callable, + params:tf.Tensor, + max_iteration:int) -> None: + + self.learning_rate = learning_rate + self.beta1 = beta1 + self.beta2 = beta2 + self.objective_fn =objective_function + self.params = params + self.max_iteration = max_iteration + self.first_momentum = tf.zeros_like(self.params) + self.second_momentum = tf.zeros_like(self.params) + self.epsilon = 1e-6 + self.losses = [] + + def optimize(self): + for t in range(1,self.max_iteration+1): + + with tf.GradientTape() as tape: + loss = self.objective_fn(self.params) + self.losses.append(loss.numpy()) + grad = tape.gradient(loss,self.params) + grad_squared = tf.square(grad) + + first_momentum_update = self.first_momentum*self.beta1 + grad * (1-self.beta1) + second_momentum_update = self.second_momentum * self.beta2 + grad_squared* (1-self.beta2) + + self.first_momentum= first_momentum_update + self.second_momentum = second_momentum_update + + first_momentum_cap = self.first_momentum/(1-tf.pow(self.beta1,t)) + second_momentum_cap = self.second_momentum/(1-tf.pow(self.beta2,t)) + + step = first_momentum_cap*self.learning_rate/tf.sqrt(second_momentum_cap+self.epsilon) + + self.params.assign_sub(step) + +def quadratic_loss(params): + return tf.reduce_sum(tf.square(params)) +if __name__ == '__main__': + + # Define initial parameters + initial_params = tf.Variable([1.0, 2.0, 3.0], dtype=tf.float32) + + # Adam optimizer parameters + learning_rate = 0.01 + beta1 = 0.9 + beta2 = 0.999 + max_iteration = 1000 + + # Create an instance of Adam optimizer + optimizer = Adam(learning_rate, beta1, beta2, quadratic_loss, initial_params, max_iteration) + + # Optimize the parameters + optimizer.optimize() + + # Get the optimized parameters + optimized_params = optimizer.params.numpy() + + print(optimized_params) + print(optimizer.losses) \ No newline at end of file