diff --git a/optimization/steepest_gp.py b/optimization/steepest_gp.py new file mode 100644 index 0000000..b3fe455 --- /dev/null +++ b/optimization/steepest_gp.py @@ -0,0 +1,29 @@ +import tensorflow as tf +import random +import pandas as pd + +def line_search(X,eta,loss_fn): + epsilon = 1e-6 + lr = tf.Variable(random.random()) + with tf.GradientTape(persistent=True) as tape1: + loss = loss_fn(X) + dloss_dx = tape1.gradient(loss,X) + + lrs = [lr.numpy()] + # Q = loss_fn[X-delta_x * lr] + while True: + with tf.GradientTape(persistent=True) as derivative1: + with tf.GradientTape(persistent=True) as derivative2: + Q = loss_fn(X-lr*dloss_dx) + Q_dash = derivative2.gradient(Q,lr) + Q_double_dash = derivative1.gradient(Q_dash,lr) + + magnitude_gradient = tf.norm(Q_dash) + print(magnitude_gradient) + if magnitude_gradient < eta: + return lr,lrs + lr.assign_sub(Q_dash/(epsilon+Q_double_dash)) + lrs.append(lr.numpy()) + +if __name__ == '__main__': + pass