[Feat]:

- Implementation for line search to get the optimal learning rate @ each step

optimization/steepest_gp.py

@@ -0,0 +1,29 @@
+import tensorflow as tf
+import random
+import pandas as pd
+
+def line_search(X,eta,loss_fn):
+    epsilon = 1e-6
+    lr = tf.Variable(random.random())
+    with tf.GradientTape(persistent=True) as tape1:
+        loss = loss_fn(X)
+    dloss_dx = tape1.gradient(loss,X)
+
+    lrs = [lr.numpy()]
+    # Q = loss_fn[X-delta_x * lr]
+    while True:
+        with tf.GradientTape(persistent=True) as derivative1:
+            with tf.GradientTape(persistent=True) as derivative2:
+                Q = loss_fn(X-lr*dloss_dx)
+            Q_dash = derivative2.gradient(Q,lr)
+        Q_double_dash = derivative1.gradient(Q_dash,lr)
+
+        magnitude_gradient = tf.norm(Q_dash)
+        print(magnitude_gradient)
+        if magnitude_gradient < eta:
+            return lr,lrs
+        lr.assign_sub(Q_dash/(epsilon+Q_double_dash))
+        lrs.append(lr.numpy())
+
+if __name__ == '__main__':
+    pass