ML-with-Netflix/regression_regularization.py at master · YuvalAtir/ML-with-Netflix · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
from __future__ import division  # to get a float and not int with simple division /
import numpy as np
from sklearn import linear_model
import matplotlib.pyplot as plt
import load_data
import utility as util
import time
#import preprocessing


def regularized_regression(reg_type):
    """
    Function: compute regression on Netflix data with regularization

    Args:
        reg_type(int) - 1 = ridge regularization
                        2 = lasso regularization

    Returns: None

    """
    # read all Netflix data
    data = load_data.Data()

    # append the "intercept", beta0
    data.x = np.hstack((np.ones((data.n_data, 1)), data.x))

    # data set split: training, cv, test
    split_data_ratio = [0.8, 0, 0.2]
    data.manual_split(split_data_ratio)

    if reg_type == 1:
        #regr = linear_model.Ridge(alpha=0.01, normalize=False)
        regr = linear_model.RidgeCV(alphas=[0, 1.0, 10, 50, 100, 500, 1000, 3000, 5000], cv=10)
    elif reg_type == 2:
        #regr = linear_model.Lasso(alpha=0.03)
        regr = linear_model.LassoCV(alphas=[0.01, 1.0, 10, 50, 100, 500, 1000, 3000, 5000], cv=10)

    # Train the model using the training sets
    regr.fit(data.x_train, data.y_train)  # calculate coefficients and intercept

    # use model to predict test data
    y_est = regr.predict(data.x_test)
    y_est_srmse = util.srmse(y_est, data.y_test)
    print("Linear Regression With Regularization Using SKLearn Root Mean Squared Error: %.3f" % y_est_srmse)
    util.count_err(y_est, data.y_test, 1)
    # print('Coefficients: \n', regr.coef_)


def find_reg_lamda(reg_type):
    """
        Function: find the hyperparameter lamda for ridge regression on Netflix data

        Args: None

        Returns: None

        """

    if reg_type == 1:
        lamda_vec = (0, 1.0, 10, 50, 100, 500, 1000, 3000, 5000)
    if reg_type == 2:
        lamda_vec = (0.0001, 0.0005, 0.001, 0.003, 0.007, 0.01, 0.05, 0.5, 1)

    alpha_vec_len = len(lamda_vec)

    # read all Netflix data
    data = load_data.Data()

    # append the "intercept", beta0
    data.x = np.hstack((np.ones((data.n_data, 1)), data.x))

    # data set split: training, cv, test
    split_data_ratio = [0.6, 0.2, 0.2]
    data.manual_split(split_data_ratio)

    # time the execution length
    start_time = time.time()

    alpha_vec = [ 1/2*x for x in lamda_vec]
    var_J_train = np.zeros((alpha_vec_len, 1))
    var_J_cv = np.zeros((alpha_vec_len, 1))
    m = 0

    for reg_alpha in alpha_vec:
        print('m = %d' % m)
        if reg_type == 1:
            regr = linear_model.Ridge(alpha=reg_alpha)
        elif reg_type == 2:
            regr = linear_model.Lasso(alpha=reg_alpha)
        regr.fit(data.x_train, data.y_train)
        reg_y_est_train = regr.predict(data.x_train)
        reg_srmse_train = util.srmse(reg_y_est_train, data.y_train)
        var_J_train[m] = reg_srmse_train

        reg_y_est_cv = regr.predict(data.x_validate)
        reg_srmse_cv = util.srmse(reg_y_est_cv, data.y_validate)
        var_J_cv[m] = reg_srmse_cv
        m += 1
    print("--- %s seconds ---" % (time.time() - start_time))

    if reg_type == 1:
        regr = linear_model.Ridge(alpha=3000, normalize=False)
    elif reg_type == 2:
        regr = linear_model.Lasso(alpha=0.05, normalize=False)
    regr.fit(data.x_test, data.y_test)
    reg_y_est_test = regr.predict(data.x_test)
    reg_srmse_test = util.srmse(reg_y_est_test, data.y_test)
    print("Linear Regression With Regularization Manual CV to find Lambda, Root Mean Squared Error: %.3f"
          % reg_srmse_test)
    util.count_err(reg_y_est_test, data.y_test, 1)

    line_tr, = plt.plot(lamda_vec, var_J_train, label='training')
    line_vl, = plt.plot(lamda_vec, var_J_cv, label='validation')
    plt.legend(handles=[line_tr, line_vl])
    plt.xlabel('Ridge Lamda')
    plt.ylabel('RMSE')
    plt.title('Curve for Finding Ridge Lamda')
    plt.show()


def main():
    # reg_type = 1 for ridge, 2 for lasso
    reg_type = 2
    regularized_regression(reg_type)
    find_reg_lamda(reg_type)


if __name__ == "__main__":
    main()