-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathexperiment_tpot_all_site.py
153 lines (119 loc) · 6 KB
/
experiment_tpot_all_site.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
"""Example of experiment."""
from pathlib import Path
import joblib
import pandas as pd
import numpy as np
from sklearn.decomposition import PCA
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold, GridSearchCV
from sklearn.preprocessing import RobustScaler
from sklearn.svm import LinearSVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline, make_union
from tpot.builtins import StackingEstimator
from sklearn.preprocessing import FunctionTransformer
from copy import copy
from sklearn.model_selection import cross_validate
from helper_functions import read_freesurfer, read_freesurfer_sites_validation
from freesurfer_columns import thk_vol_curv as COLUMNS_NAMES
PROJECT_ROOT = Path.cwd()
# --------------------------------------------------------------------------
random_seed = 42
np.random.seed(random_seed)
# --------------------------------------------------------------------------
# Create experiment's output directory
output_dir = PROJECT_ROOT / 'output' / 'experiments'
output_dir.mkdir(exist_ok=True)
experiment_name = 'tpot_all_subjects_r' # Change here*
experiment_dir = output_dir / experiment_name
experiment_dir.mkdir(exist_ok=True)
cv_dir = experiment_dir / 'cv'
cv_dir.mkdir(exist_ok=True)
# --------------------------------------------------------------------------
# Input data directory (plz, feel free to use NAN shared folder)
input_dir = PROJECT_ROOT / 'data' / 'freesurfer'
demographic_path = PROJECT_ROOT / 'data' / 'PAC2019_BrainAge_Training.csv'
# Reading data. If necessary, create new reader in helper_functions.
x, demographic_df = read_freesurfer(str(input_dir),
str(demographic_path),
COLUMNS_NAMES)
# --------------------------------------------------------------------------
# Using only age
y = demographic_df['age'].values
# If necessary, extract gender and site from demographic_df too.
# --------------------------------------------------------------------------
n_folds = 5
kf = KFold(n_splits=n_folds, shuffle=True, random_state=random_seed)
# --------------------------------------------------------------------------
predictions_df = pd.DataFrame(demographic_df[['age']])
predictions_df['predictions'] = np.nan
mae_cv = np.zeros((n_folds, 1))
# -------------------------------------------------------------------------- for
for i_fold, (train_idx, test_idx) in enumerate(kf.split(x, y)):
x_train, x_test = x[train_idx], x[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
print('CV iteration: %d' % (i_fold + 1))
#--------------------------------------------------------------------------
# Normalization/Scaling/Standardization
scaler = RobustScaler()
x_train_norm = scaler.fit_transform(x_train)
x_test_norm = scaler.transform(x_test)
#--------------------------------------------------------------------------
# Model
clf = make_pipeline( make_union(StackingEstimator
(estimator=LinearRegression()), FunctionTransformer(copy)),
RandomForestRegressor(bootstrap=False, max_features=0.45,
min_samples_leaf=15, min_samples_split=19, n_estimators=100,
random_state=42))
#--------------------------------------------------------------------------
# Gridsearch
clf.fit(x_train_norm, y_train)
#--------------------------------------------------------------------------
y_test_predicted = clf.predict(x_test_norm)
for row, value in zip(test_idx, y_test_predicted):
predictions_df.iloc[row, predictions_df.columns.get_loc('predictions')] = value
#--------------------------------------------------------------------------
mae_test = mean_absolute_error(y_test, y_test_predicted)
print('MAE: %.3f' % mae_test)
mae_cv[i_fold, :] = mae_test
joblib.dump(clf, cv_dir / ('model_%d.joblib' % i_fold))
print('CV results')
print('MAE: Mean(SD) = %.3f(%.3f)' % (mae_cv.mean(), mae_cv.std()))
mae_cv_df = pd.DataFrame(columns=['MAE'], data=mae_cv)
mae_cv_df.to_csv(cv_dir / 'mae_cv.csv', index=False)
predictions_df.to_csv(cv_dir / 'predictions_cv.csv', index=True)
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------
# Training on whole data
scaler = RobustScaler()
scaler_fit = scaler.fit(x)
x_norm = scaler_fit.transform(x)
clf_final = make_pipeline(
make_union(StackingEstimator
(estimator=LinearRegression()),
FunctionTransformer(copy)),
RandomForestRegressor(bootstrap=False,
max_features=0.45,
min_samples_leaf=15,
min_samples_split=19,
n_estimators=100,
random_state=42))
training_model = clf_final.fit(x_norm, y)
joblib.dump(scaler_fit, experiment_dir / 'scaler.joblib')
# --------------------------------------------------------------------------
# Validation data
# --------------------------------------------------------------------------
# Load the freesurfer dataset
freesurfer_dir_validation = PROJECT_ROOT / 'data' / 'freesurfer'/ 'test_set'
demographic_path_validation = PROJECT_ROOT / 'data' / 'PAC2019_BrainAge_Test_Upload.csv'
x_validation, demographic_df_validation = read_freesurfer_sites_validation(
freesurfer_dir_validation,
demographic_path_validation,
COLUMNS_NAMES,
-1)
# scale the dataset
x_val_scaled = scaler_fit.transform(x_validation)
predicted_val = training_model.predict(x_val_scaled)
predictions_val_df = pd.DataFrame(index=x_validation.index, data=predicted_val)
predictions_val_df.to_csv(scaler_fit, experiment_dir / 'predictions_cv_val.csv', index=True)