-
Notifications
You must be signed in to change notification settings - Fork 0
/
helper_logging.py
185 lines (157 loc) · 10.8 KB
/
helper_logging.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
import os
import csv
import torch
from torch.utils.tensorboard import SummaryWriter
class tensorboard_logger():
def __init__(self, run_name, train_mode, version, write_embeddings=False):
self.run_name = run_name
self.writer = SummaryWriter('../logs_'+version+'/tensorboard_logs/'+self.run_name+'/')
self.train_mode = train_mode
self.write_embeddings = write_embeddings
#print("Hi - I am going to handle your Tensorboard Logs")
def write_state(self,state):
## Losses
self.writer.add_scalar("01-Train-Loss/Total-Train-Loss", state['scalars']['total-train-loss'], state['epoch'])
self.writer.add_scalar("02-Validation-Loss/Total-Validation-Loss", state['scalars']['total-val-loss'], state['epoch'])
if self.train_mode == 'mmd':
self.writer.add_scalar("01-Train-Loss/MMD-Train-Loss", state['scalars']['mmd-train-loss'], state['epoch'])
self.writer.add_scalar("01-Train-Loss/CE-Train-Loss", state['scalars']['ce-train-loss'], state['epoch'])
self.writer.add_scalar("01-Train-Loss/MMD-Train-Share", state['scalars']['mmd-train-share'], state['epoch'])
self.writer.add_scalar("02-Validation-Loss/MMD-Validation-Loss", state['scalars']['mmd-val-loss'], state['epoch'])
self.writer.add_scalar("02-Validation-Loss/CE-Validation-Loss", state['scalars']['ce-val-loss'], state['epoch'])
self.writer.add_scalar("02-Validation-Loss/MMD-Validation-Share", state['scalars']['mmd-val-share'], state['epoch'])
## Kappa
self.writer.add_scalar("00-Kappa", state['kappa'], state['epoch'])
## Accuracies
self.writer.add_scalar("03-Validation-Accuracy/Accuracy", state['scalars']['val-acc'], state['epoch'])
if self.write_embeddings:
# Latent Representation
if state['epoch'] in [1,5,6,10,20,30,40,50,75,100,150,200,250,300]:
self.writer.add_embedding(state['embeddings']['z-train'], state['embeddings']['d-train'], global_step=state['epoch'], tag='Train-Data')
self.writer.add_embedding(state['embeddings']['z-val'], state['embeddings']['d-val'], global_step=state['epoch'], tag='Validation-Data')
## Classification Report
targets = ['Negative', 'Neutral', 'Positive']
for target in targets:
self.writer.add_scalar("04-Classification-Report/%s/Precision"%target, state['classification-report'][target]['precision'], state['epoch'])
self.writer.add_scalar("04-Classification-Report/%s/Recall"%target, state['classification-report'][target]['recall'], state['epoch'])
self.writer.add_scalar("04-Classification-Report/%s/F1-Score"%target, state['classification-report'][target]['f1-score'], state['epoch'])
## Layer Histograms
for name, params in state['models']['model'].named_parameters():
self.writer.add_histogram(name, torch.histc(params), state['epoch'])
## Domain Invariance
if not(self.train_mode=='single-source'):
for key in state['domain-invariance']['train'].keys():
self.writer.add_scalar(f"06-Domain-Invariance/{key}", state['domain-invariance']['train'][key], state['epoch'])
for key in state['domain-invariance']['val'].keys():
self.writer.add_scalar(f"06-Domain-Invariance/{key}", state['domain-invariance']['val'][key], state['epoch'])
self.writer.flush()
def write_test_results(self, state):
self.writer.add_hparams(state['hyperparams'], state['scalars'], run_name=f"test-{self.run_name}")
self.writer.flush()
#def __del__(self):
# self.writer.flush()
# self.writer.close()
class print_logger():
def __init__(self, train_mode):
self.train_mode = train_mode
#print("Hi - I am going to handle your Print Logs")
def write_state(self, state):
if self.train_mode == 'mmd':
print("[%s] Epoch %i: - kappa: %4.2f - Total-Train-Loss: %4.2f - CLA-Train-Loss: %4.2f - MMD-Train-Loss: %4.2f - Total-Val-Loss: %4.2f - CLA-Val-Loss: %4.2f - MMD-Validation-Loss: %4.2f - Val-Accuracy: %4.2f - Elapsed Time: %4.2f s"%(
state['run-name'], state['epoch'], state['kappa'], state['scalars']['total-train-loss'], state['scalars']['ce-train-loss'], state['scalars']['mmd-train-loss'], state['scalars']['total-val-loss'], state['scalars']['ce-val-loss'], state['scalars']['mmd-val-loss'], state['scalars']['val-acc'], state['end-time']-state['start-time']))
train_str = "[" + str(state['run-name']) + "] "
for name, value in state['domain-invariance']['train'].items():
train_str = train_str + name + ": {:4.2f} - ".format(value)
train_str = train_str[:-2]
val_str = "[" + str(state['run-name']) + "] "
for name, value in state['domain-invariance']['val'].items():
val_str = val_str + name + ": {:4.2f} - ".format(value)
val_str = val_str[:-2]
print(train_str)
print(val_str)
elif self.train_mode == 'standard' or self.train_mode=='single-source':
print("[%s] Epoch %i: - Train-Loss: %4.2f - Val-Loss: %4.2f - Val-Accuracy: %4.2f - Elapsed Time: %4.2f s"
%(state['run-name'], state['epoch'], state['scalars']['total-train-loss'], state['scalars']['total-val-loss'], state['scalars']['val-acc'], state['end-time']-state['start-time']))
else:
raise NotImplementedError
def write_test_results(self, state):
if self.train_mode == 'adversarial':
raise NotImplementedError
elif self.train_mode == 'standard' or self.train_mode=='mmd':
print("[%s] Test-Accuracy: %4.2f - SVM-Accuracy: %4.2f -LSVM-Accuracy: %4.2f - NB-Accuracy: %4.2f - XGB-Accuracy: %4.2f - LDA-Accuracy: %4.2f"
%(state['run-name'], state['scalars']['05-Scores/test-acc'], state['scalars']['05-Scores/svm-acc'], state['scalars']['05-Scores/lsvm-acc'], state['scalars']['05-Scores/nb-acc'], state['scalars']['05-Scores/xgb-acc'], state['scalars']['05-Scores/lda-acc']))
elif self.train_mode == 'single-source':
print("[%s] Test-Accuracy: %4.2f"%(state['run-name'], state['scalars']['05-Scores/test-acc']))
class csv_logger():
def __init__(self, logpath, train_mode):
self.train_mode = train_mode
self.logpath = logpath
self.header_written = False
#print("Hi - I am going to handle your CSV Logs")
def write_train_header(self):
# Prepare Logging
if not (os.path.isdir(self.logpath)):
os.makedirs(os.path.join(self.logpath))
elif os.path.isfile(os.path.join(self.logpath, 'logs.csv')):
print("LogFile already exists! Rename or remove it, and restart the training")
raise Exception
if self.train_mode == 'mmd':
header = ['Epoch', 'kappa', 'Total-Train-Loss', 'CLA-Train-Loss', 'MMD-Train-Loss', 'Total-Validation-Loss', 'CLA-Validation-Loss', 'MMD-Validation-Loss', 'Validation-Accuracy']
elif self.train_mode == 'standard' or self.train_mode == 'single-source':
header = ['Epoch', 'Train-Loss', 'Validation-Loss', 'Validation-Accuracy']
else:
raise NotImplementedError
with open(os.path.join(self.logpath, 'logs.csv'), 'w', encoding='UTF8') as f:
writer = csv.writer(f)
writer.writerow(header)
def write_domain_invariance_header(self, state):
# Prepare Logging
if not (os.path.isdir(self.logpath)):
os.makedirs(os.path.join(self.logpath))
elif os.path.isfile(os.path.join(self.logpath, 'domain_invariance.csv')):
print("LogFile already exists! Rename or remove it, and restart the training")
raise NameError
if self.train_mode == 'mmd' or self.train_mode == 'standard':
header = ['Epoch']
header.extend(name for name in state['domain-invariance']['train'])
header.extend(name for name in state['domain-invariance']['val'])
with open(os.path.join(self.logpath, 'domain_invariance.csv'), 'w', encoding='UTF8') as f:
writer = csv.writer(f)
writer.writerow(header)
elif self.train_mode == 'adversarial':
raise NotImplementedError
def write_state(self, state):
if not(self.header_written):
self.write_train_header()
self.write_domain_invariance_header(state)
self.header_written = True
with open(os.path.join(self.logpath, 'logs.csv'), 'a', encoding='UTF8') as f:
writer = csv.writer(f)
if self.train_mode == 'mmd':
writer.writerow([str(state['epoch']), str(state['kappa']), str(state['scalars']['total-train-loss']), str(state['scalars']['ce-train-loss']), str(state['scalars']['mmd-train-loss']), str(state['scalars']['total-val-loss']), str(state['scalars']['ce-val-loss']), str(state['scalars']['mmd-val-loss']), str(state['scalars']['val-acc'])])
elif self.train_mode == 'standard' or self.train_mode == 'single-source':
writer.writerow([str(state['epoch']), str(state['scalars']['total-train-loss']), state['scalars']['total-val-loss'], str(state['scalars']['val-acc'])])
else:
raise NotImplementedError
if self.train_mode == 'mmd' or self.train_mode == 'standard':
with open(os.path.join(self.logpath, 'domain_invariance.csv'), 'a', encoding='UTF8') as f:
writer = csv.writer(f)
row = [str(state['epoch'])]
row.extend(str(state['domain-invariance']['train'][name]) for name in state['domain-invariance']['train'])
row.extend(str(state['domain-invariance']['val'][name]) for name in state['domain-invariance']['val'])
writer.writerow(row)
elif self.train_mode == 'adversarial':
raise NotImplementedError
def write_test_results(self, state):
if self.train_mode == 'adversarial':
raise NotImplementedError
elif self.train_mode == 'standard' or self.train_mode=='mmd':
header = ['Test-Accuracy', 'SVM-Accuracy', 'Linear-SVM-Accuracy', 'NB-Accuracy', 'XGB-Accuracy', 'LDA-Accuracy']
row = [str(state['scalars']['05-Scores/test-acc']), str(state['scalars']['05-Scores/svm-acc']), str(state['scalars']['05-Scores/lsvm-acc']), str(state['scalars']['05-Scores/nb-acc']), str(state['scalars']['05-Scores/xgb-acc']), str(state['scalars']['05-Scores/lda-acc'])]
elif self.train_mode == 'single-source':
header = ['Test-Accuracy']
row = [str(state['scalars']['05-Scores/test-acc'])]
with open(os.path.join(self.logpath, 'test_logs.csv'), 'a', encoding='UTF8') as f:
writer = csv.writer(f)
writer.writerow(header)
writer.writerow(row)