forked from LPD-EPFL/DifferentialByzantine
-
Notifications
You must be signed in to change notification settings - Fork 0
/
reproduce.py
309 lines (283 loc) · 11.8 KB
/
reproduce.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
# coding: utf-8
###
# @file reproduce.py
# @author Sébastien Rouault <[email protected]>
#
# @section LICENSE
#
# Copyright © 2019-2021 École Polytechnique Fédérale de Lausanne (EPFL).
# See LICENSE file.
#
# @section DESCRIPTION
#
# Reproduce the (missing) experiments and plots.
###
import tools
tools.success("Module loading...")
import argparse
import pathlib
import signal
import sys
import traceback
import torch
import experiments
# ---------------------------------------------------------------------------- #
# Miscellaneous initializations
tools.success("Miscellaneous initializations...")
# "Exit requested" global variable accessors
exit_is_requested, exit_set_requested = tools.onetime("exit")
# Signal handlers
signal.signal(signal.SIGINT, exit_set_requested)
signal.signal(signal.SIGTERM, exit_set_requested)
# ---------------------------------------------------------------------------- #
# Command-line processing
tools.success("Command-line processing...")
def process_commandline():
""" Parse the command-line and perform checks.
Returns:
Parsed configuration
"""
# Description
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("--data-directory",
type=str,
default="results-data",
help="Path of the data directory, containing the data gathered from the experiments")
parser.add_argument("--plot-directory",
type=str,
default="results-plot",
help="Path of the plot directory, containing the graphs traced from the experiments")
parser.add_argument("--devices",
type=str,
default="cpu",
help="Comma-separated list of devices on which to run the experiments, used in a round-robin fashion")
parser.add_argument("--supercharge",
type=int,
default=1,
help="How many experiments are run in parallel per device, must be positive")
parser.add_argument("--only-plot",
default=False,
action="store_true",
help="Only build the plots (useful to get some plots while experiments are still running)")
# Parse command line
return parser.parse_args(sys.argv[1:])
with tools.Context("cmdline", "info"):
args = process_commandline()
# Check the "supercharge" parameter
if args.supercharge < 1:
tools.fatal(f"Expected a positive supercharge value, got {args.supercharge}")
# Make the result directories
def check_make_dir(path):
path = pathlib.Path(path)
if path.exists():
if not path.is_dir():
tools.fatal(f"Given path {str(path)!r} must point to a directory")
else:
path.mkdir(mode=0o755, parents=True)
return path
args.data_directory = check_make_dir(args.data_directory)
args.plot_directory = check_make_dir(args.plot_directory)
# Preprocess/resolve the devices to use
if args.devices == "auto":
if torch.cuda.is_available():
args.devices = list(f"cuda:{i}" for i in range(torch.cuda.device_count()))
else:
args.devices = ["cpu"]
else:
args.devices = list(name.strip() for name in args.devices.split(","))
# ---------------------------------------------------------------------------- #
# Serial preloading of the dataset
tools.success("Pre-downloading datasets...")
# Pre-load the datasets to prevent the first parallel runs from downloading them several times
with tools.Context("dataset", "info"):
for name in ("svm-phishing",):
with tools.Context(name, "info"):
experiments.make_datasets(name, 1, 1)
# ---------------------------------------------------------------------------- #
# Run (missing) experiments
if not args.only_plot:
tools.success("Running experiments...")
# Command maker helper
def make_command(params):
cmd = ["python3", "train.py"]
cmd += tools.dict_to_cmdlist(params)
return tools.Command(cmd)
# Jobs
jobs = tools.Jobs(args.data_directory, devices=args.devices, devmult=args.supercharge)
seeds = jobs.get_seeds()
# Base parameters for the experiments
params_common = {
"loss": "mse",
"learning-rate": 2,
"criterion": "sigmoid",
"momentum": 0.99,
"evaluation-delta": 50,
"nb-steps": 1000,
"nb-workers": 11,
"nb-decl-byz": 5,
"nb-real-byz": 5,
"batch-size-test": 10,
"test-repeat": 45,
"gradient-clip": 0.01,
"privacy-delta": 1e-6 }
# Submit all the experiments (if not disabled)
if not args.only_plot:
for ds, dsa in (("svm-phishing", None),):
for md, mda in (("simples-logit", "din:68"),):
for gar, attacks in (("average", (("nan", None),)),("brute", (("little", ("factor:1.5", "negative:True")), ("empire", "factor:1.1")))): #
for attack, attargs in attacks:
for epsilon in (None, 0.2): #0.1, 0.2, 0.5
for batch_size in (10, ): #25, 50, 100, 250, 500
name = f"{ds}-{md}-{gar}-{attack}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}"
# Submit experiment
params = params_common.copy()
if gar == "average":
# Disable attack for 'average' GAR
params["nb-real-byz"] = 0
params["dataset"] = ds
params["dataset-args"] = dsa
params["model"] = md
params["model-args"] = mda
params["gar"] = gar
params["attack"] = attack
params["attack-args"] = attargs
params["privacy"] = epsilon is not None
params["privacy-epsilon"] = epsilon
params["batch-size"] = batch_size
jobs.submit(name, make_command(params))
# added
for gar, attacks in (("average", (("nan", None),)), ("brute", (("little", ("factor:1.5", "negative:True")), ("empire", "factor:1.1")))): #
for attack, attargs in attacks:
for epsilon in (None, 0.2): #0.1, 0.2, 0.5
for batch_size in (50, ): #25, 50, 100, 250, 500
ds = "mnist"
mda = None
dsa = None
md = 'simples-conv'
name = f"{ds}-{md}-{gar}-{attack}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}"
# Submit experiment
params = params_common.copy()
if gar == "average":
# Disable attack for 'average' GAR
params["nb-real-byz"] = 0
epsilon = 0.2
params["gradient-clip"] = 5
params["learning-rate"] = 0.01
params["momentum"] = 0.9
params["loss"] = "nll"
params["dataset"] = ds
params["dataset-args"] = dsa
params["model"] = md
params["model-args"] = mda
params["gar"] = gar
params["attack"] = attack
params["attack-args"] = attargs
params["privacy"] = epsilon is not None
params["privacy-epsilon"] = epsilon
params["batch-size"] = batch_size
params["batch-size-test"] = 10
params['criterion'] = 'top-k'
jobs.submit(name, make_command(params))
# Wait for the jobs to finish and close the pool
jobs.wait(exit_is_requested)
jobs.close()
# Check if exit requested before going to plotting the results
if exit_is_requested():
exit(0)
# ---------------------------------------------------------------------------- #
# Produce graphs
# Import additional modules
try:
import histogram
import numpy
import pandas
except ImportError as err:
tools.fatal(f"Unable to plot results: {err}")
# Map gar name in code to key in graph
gar_to_legend = {"brute": "MDA"}
def compute_avg_err(name, *cols, avgs="", errs="-err"):
""" Compute the average and standard deviation of the selected columns over the given experiment.
Args:
name Given experiment name
... Selected column names (through 'histogram.select')
avgs Suffix for average column names
errs Suffix for standard deviation (or "error") column names
Returns:
Data frames, each for the computed columns
"""
# Load all the runs for the given experiment name, and keep only a subset
datas = tuple(histogram.select(histogram.Session(args.data_directory / f"{name}-{seed}"), *cols) for seed in seeds)
# Make the aggregated data frames
def make_df(col):
nonlocal datas
# For every selected columns
subds = tuple(histogram.select(data, col).dropna() for data in datas)
res = pandas.DataFrame(index=subds[0].index)
for col in subds[0]:
# Generate compound column names
avgn = col + avgs
errn = col + errs
# Compute compound columns
numds = numpy.stack(tuple(subd[col].to_numpy() for subd in subds))
res[avgn] = numds.mean(axis=0)
res[errn] = numds.std(axis=0)
# Return the built data frame
return res
# Return the built data frames
return tuple(make_df(col) for col in cols)
with tools.Context("plot", "info"):
# Plot all the experiments
for ds, dsa in (("svm-phishing", None),):
for md, mda in (("simples-logit", "din:68"),):
for epsilon in (None, 0.2): #0.1, 0.2, 0.5
for batch_size in (10,): # 25, 50, 100, 250, 500
legend = list()
results = list()
# Pre-process results for all available combinations of GAR and attack
for gar, attacks in (("average", (("nan", None),)), ("brute", (("little", ("factor:1.5", "negative:True")), ("empire", "factor:1.1")))): #
for attack, _ in attacks:
name = f"{ds}-{md}-{gar}-{attack}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}"
key = f"{gar_to_legend.get(gar, gar.capitalize())} ({'no attack' if gar == 'average' else attack})"
legend.append(key)
results.append(compute_avg_err(name, "Accuracy", "Average loss"))
# Plot top-1 cross-accuracy
plot = histogram.LinePlot()
for crossacc, _ in results:
plot.include(crossacc, "Accuracy", errs="-err", lalp=0.8)
plot.finalize(None, "Step number", "Cross-accuracy", xmin=0, xmax=1000, ymin=0, ymax=1, legend=legend)
plot.save(args.plot_directory / f"{ds}-{md}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}.png", xsize=3, ysize=1.5)
# Plot average loss
plot = histogram.LinePlot()
for _, avgloss in results:
plot.include(avgloss, "Average loss", errs="-err", lalp=0.8)
plot.finalize(None, "Step number", "Average loss", xmin=0, xmax=1000, ymin=0, ymax=.6, legend=legend)
plot.save(args.plot_directory / f"{ds}-{md}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}-loss.png", xsize=3, ysize=1.5)
# added
ds = "mnist"
dsa = None
md = "simples-conv"
mda = None
for epsilon in (None, 0.2): # 0.1, 0.2, 0.5
for batch_size in (50,): # 25, 50, 100, 250, 500
legend = list()
results = list()
# Pre-process results for all available combinations of GAR and attack
for gar, attacks in (("average", (("nan", None),)), ("brute", (("little", ("factor:1.5", "negative:True")), ("empire", "factor:1.1")))): #
for attack, _ in attacks:
name = f"{ds}-{md}-{gar}-{attack}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}"
key = f"{gar_to_legend.get(gar, gar.capitalize())} ({'no attack' if gar == 'average' else attack})"
legend.append(key)
results.append(compute_avg_err(name, "Accuracy", "Average loss"))
# Plot top-1 cross-accuracy
plot = histogram.LinePlot()
for crossacc, _ in results:
plot.include(crossacc, "Accuracy", errs="-err", lalp=0.8)
plot.finalize(None, "Step number", "Cross-accuracy", xmin=0, xmax=1000, ymin=0, ymax=1, legend=legend)
plot.save(args.plot_directory / f"{ds}-{md}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}.png",
xsize=3, ysize=1.5)
# Plot average loss
plot = histogram.LinePlot()
for _, avgloss in results:
plot.include(avgloss, "Average loss", errs="-err", lalp=0.8)
plot.finalize(None, "Step number", "Average loss", xmin=0, xmax=1000, ymin=0, ymax=.6, legend=legend)
plot.save(args.plot_directory / f"{ds}-{md}-e_{'inf' if epsilon is None else epsilon}-b_{batch_size}-loss.png", xsize=3, ysize=1.5)