-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathcombined_developing_real_graph.py
329 lines (285 loc) · 17.1 KB
/
combined_developing_real_graph.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
## Author: Scott Emmons ([email protected])
## Purpose: To automate the command line calls necessary to execute the full clustering analysis script workflow.
## Date: January 2, 2015
## Depends on compiled LFR benchmark graph generation code and compiled Lancichinetti clustering repository code.
import argparse
import os
import shutil
import errno
import time
import math
import random
import glob
import subprocess
supported_modules = ['generate', 'sample', 'cluster', 'measure', 'visualize']
bench_directory_lock = 'bench_directory.lock'
lancichinetti_program_lock = 'lancichinetti_program.lock'
def handleArgs():
"""Handles the command-line input arguments, placing them in the global Namespace variable 'args'."""
parser = argparse.ArgumentParser(description="Automates the command line calls necessary to execute the full clustering analysis script workflow")
parser.add_argument("--mu", type=float, required=True, help="the mixing parameter. Fortunato suggests we test values of 0.40, 0.50, and 0.60", dest="mu")
parser.add_argument("-t", "--times", default=1, type=int, help="the number of clusterings to run on each graph", dest="numtrials")
parser.add_argument("-n", "--nlist", nargs="+", required=True, help="the values of N, or number of nodes in the graph, for which to execute the workflow", dest="n_list")
parser.add_argument("-s", "--start", default = 1, type=int, help="the file number at which to start, inclusive", dest="start")
parser.add_argument("-e", "--end", default = 10, type=int, help="the file number at which to end, inclusive", dest="end")
parser.add_argument("-m", "--modules", nargs="+", default=supported_modules, type=str.lower, choices=supported_modules, help="the names of the modules to run. defaults to all of them", dest="modules")
parser.add_argument("-b", "--benchmark", default=os.getcwd()+"/Networks_with_ground_truth_communities/", help="the path to the installed LFR generation software, required for functionality of the generate module. Defaults to the current working directory + '/binary_networks/'", dest="bench_directory_stem")
parser.add_argument("--lp", default=os.getcwd()+"/clustering_programs_5_2/", help="the path at which the Lancichinetti clustering program is installed, required for full functionality of the measure module. Defaults to the current working directory + '/clustering_programs_5_2/'", dest="lancichinetti_program_path")
parser.add_argument("--Xmx", default="64m", help="A value for the Xmx Java parameter, to be passed to the call to the slm clustering algorithm. For example, if '600m' is given, the flag '-Xmx600m' will be passed to the java call to run the slm clustering algorithm. Defaults to '64m'", dest="xmx")
parser.add_argument("-sr", "--samplerate", default=0.7, type=float, help="the sample percentage of the entire graph", dest="samplerate")
parser.add_argument("-delta","--deltavalue", nargs="+", default="0.5 0.6 0.7 0.8 0.9", help="the delta parameter(the default value is 0.5 0.6 0.7 0.8 0.9)", dest="delta")
parser.add_argument("-sac","--sampling_condition", default="induced_random_edge", help="choose the sampling algorithm", dest="sampling_condition")
parser.add_argument("-nc", "--numcluster", default = 10, type=int, help="the numbers of nodes", dest="numcluster")
global args
args = parser.parse_args()
def touch(filename, times=None):
"""Python implementation of 'touch' command in Ubuntu terminal. Not completely race-free. Credits to community wiki on SO: def touch(fname, times=None): http://stackoverflow.com/questions/1158076/implement-touch-using-python"""
with open(filename, 'a'):
os.utime(filename, times)
def combineDataFiles(name, extension, directory):
"""Combines the data files matching directory + '/' + name + '_s*_e*' + extension in directory.
The arguments name, extension, and directory should be strings."""
def deleteFileIfNeeded(file):
try:
os.remove(file)
except OSError as error:
if error.errno != errno.ENOENT:
raise
def appendFileToFile(append_file_name, to_file_name, write_header = True):
"""Write the content in append_file_name into to_file_name. Write header determines whether or not the first line in append_file_name is written to to_file_name."""
to_f = open(to_file_name, 'a')
append_f = open(append_file_name, 'r')
if not write_header:
append_f.readline()
for line in append_f:
to_f.write(line)
to_f.close()
append_f.close()
filtered_files = glob.glob(directory + '/' + name + '_s*_e*' + extension)
# assert len(filtered_files) > 0
combined_file = directory + '/' + name + extension
deleteFileIfNeeded(combined_file)
written_header = False
for data_file in filtered_files:
appendFileToFile(data_file, combined_file, not written_header)
written_header = True
def incrementBenchSeed(bench_directory_stem, times = 1):
"""Increments the value of the random seed in Lancichinetti's bencmark generation program."""
seed_file = bench_directory_stem + 'time_seed.dat'
with open(seed_file, 'r') as f:
value = f.readline().split()[0]
with open(seed_file, 'wb') as f:
f.write(str(int(value) + 1 * times) + '\n')
handleArgs()
#To scatter parallel runs executed simultaneously
##time.sleep(random.randrange(500, 18000) / 100.0)
##args.n_list = ['10', '100', '1000', '10000', '100000']
for n in args.n_list:
benchmark_directory = 'generated_benches_u'+str(int(100*args.mu))+'_p_'+ str(int(100*args.samplerate))+'_condition_'+ args.sampling_condition+'/'
working_directory = 'generated_benches_u'+str(int(100*args.mu))+'_p_'+ str(int(100*args.samplerate))+'_condition_'+ args.sampling_condition+'/n_' + n + '/'
new_bench_directory_stem = working_directory + 'binary_networks_s_' + str(args.start) + '_e_' + str(args.end) + '/'
new_lancichinetti_program_path = working_directory + 'clustering_programs_5_2_s_' + str(args.start) + '_e_' + str(args.end) + '/'
if 'generate' in args.modules:
# python generate.py -n 1000 --mu 0.40 -s 1 -e 10 -b binary_networks/ -o generated_benches/n_1000/
command_list = []
command_list.append('python')
command_list.append('generate_real_graph.py')
command_list.append('-n')
command_list.append(n)
command_list.append('--maxk')
command_list.append(str(int(0.10 * float(n))))
command_list.append('--mu')
command_list.append(str(args.mu))
command_list.append('--minc')
command_list.append('50')
command_list.append('--maxc')
command_list.append(str(int(0.10 * float(n))))
command_list.append('-s')
command_list.append(str(args.start))
command_list.append('-e')
command_list.append(str(args.end))
command_list.append('-b')
command_list.append(args.bench_directory_stem)
command_list.append('-o')
command_list.append(working_directory)
#To be run in parallel
# bench_copy_start = time.clock()
# shutil.copytree(args.bench_directory_stem, new_bench_directory_stem)
# bench_copy_end = time.clock()
# print '\nTime to copy bench_directory: ' + str(bench_copy_end - bench_copy_start) + '\n'
# increment_by = args.start + (int(math.ceil(math.log(int(n), 10))) - 3) * 100
# incrementBenchSeed(new_bench_directory_stem, times = increment_by)
# print '\nIncremented bench seed by: ' + str(increment_by) + '\n'
# touch('timeseed_inc_token_n_' + n + '_s_' + str(args.start) + '_e_' + str(args.end))
order = ' '.join(command_list)
subprocess.call(command_list)
# bench_remove_start = time.clock()
# shutil.rmtree(new_bench_directory_stem)
# bench_remove_end = time.clock()
# print '\nTime to remove bench_directory: ' + str(bench_remove_end - bench_remove_start) + '\n'
print '\n' + ('*' * (62 + len(n)))
print '* Completed clustering analysis "generate" workflow for N = ' + n + ' *'
print '*' * (62 + len(n)) + '\n'
if 'sample' in args.modules:
#python sample.py -n 1000 -s 1 -e 3 -o generated_benches/n_1000/ --sample_percentage 0.3 0.5 0.7 --sampling_condition induced_random_edge
command_list = []
command_list.append('python')
command_list.append('sample_developing.py')
command_list.append('-n')
command_list.append(n)
command_list.append('-s')
command_list.append(str(args.start))
command_list.append('-e')
command_list.append(str(args.end))
command_list.append('-o')
command_list.append(working_directory)
command_list.append('--sample_percentage')
command_list.append(str(args.samplerate))
# command_list.append('0.3')
# command_list.append('0.5')
# command_list.append('0.7')
command_list.append('--sampling_condition')
command_list.append(args.sampling_condition)
command_list.append('--numcluster')
command_list.append(str(args.numcluster))
subprocess.call(command_list)
order_sample= ' '.join(command_list)
print order_sample
print '\n' + ('*' * (61 + len(n)))
print '* Completed clustering analysis "measure" workflow for N = ' + n + ' *'
print '*' * (61 + len(n)) + '\n'
if 'cluster' in args.modules:
# python cluster.py -m blondel infomap label_propagation oslom modularity_optimization -sample 0.7 -t 2 -u --gpre generated_benches/n_1000/network_sample_p70_v --gsuf .dat -s 1 -e 3 --lp clustering_programs_5_2/ -o generated_benches/n_1000/
# python cluster.py -m blondel infomap label_propagation slm -t 1 -u --gpre generated_benches/n_1000/network_v --gsuf .dat -s 1 -e 10 --lp clustering_programs_5_2/ -o generated_benches/n_1000/
# python cluster.py -m blondel infomap label_propagation slm -t 1 -u --gpre real_world_graphs/flickrEdges_renum_v --gsuf .txt -s 1 -e 1 --lp clustering_programs_5_2/ -o real_world_graphs/
command_list = []
command_list.append('python')
command_list.append('cluster.py')
command_list.append('-m')
command_list.append('blondel')
command_list.append('infomap')
command_list.append('label_propagation')
command_list.append('oslom')
command_list.append('mod_opt')
command_list.append('-sample')
command_list.append(str(args.samplerate))
command_list.append('-t')
command_list.append(str(args.numtrials))
command_list.append('-u')
command_list.append('--gpre')
command_list.append(working_directory + 'network_sample_p'+ str(int(100*args.samplerate)) +'_v')
command_list.append('--gsuf')
command_list.append('.dat')
command_list.append('-s')
command_list.append(str(args.start))
command_list.append('-e')
command_list.append(str(args.end))
command_list.append('--lp')
command_list.append(new_lancichinetti_program_path)
command_list.append('--Xmx')
command_list.append(args.xmx)
command_list.append('-o')
command_list.append(working_directory)
#To be run in parallel
lancichinetti_copy_start = time.clock()
shutil.copytree(args.lancichinetti_program_path, new_lancichinetti_program_path)
lancichinetti_copy_end = time.clock()
print '\nTime to copy lancichinetti_program: ' + str(lancichinetti_copy_end - lancichinetti_copy_start) + '\n'
subprocess.call(command_list)
lancichinetti_remove_start = time.clock()
shutil.rmtree(new_lancichinetti_program_path)
lancichinetti_remove_end = time.clock()
print '\nTime to remove lancichinetti_program: ' + str(lancichinetti_remove_end - lancichinetti_remove_start) + '\n'
print '\n' + ('*' * (61 + len(n)))
print '* Completed clustering analysis "cluster" workflow for N = ' + n + ' *'
print '*' * (61 + len(n)) + '\n'
if 'measure' in args.modules:
# python measure.py --gmap gmap/ --lnmi mutual3/ --gpre generated_benches/n_1000/network_sample_p70_v --gsuf .dat
# -u --srun --spre generated_benches/n_1000/community_sample_p70_v --ssuf .dat -s 1 -e 3
# --cnames blondel infomap label_propagation oslom modularity_optimization
# --cpre generated_benches/n_1000/blondel_p70_clustering_v
# generated_benches/n_1000/infomap_p70_clustering_v
# generated_benches/n_1000/label_propagation_p70_clustering_v
# generated_benches/n_1000/oslom_p70_clustering_v
# generated_benches/n_1000/modularity_optimization_p70_clustering_v
# --csuf .dat --cnum 2 -o generated_benches/n_1000/ -delta 0.4 -sr 0.7
# python measure.py --gmap gmap/ --lnmi mutual3/ --gpre generated_benches/n_1000/network_v --gsuf .dat --gsep $'\t' -u --srun --spre generated_benches/n_1000/community_v --ssuf .dat --ssep $'\t' -s 1 -e 10 --cnames blondel infomap label_propagation slm --cpre generated_benches/n_1000/blondel_clustering_v generated_benches/n_1000/infomap_clustering_v generated_benches/n_1000/label_propagation_clustering_v generated_benches/n_1000/slm_clustering_v --csuf .dat --csep $'\t' --cnum 10 -o generated_benches/n_1000/
command_list = []
command_list.append('python')
command_list.append('measure_developing.py')
command_list.append('--gmap')
command_list.append('gmap/')
command_list.append('--lnmi')
command_list.append('mutual3/')
command_list.append('--gpre')
command_list.append(working_directory + 'network_sample_p'+ str(int(100*args.samplerate)) +'_v')
command_list.append('--gsuf')
command_list.append('.dat')
command_list.append('-u')
command_list.append('--srun')
command_list.append('--spre')
# command_list.append(working_directory + 'community_sample_p'+ str(int(100*args.samplerate)) +'_v')
command_list.append(working_directory + 'community_v')
command_list.append('--ssuf')
command_list.append('.dat')
command_list.append('-s')
command_list.append(str(args.start))
command_list.append('-e')
command_list.append(str(args.end))
command_list.append('--cnames')
command_list.append('blondel')
command_list.append('infomap')
command_list.append('label_propagation')
command_list.append('oslom')
command_list.append('mod_opt')
command_list.append('--cpre')
command_list.append(working_directory + 'blondel_p'+ str(int(100*args.samplerate)) +'_clustering_v')
command_list.append(working_directory + 'infomap_p'+ str(int(100*args.samplerate)) +'_clustering_v')
command_list.append(working_directory + 'label_propagation_p'+ str(int(100*args.samplerate)) +'_clustering_v')
command_list.append(working_directory + 'oslom_p'+ str(int(100*args.samplerate)) +'_clustering_v')
command_list.append(working_directory + 'mod_opt_p'+ str(int(100*args.samplerate)) +'_clustering_v')
command_list.append('--csuf')
command_list.append('.dat')
command_list.append('--cnum')
command_list.append(str(args.numtrials))
command_list.append('-o')
command_list.append(working_directory)
command_list.append('-delta')
for ele in args.delta:
command_list.append(ele)
command_list.append('-sr')
command_list.append(str(args.samplerate))
order_measure = ' '.join(command_list)
subprocess.call(command_list)
print '\n' + ('*' * (61 + len(n)))
print '* Completed clustering analysis "measure" workflow for N = ' + n + ' *'
print '*' * (61 + len(n)) + '\n'
if 'visualize' in args.modules:
# Combine output raw data from previous step
#'generated_benches_u'+str(int(100*args.mu))+'_p_'+ str(int(100*args.samplerate))+'_condition_'+ args.sampling_condition
for n in args.n_list:
# combineDataFiles('raw_data', '.csv', 'generated_benches/n_' + n)
combineDataFiles('raw_data', '.csv', benchmark_directory+'n_'+n)
# Combined output timing data from previous step
for n in args.n_list:
# combineDataFiles('timing_data', '.csv', 'generated_benches/n_' + n)
combineDataFiles('timing_data', '.csv', benchmark_directory+'n_'+n)
# python visualize.py -f generated_benches/n_100/raw_data.csv generated_benches/n_1000/raw_data.csv generated_benches/n_10000/raw_data.csv --names n_100 n_1000 n_10000 --nodes 100 1000 10000
command_list = []
command_list.append('python')
command_list.append('visualize_developing.py')
command_list.append('-f')
for n in args.n_list:
command_list.append(benchmark_directory+'n_'+ n + '/raw_data.csv')
command_list.append('--names')
for n in args.n_list:
command_list.append('n_' + n)
command_list.append('--nodes')
for n in args.n_list:
command_list.append(n)
command_list.append('--working_dir')
command_list.append(benchmark_directory)
subprocess.call(command_list)
print '\n' + ('*' * (63 + len(n)))
print '* Completed clustering analysis "visualize" workflow for N = ' + n + ' *'
print '*' * (63 + len(n)) + '\n'