-
Notifications
You must be signed in to change notification settings - Fork 5
/
run_experiment.py
149 lines (132 loc) · 7.66 KB
/
run_experiment.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
# Created by MegaFace Team
# Please cite the our paper if you use our code, results, or dataset in a publication
# http://megaface.cs.washington.edu/
import argparse
import os
import sys
import json
import random
import subprocess
import nsml
MODEL = os.path.join('models', 'jb_identity.bin')
IDENTIFICATION_EXE = os.path.join('bin', 'Identification')
FUSE_RESULTS_EXE = os.path.join('bin', 'FuseResults')
MEGAFACE_LIST_BASENAME = os.path.join('templatelists','megaface_features_list.json')
PROBE_LIST_BASENAME = os.path.join('templatelists','facescrub_features_list.json')
def main():
# print(os.environ['LD_LIBRARY_PATH'])
parser = argparse.ArgumentParser(description=
'Runs the MegaFace challenge experiment with the provided feature files')
parser.add_argument('--distractor_feature_path', help='Path to MegaFace Features')
parser.add_argument('--probe_feature_path', help='Path to FaceScrub Features')
parser.add_argument('--file_ending',help='Ending appended to original photo files. i.e. 11084833664_0.jpg_LBP_100x100.bin => _LBP_100x100.bin')
parser.add_argument(
'--out_root', help='File output directory, outputs results files, score matrix files, and feature lists used')
parser.add_argument('-s', '--sizes', type=int, nargs='+',
help='(optional) Size(s) of feature list(s) to create. Default: 10 100 1000 10000 100000 1000000')
parser.add_argument('-m', '--model', type=str,
help='(optional) Scoring model to use. Default: ../models/jb_identity.bin')
parser.add_argument('-ns','--num_sets', help='Set to change number of sets to run on. Default: 1')
parser.add_argument('-d','--delete_matrices', dest='delete_matrices', action='store_true', help='Deletes matrices used while computing results. Reduces space needed to run test.')
parser.add_argument('-p','--probe_list', help='Set to use different probe list. Default: ../templatelists/facescrub_features_list.json')
parser.add_argument('-dlp','--distractor_list_path', help='Set to change path used for distractor lists')
parser.set_defaults(model=MODEL, num_sets=1, file_endint='.bin', sizes=[1000000], probe_list=PROBE_LIST_BASENAME, distractor_list_path=os.path.dirname(MEGAFACE_LIST_BASENAME))
args = parser.parse_args()
distractor_feature_path = args.distractor_feature_path
out_root = args.out_root
probe_feature_path = args.probe_feature_path
file_ending = args.file_ending
# feature_path
distractor_feature_path = os.path.join(file_ending.split('.')[0], 'megaface_bin')
print(distractor_feature_path)
probe_feature_path = os.path.join(file_ending.split('.')[0], 'facescrub_bin')
print(probe_feature_path)
model = args.model
num_sets = args.num_sets
sizes = args.sizes
alg_name = file_ending.split('.')[0].strip('_')
delete_matrices = args.delete_matrices
probe_list_basename = args.probe_list
megaface_list_basename = os.path.join(args.distractor_list_path,os.path.basename(MEGAFACE_LIST_BASENAME))
set_indices = range(1,int(num_sets) + 1)
os.chmod(IDENTIFICATION_EXE, 0o755)
os.chmod(FUSE_RESULTS_EXE, 0o755)
assert os.path.exists(distractor_feature_path)
assert os.path.exists(probe_feature_path)
if not os.path.exists(out_root):
os.makedirs(out_root)
if(not os.path.exists(os.path.join(out_root, "otherFiles"))):
os.makedirs(os.path.join(out_root, "otherFiles"))
other_out_root = os.path.join(out_root, "otherFiles")
probe_name = os.path.basename(probe_list_basename).split('_')[0]
distractor_name = os.path.basename(megaface_list_basename).split('_')[0]
#Create feature lists for megaface for all sets and sizes and verifies all features exist
missing = False
for index in set_indices:
for size in sizes:
print('Creating feature list of {} photos for set {}'.format(size,str(index)))
cur_list_name = megaface_list_basename + "_{}_{}".format(str(size), str(index))
print(cur_list_name)
with open(cur_list_name) as fp:
featureFile = json.load(fp)
path_list = featureFile["path"]
for i in range(len(path_list)):
path_list[i] = os.path.join(distractor_feature_path,os.path.splitext(path_list[i])[0] + file_ending)
if(not os.path.isfile(path_list[i])):
print(path_list[i] + " is missing")
missing = True
if (i % 10000 == 0 and i > 0):
print(str(i) + " / " + str(len(path_list)))
featureFile["path"] = path_list
json.dump(featureFile, open(os.path.join(
other_out_root, '{}_features_{}_{}_{}'.format(distractor_name,alg_name,size,index)), 'w'), sort_keys=True, indent=4)
if(missing):
sys.exit("Features are missing...")
#Create feature list for probe set
with open(probe_list_basename) as fp:
featureFile = json.load(fp)
path_list = featureFile["path"]
for i in range(len(path_list)):
path_list[i] = os.path.join(probe_feature_path,os.path.splitext(path_list[i])[0] + file_ending)
if(not os.path.isfile(path_list[i])):
print(path_list[i] + " is missing")
missing = True
featureFile["path"] = path_list
json.dump(featureFile, open(os.path.join(
other_out_root, '{}_features_{}'.format(probe_name,alg_name)), 'w'), sort_keys=True, indent=4)
probe_feature_list = os.path.join(other_out_root, '{}_features_{}'.format(probe_name,alg_name))
if(missing):
sys.exit("Features are missing...")
print('Running probe to probe comparison')
probe_score_filename = os.path.join(
other_out_root, '{}_{}_{}.bin'.format(probe_name, probe_name, alg_name))
proc = subprocess.Popen(
[IDENTIFICATION_EXE, model, "path", probe_feature_list, probe_feature_list, probe_score_filename])
proc.communicate()
for index in set_indices:
for size in sizes:
print('Running test with size {} images for set {}'.format(
str(size), str(index)))
args = [IDENTIFICATION_EXE, model, "path", os.path.join(other_out_root, '{}_features_{}_{}_{}'.format(distractor_name,alg_name,size,index)
), probe_feature_list, os.path.join(other_out_root, '{}_{}_{}_{}_{}.bin'.format(probe_name, distractor_name, alg_name, str(size),str(index)))]
proc = subprocess.Popen(args)
proc.communicate()
print('Computing test results with {} images for set {}'.format(
str(size), str(index)))
args = [FUSE_RESULTS_EXE]
args += [os.path.join(other_out_root, '{}_{}_{}_{}_{}.bin'.format(
probe_name, distractor_name, alg_name, str(size), str(index)))]
args += [os.path.join(other_out_root, '{}_{}_{}.bin'.format(
probe_name, probe_name, alg_name)), probe_feature_list, str(size)]
args += [os.path.join(out_root, "cmc_{}_{}_{}_{}_{}.json".format(
probe_name, distractor_name, alg_name, str(size), str(index)))]
args += [os.path.join(out_root, "matches_{}_{}_{}_{}_{}.json".format(
probe_name, distractor_name, alg_name, str(size), str(index)))]
proc = subprocess.Popen(args)
proc.communicate()
print('Finish test with size {} images for set {}'.format(str(size), str(index)))
if(delete_matrices):
os.remove(os.path.join(other_out_root, '{}_{}_{}_{}_{}.bin'.format(
probe_name, distractor_name, alg_name, str(size), str(index))))
if __name__ == '__main__':
main()