-
Notifications
You must be signed in to change notification settings - Fork 55
/
test_baselines.py
109 lines (86 loc) · 3.7 KB
/
test_baselines.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
"""Evaluating simple baselines for the CVPR2020 video pentathlon challenge
Example usage (evaluates checkpoints produced by the train_baselines.py script):
# add the project root folder to the pythonpath
export PYTHONPATH=$(pwd):$PYTHONPATH
# provide path to checkpoints (produced by train_baselines.py)
CKPT_LIST=data/cvpr2020-challenge-submissions/ckpts-baselines-2020-04-05_09-53-14-public_server_val-MSRVTT-MSVD-DiDeMo-YouCook2-activity-net.json
python misc/cvpr2020_challenge/test_baselines.py --ckpt_list_path ${CKPT_LIST}
"""
import json
import argparse
from pathlib import Path
from datetime import datetime
from typeguard import typechecked
from misc.cvpr2020_challenge.prepare_submission import generate_predictions
from misc.cvpr2020_challenge.train_baselines import (
parse_paths_from_logs,
launch_and_monitor_cmd
)
@typechecked
def json_key2dataset_name(json_key: str) -> str:
"""Convert json key used for a given datset into the name used in the codebase.
Args:
json_key: the key to be converted
Returns:
the name of the dataset
NOTE: This is used to patch around inconsistency in the naming scheme used for
ActivityNet and every other datsaet.
"""
dataset_name = {"ActivityNet": "activity-net"}.get(json_key, json_key)
return dataset_name
@typechecked
def evaluate_from_ckpts(
ckpt_list_path: Path,
challenge_config_dir: Path,
dest_dir: Path,
device: int,
):
with open(ckpt_list_path, "r") as f:
ckpts = json.load(f)
challenge_phase = "public_server_test"
timestamp = datetime.now().strftime(r"%Y-%m-%d_%H-%M-%S")
test_set_prediction_paths = {}
datasets = []
for json_key, ckpt_path in ckpts.items():
dataset = json_key2dataset_name(json_key)
datasets.append(dataset)
folder = dataset.lower()
config_path = challenge_config_dir / folder / "baseline-public-test.json"
flags = (f" --config {config_path} --device {device} --eval_from_training_config"
f" --resume {ckpt_path}")
cmd = f"python -u test.py {flags}"
print(f"Launching baseline evaluation for {dataset} with command: {cmd}")
logs = launch_and_monitor_cmd(cmd=cmd)
exp_paths = parse_paths_from_logs(logs, queries=["predictions"])
test_set_prediction_paths[json_key] = {challenge_phase: exp_paths["predictions"]}
fname = f"baselines-{timestamp}-{challenge_phase}-{'-'.join(datasets)}.json"
dest_path = dest_dir / f"predictions-{fname}"
print(f"Writing baseline predicitions list to {dest_path}")
with open(dest_path, "w") as f:
json.dump(test_set_prediction_paths, f, indent=4, sort_keys=True)
generate_predictions(
refresh=True,
validate=True,
dest_dir=dest_dir,
challenge_phase=challenge_phase,
predictions_path=dest_path,
)
def main():
parser = argparse.ArgumentParser(description="evaluate baselines from ckpts")
parser.add_argument("--device", type=int, default=0,
help="gpu device to use for training")
parser.add_argument("--ckpt_list_path", type=Path, required=True,
help="Path to json file containing model checkpoint locations")
parser.add_argument('--challenge_config_dir', type=Path,
default="configs/cvpr2020-challenge")
parser.add_argument('--dest_dir', type=Path,
default="data/cvpr2020-challenge-submissions")
args = parser.parse_args()
evaluate_from_ckpts(
device=args.device,
dest_dir=args.dest_dir,
challenge_config_dir=args.challenge_config_dir,
ckpt_list_path=args.ckpt_list_path,
)
if __name__ == "__main__":
main()