task_config_template.cfg

[metrics_parameters_inference]
patterns = ['Prediction-Time: (\d+\.\d+|\d+) milliseconds']
metrics = ['prediction_time']
compute_method = ['average']

[metrics_parameters_imperative_hybrid_top_1]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']

[metrics_parameters_images_top_1]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total', 'last']

[metrics_parameters_images_top_5]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Train-top_k_accuracy_5=(\d+\.\d+|\d+)','Validation-top_k_accuracy_5=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc','Train-top_k_accuracy_5','Validation-top_k_accuracy_5']
compute_method = ['average', 'last', 'total', 'last','last','last']

[resnet50_cifar10_symbolic]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total', 'last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp16_batch_size64]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --dtype float16 --batch-size 64 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp32_batch_size32]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --batch-size 32  --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp16_batch_size32]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --dtype float16 --batch-size 32 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp32_batch_size16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --batch-size 16 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp32_batch_size64]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --batch-size 64 --kvstore device
num_gpus = 8

[resnet50_cifar10_hybrid]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode hybrid --gpus 8 --epochs 20 --log-interval 50 --kvstore device
num_gpus = 8

[resnet50_cifar10_imperative]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --gpus 8 --epochs 20 --log-interval 50 --kvstore device
num_gpus = 8

[dawnbench_cifar10_symbolic]
patterns = ['Epoch \d+, Batch \d+, Speed=(\d+\.\d+|\d+)', 'Epoch \d+, Training accuracy=(\d+\.\d+|\d+)', 'Epoch \d+, Validation accuracy=(\d+\.\d+|\d+)', 'Epoch \d+, Duration=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']
command_to_execute = python dawnbench/cifar10.py --gpus 4 --early-stopping-acc 0.86 --epochs 400 --lr 0.05 --total-batch-size 256
num_gpus = 4

[lstm_ptb_imperative]
patterns = ['time cost (\d+\.\d+|\d+)', 'valid loss (\d+\.\d+|\d+)', 'valid ppl (\d+\.\d+|\d+)', 'test loss (\d+\.\d+|\d+)', 'test ppl (\d+\.\d+|\d+)']
metrics = ['total_training_time', 'validation_loss', 'validation_perplexity', 'test_loss', 'test_perplexity']
compute_method = ['total', 'last', 'last', 'last', 'last']
command_to_execute = python word_language_model/word_language_model.py --gpus 8 --nhid 650 --emsize 650 --dropout 0.5 --epochs 40 --data word_language_model/data/ptb. --mode imperative --kvstore device
num_gpus = 8

[lstm_ptb_hybrid]
patterns = ['time cost (\d+\.\d+|\d+)', 'valid loss (\d+\.\d+|\d+)', 'valid ppl (\d+\.\d+|\d+)', 'test loss (\d+\.\d+|\d+)', 'test ppl (\d+\.\d+|\d+)']
metrics = ['total_training_time', 'validation_loss', 'validation_perplexity', 'test_loss', 'test_perplexity']
compute_method = ['total', 'last', 'last', 'last', 'last']
command_to_execute = python word_language_model/word_language_model.py --gpus 8 --nhid 650 --emsize 650 --dropout 0.5 --epochs 40 --data word_language_model/data/ptb. --mode hybrid --kvstore device
num_gpus = 8

[lstm_ptb_symbolic]
patterns = ['Time cost=(\d+\.\d+|\d+)', 'Train-perplexity=(\d+\.\d+|\d+)', 'Validation-perplexity=(\d+\.\d+|\d+)', 'Speed: (\d+\.\d+|\d+) samples/sec']
metrics = ['total_training_time', 'train_perplexity', 'validation_perplexity', 'speed']
compute_method = ['total', 'last', 'last', 'average']
command_to_execute = python word_language_model/lstm_bucketing.py --num-hidden 650 --num-embed 650 --gpus 8 --epochs 25 --kv-store device
num_gpus = 8

[resnet50_imagenet_symbolic_fp16_batch_size32_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 256 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size64_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size128_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 1024 --data-nthreads 32 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size32_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 256 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size64_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size32_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 256 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size64_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 512 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size32_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 256 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size64_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 512 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[metrics_parameters_distributed_top_k]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Train-top_k_accuracy_\d=(\d+\.\d+|\d+)','Validation-top_k_accuracy_\d=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time','training_acc_top5','validation_acc_top5']
compute_method = ['average_aggregate', 'last', 'last', 'total','last','last']

[metrics_parameters_distributed]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average_aggregate', 'last', 'last', 'total']

[tensorflow_resnet50_p3_2xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=1 --batch_size=32 --model=resnet50 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 1

[tensorflow_resnet50_p3_8xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=4 --batch_size=32 --model=resnet50 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 4

[tensorflow_resnet50_p3_16xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=8 --batch_size=32 --model=resnet50 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 8

[tensorflow_resnet152_p3_2xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=1 --batch_size=32 --model=resnet152 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 1

[tensorflow_resnet152_p3_8xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=4 --batch_size=32 --model=resnet152 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 4

[tensorflow_resnet152_p3_16xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=8 --batch_size=32 --model=resnet152 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 8

[tensorflow_resnet56_p3_2xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=cifar10 --data_dir=cifar-10-batches-py --num_gpus=1 --batch_size=32 --model=resnet56 --variable_update=replicated --print_training_accuracy=True --use_fp16=True --use_tf_layers=False --target_accuracy=0.94
num_gpus = 1

[tensorflow_resnet56_p3_8xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=cifar10 --data_dir=cifar-10-batches-py --num_gpus=4 --batch_size=32 --model=resnet56 --variable_update=replicated --print_training_accuracy=True --use_fp16=True --use_tf_layers=False --target_accuracy=0.94
num_gpus = 4

[tensorflow_resnet56_p3_16xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=cifar10 --data_dir=cifar-10-batches-py --num_gpus=8 --batch_size=32 --model=resnet56 --variable_update=replicated --print_training_accuracy=True --use_fp16=True --use_tf_layers=False --target_accuracy=0.94
num_gpus = 8

[tensorflow_resnet50_p3_2xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=1 --batch_size=32 --model=resnet50 --variable_update=replicated --print_training_accuracy=True --num_batches=100 --use_fp16=True --use_tf_layers=False
num_gpus = 1

[tensorflow_resnet50_p3_8xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=4 --batch_size=32 --model=resnet50 --variable_update=replicated --print_training_accuracy=True --num_batches=100 --use_fp16=True --use_tf_layers=False
num_gpus = 4

[tensorflow_resnet50_p3_16xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=8 --batch_size=32 --model=resnet50 --variable_update=replicated --print_training_accuracy=True --num_batches=100 --use_fp16=True --use_tf_layers=False
num_gpus = 8

[mms_resnet18_cpu]
patterns = ['Throughput_concurrency_50_req_25 :(\d+\.\d+|\d+)','Average_latency_concurrency_50_req_25 :(\d+\.\d+|\d+)','Median_latency_concurrency_50_req_25 :(\d+\.\d+|\d+)','P90_latency_concurrency_50_req_25 :(\d+\.\d+|\d+)','Error_rate_concurrency_50_req_25 :(\d+\.\d+|\d+)','Throughput_concurrency_50_req_50 :(\d+\.\d+|\d+)','Average_latency_concurrency_50_req_50 :(\d+\.\d+|\d+)','Median_latency_concurrency_50_req_50 :(\d+\.\d+|\d+)','P90_latency_concurrency_50_req_50 :(\d+\.\d+|\d+)','Error_rate_concurrency_50_req_50 :(\d+\.\d+|\d+)','Throughput_concurrency_50_req_100 :(\d+\.\d+|\d+)','Average_latency_concurrency_50_req_100 :(\d+\.\d+|\d+)','Median_latency_concurrency_50_req_100 :(\d+\.\d+|\d+)','P90_latency_concurrency_50_req_100 :(\d+\.\d+|\d+)','Error_rate_concurrency_50_req_100 :(\d+\.\d+|\d+)','Throughput_concurrency_100_req_25 :(\d+\.\d+|\d+)','Average_latency_concurrency_100_req_25 :(\d+\.\d+|\d+)','Median_latency_concurrency_100_req_25 :(\d+\.\d+|\d+)','P90_latency_concurrency_100_req_25 :(\d+\.\d+|\d+)','Error_rate_concurrency_100_req_25 :(\d+\.\d+|\d+)', 'Throughput_concurrency_100_req_50 :(\d+\.\d+|\d+)','Average_latency_concurrency_100_req_50 :(\d+\.\d+|\d+)','Median_latency_concurrency_100_req_50 :(\d+\.\d+|\d+)','P90_latency_concurrency_100_req_50 :(\d+\.\d+|\d+)','Error_rate_concurrency_100_req_50 :(\d+\.\d+|\d+)', 'Throughput_concurrency_100_req_100 :(\d+\.\d+|\d+)','Average_latency_concurrency_100_req_100 :(\d+\.\d+|\d+)','Median_latency_concurrency_100_req_100 :(\d+\.\d+|\d+)','P90_latency_concurrency_100_req_100 :(\d+\.\d+|\d+)','Error_rate_concurrency_100_req_100 :(\d+\.\d+|\d+)' ]
metrics = ['Throughput_concurrency_50_req_25','Average_latency_concurrency_50_req_25','Median_latency_concurrency_50_req_25','P90_latency_concurrency_50_req_25','Error_rate_concurrency_50_req_25','Throughput_concurrency_50_req_50','Average_latency_concurrency_50_req_50','Median_latency_concurrency_50_req_50','P90_latency_concurrency_50_req_50','Error_rate_concurrency_50_req_50','Throughput_concurrency_50_req_100','Average_latency_concurrency_50_req_100','Median_latency_concurrency_50_req_100','P90_latency_concurrency_50_req_100','Error_rate_concurrency_50_req_100','Throughput_concurrency_100_req_25','Average_latency_concurrency_100_req_25','Median_latency_concurrency_100_req_25','P90_latency_concurrency_100_req_25','Error_rate_concurrency_100_req_25','Throughput_concurrency_100_req_50','Average_latency_concurrency_100_req_50','Median_latency_concurrency_100_req_50','P90_latency_concurrency_100_req_50','Error_rate_concurrency_100_req_50','Throughput_concurrency_100_req_100','Average_latency_concurrency_100_req_100','Median_latency_concurrency_100_req_100','P90_latency_concurrency_100_req_100','Error_rate_concurrency_100_req_100']
compute_method = ['last', 'last', 'last', 'last', 'last','last', 'last', 'last', 'last', 'last','last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last','last', 'last', 'last', 'last', 'last']
command_to_execute =sudo bash /mxnet-model-server/ci/benchmark_ai.sh -g 0
num_gpus = 0

[mms_resnet18_gpu]
patterns = ['Throughput_concurrency_50_req_25 :(\d+\.\d+|\d+)', 'Average_latency_concurrency_50_req_25 :(\d+\.\d+|\d+)','Median_latency_concurrency_50_req_25 :(\d+\.\d+|\d+)','P90_latency_concurrency_50_req_25 :(\d+\.\d+|\d+)','Error_rate_concurrency_50_req_25 :(\d+\.\d+|\d+)','Throughput_concurrency_50_req_50 :(\d+\.\d+|\d+)','Average_latency_concurrency_50_req_50 :(\d+\.\d+|\d+)','Median_latency_concurrency_50_req_50 :(\d+\.\d+|\d+)','P90_latency_concurrency_50_req_50 :(\d+\.\d+|\d+)','Error_rate_concurrency_50_req_50 :(\d+\.\d+|\d+)','Throughput_concurrency_50_req_100 :(\d+\.\d+|\d+)','Average_latency_concurrency_50_req_100 :(\d+\.\d+|\d+)','Median_latency_concurrency_50_req_100 :(\d+\.\d+|\d+)','P90_latency_concurrency_50_req_100 :(\d+\.\d+|\d+)','Error_rate_concurrency_50_req_100 :(\d+\.\d+|\d+)','Throughput_concurrency_100_req_25 :(\d+\.\d+|\d+)','Average_latency_concurrency_100_req_25 :(\d+\.\d+|\d+)','Median_latency_concurrency_100_req_25 :(\d+\.\d+|\d+)','P90_latency_concurrency_100_req_25 :(\d+\.\d+|\d+)','Error_rate_concurrency_100_req_25 :(\d+\.\d+|\d+)','Throughput_concurrency_100_req_50 :(\d+\.\d+|\d+)','Average_latency_concurrency_100_req_50 :(\d+\.\d+|\d+)','Median_latency_concurrency_100_req_50 :(\d+\.\d+|\d+)','P90_latency_concurrency_100_req_50 :(\d+\.\d+|\d+)','Error_rate_concurrency_100_req_50 :(\d+\.\d+|\d+)','Throughput_concurrency_100_req_100 :(\d+\.\d+|\d+)','Average_latency_concurrency_100_req_100 :(\d+\.\d+|\d+)','Median_latency_concurrency_100_req_100 :(\d+\.\d+|\d+)','P90_latency_concurrency_100_req_100 :(\d+\.\d+|\d+)', 'Error_rate_concurrency_100_req_100 :(\d+\.\d+|\d+)' ]
metrics = ['Throughput_concurrency_50_req_25','Average_latency_concurrency_50_req_25','Median_latency_concurrency_50_req_25','P90_latency_concurrency_50_req_25','Error_rate_concurrency_50_req_25','Throughput_concurrency_50_req_50','Average_latency_concurrency_50_req_50','Median_latency_concurrency_50_req_50','P90_latency_concurrency_50_req_50','Error_rate_concurrency_50_req_50','Throughput_concurrency_50_req_100','Average_latency_concurrency_50_req_100','Median_latency_concurrency_50_req_100','P90_latency_concurrency_50_req_100','Error_rate_concurrency_50_req_100','Throughput_concurrency_100_req_25','Average_latency_concurrency_100_req_25','Median_latency_concurrency_100_req_25','P90_latency_concurrency_100_req_25','Error_rate_concurrency_100_req_25','Throughput_concurrency_100_req_50','Average_latency_concurrency_100_req_50','Median_latency_concurrency_100_req_50','P90_latency_concurrency_100_req_50','Error_rate_concurrency_100_req_50','Throughput_concurrency_100_req_100','Average_latency_concurrency_100_req_100','Median_latency_concurrency_100_req_100','P90_latency_concurrency_100_req_100','Error_rate_concurrency_100_req_100']
compute_method = ['last', 'last', 'last', 'last', 'last','last', 'last', 'last', 'last', 'last','last', 'last', 'last', 'last', 'last','last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last','last', 'last', 'last', 'last', 'last']
command_to_execute =sudo bash /mxnet-model-server/ci/benchmark_ai.sh -g 1
num_gpus = 4

[onnx_mxnet_import_model_inference_test_cpu]
patterns = ['Average_inference_time_bvlc_alexnet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_googlenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_caffenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_densenet121_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_resnet50_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_shufflenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_squeezenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_vgg19_cpu: (\d+\.\d+|\d+)']
metrics            = ['Average_inference_time_bvlc_alexnet_cpu','Average_inference_time_bvlc_googlenet_cpu','Average_inference_time_bvlc_reference_caffenet_cpu','Average_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu','Average_inference_time_densenet121_cpu','Average_inference_time_resnet50_cpu','Average_inference_time_shufflenet_cpu','Average_inference_time_squeezenet_cpu','Average_inference_time_vgg19_cpu']
compute_method     = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = sudo bash ./onnx_benchmark/setup.sh "cpu" && python3  ./onnx_benchmark/import_benchmarkscript.py "cpu"
num_gpus           = 1

[onnx_mxnet_import_model_inference_test_gpu]
patterns = ['Average_inference_time_bvlc_alexnet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_googlenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_caffenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_densenet121_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_resnet50_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_shufflenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_squeezenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_vgg19_gpu: (\d+\.\d+|\d+)']
metrics  =['Average_inference_time_bvlc_alexnet_gpu','Average_inference_time_bvlc_googlenet_gpu','Average_inference_time_bvlc_reference_caffenet_gpu','Average_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu','Average_inference_time_densenet121_gpu','Average_inference_time_resnet50_gpu','Average_inference_time_shufflenet_gpu','Average_inference_time_squeezenet_gpu','Average_inference_time_vgg19_gpu']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = sudo bash ./onnx_benchmark/setup.sh "gpu"  && python3  ./onnx_benchmark/import_benchmarkscript.py "gpu"
num_gpus    = 1

[resnet50_imagenet_sagemaker_tf_docker]
patterns = ['QueueInput/queue_size: (\d+\.\d+|\d+)', 'l2_regularize_loss: (\d+\.\d+|\d+)', 'learning_rate: (\d+\.\d+|\d+)', 'train-error-top1: (\d+\.\d+|\d+)', 'train-error-top5: (\d+\.\d+|\d+)', 'xentropy-loss: (\d+\.\d+|\d+)']
metrics = ['queue_size', 'l2_regularize_loss', 'learning_rate', 'train-error-top1', 'train-error-top5', 'xentropy-loss']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = bash sagemaker_testing/tensorflow/tensorpack-resnet-imagenet.sh
num_gpus = 8

[resnet50_imagenet_sagemaker_pt_docker]
patterns = ['Test.*Time (\d+\.\d+|\d+)', 'Data (\d+\.\d+|\d+)', 'Test.*Loss (\d+\.\d+|\d+)', 'Test.*Prec@1 (\d+\.\d+|\d+)', 'Test.*Prec@5 (\d+\.\d+|\d+)']
metrics = ['seconds', 'seconds', 'loss', 'top 1 precision', 'top 5 precision']
compute_method = ['last', 'last', 'last', 'last', 'last']
command_to_execute = bash sagemaker_testing/pytorch/pytorch-examples-resnet50.sh
num_gpus = 8

[resnet50_imagenet_sagemaker_ch_docker]
patterns = []
metrics = []
compute_method = []
command_to_execute = bash sagemaker_testing/chainer/chainer_resnet50.sh
num_gpus = 8

[resnet50_imagenet_sagemaker_mx_docker]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total', 'last']
command_to_execute = bash image_classification/scripts/test.sh
num_gpus = 8

[conda_dlami_tensorflow_180_p36_resnet50_imagenet]
patterns = ['QueueInput/queue_size: (\d+\.\d+|\d+)', 'l2_regularize_loss: (\d+\.\d+|\d+)', 'learning_rate: (\d+\.\d+|\d+)', 'train-error-top1: (\d+\.\d+|\d+)', 'train-error-top5: (\d+\.\d+|\d+)', 'xentropy-loss: (\d+\.\d+|\d+)']
metrics = ['queue_size', 'l2_regularize_loss', 'learning_rate', 'train-error-top1', 'train-error-top5', 'xentropy-loss']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = bash conda_dlami_testing/tensorflow/tensorpack-resnet-imagenet.sh
num_gpus = 8

[mxnet_resnet50_synthetic_p3_16xlg]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Time cost=(\d+\.\d+|\d+)']
metrics = ['speed(samples/sec)', 'Training Time']
compute_method = ['average', 'last']
command_to_execute = python ./conda_dlami_testing/gluon/image_classification.py --model resnet50_v1 --dataset dummy --mode symbolic --gpus "0,1,2,3,4,5,6,7" --epochs 10 --log-interval 50 --batch-size 128  --kvstore device
num_gpus = 8

[tensorflow_resnet50_synthetic_p3_16xlg]
patterns = ['images/sec: (\d+\.\d+)']
metrics = ['Speed(Images/sec)']
compute_method = ['average']
command_to_execute = python ./tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --num_gpus=8 --batch_size=256 --model=resnet50 --variable_update=parameter_server --num_batches=100 --local_parameter_device=cpu --optimizer=sgd --use_fp16=True
num_gpus = 8

[tensorflow_resnet50_synthetic_p3_16xlg_runner]
patterns = ['images/sec: (\d+\.\d+)']
metrics = ['Speed(Images/sec)']
compute_method = ['average']

[pytorch_resnet50_synthetic_p3_16xlg_runner]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'BatchSpeed (\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)']
metrics = ['speed(samples/sec)', 'batchspeed(samples/sec)', 'Time']
compute_method = ['average', 'average', 'last']

[resnet50_synthetic_p3_16xlg_runner]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Time cost=(\d+\.\d+|\d+)']
metrics = ['speed(samples/sec)', 'Time']
compute_method = ['average', 'last']