From 26807d2328ddad8c90b6843b661e05e643a6b31f Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Mon, 31 Aug 2020 12:31:27 -0700 Subject: [PATCH] Add user.conf support in RN50/SSDs/DLRM reference implementation --- language/bert/run.py | 2 +- recommendation/dlrm/pytorch/README.md | 7 +++++-- recommendation/dlrm/pytorch/python/main.py | 18 +++++++++++++----- speech_recognition/rnnt/run.py | 2 +- vision/classification_and_detection/README.md | 10 +++++++--- .../python/main.py | 18 +++++++++++++----- vision/medical_imaging/3d-unet/run.py | 2 +- 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/language/bert/run.py b/language/bert/run.py index 26f25a1c7..37d5e8844 100644 --- a/language/bert/run.py +++ b/language/bert/run.py @@ -32,7 +32,7 @@ def get_args(): parser.add_argument("--quantized", action="store_true", help="use quantized model (only valid for onnxruntime backend)") parser.add_argument("--profile", action="store_true", help="enable profiling (only valid for onnxruntime backend)") parser.add_argument("--mlperf_conf", default="build/mlperf.conf", help="mlperf rules config") - parser.add_argument("--user_conf", default="user.conf", help="mlperf rules config") + parser.add_argument("--user_conf", default="user.conf", help="user config for user LoadGen settings such as target QPS") args = parser.parse_args() return args diff --git a/recommendation/dlrm/pytorch/README.md b/recommendation/dlrm/pytorch/README.md index 035dcf6cd..c67fea4d3 100755 --- a/recommendation/dlrm/pytorch/README.md +++ b/recommendation/dlrm/pytorch/README.md @@ -320,7 +320,8 @@ python tools/accuracy-dlrm.py --mlperf-accuracy-file --d ### Usage ``` usage: main.py [-h] - [--config ../mlperf.conf] + [--mlperf_conf ../../mlperf.conf] + [--user_conf user.conf] [--model MODEL] --model-path MODEL_PATH [--dataset {kaggle,terabyte}] --dataset-path DATASET_PATH [--profile {defaults,dlrm-kaggle-pytorch,dlrm-terabyte-pytorch}] @@ -342,7 +343,9 @@ usage: main.py [-h] [--accuracy] [--find-peak-performance] ``` -`--config` the mlperf config file to use (default: `v0.5/mlperf.conf`). +`--mlperf_conf` the mlperf config file to use for rules compliant parameters (default: ../../mlperf.conf) + +`--user_conf` the user config file to use for user LoadGen settings such as target QPS (default: user.conf) `--model` model name, i.e. `dlrm`. diff --git a/recommendation/dlrm/pytorch/python/main.py b/recommendation/dlrm/pytorch/python/main.py index 4dc294cfa..c3d7ebdec 100755 --- a/recommendation/dlrm/pytorch/python/main.py +++ b/recommendation/dlrm/pytorch/python/main.py @@ -133,7 +133,9 @@ def get_args(): parser.add_argument("--find-peak-performance", action="store_true", help="enable finding peak performance pass") # file to use mlperf rules compliant parameters - parser.add_argument("--config", default="../mlperf.conf", help="mlperf rules config") + parser.add_argument("--mlperf_conf", default="../../mlperf.conf", help="mlperf rules config") + # file for user LoadGen settings such as target QPS + parser.add_argument("--user_conf", default="user.conf", help="user config for user LoadGen settings such as target QPS") # below will override mlperf rules compliant settings - don't use for official submission parser.add_argument("--duration", type=int, help="duration in milliseconds (ms)") @@ -459,9 +461,14 @@ def main(): "cmdline": str(args), } - config = os.path.abspath(args.config) - if not os.path.exists(config): - log.error("{} not found".format(config)) + mlperf_conf = os.path.abspath(args.mlperf_conf) + if not os.path.exists(mlperf_conf): + log.error("{} not found".format(mlperf_conf)) + sys.exit(1) + + user_conf = os.path.abspath(args.user_conf) + if not os.path.exists(user_conf): + log.error("{} not found".format(user_conf)) sys.exit(1) if args.output: @@ -504,7 +511,8 @@ def process_latencies(latencies_ns): last_timeing = [t / NANO_SEC for t in latencies_ns] settings = lg.TestSettings() - settings.FromConfig(config, args.model, args.scenario) + settings.FromConfig(mlperf_conf, args.model_name, args.scenario) + settings.FromConfig(user_conf, args.model_name, args.scenario) settings.scenario = scenario settings.mode = lg.TestMode.PerformanceOnly diff --git a/speech_recognition/rnnt/run.py b/speech_recognition/rnnt/run.py index 3ca632e3f..8ee3143ac 100644 --- a/speech_recognition/rnnt/run.py +++ b/speech_recognition/rnnt/run.py @@ -30,7 +30,7 @@ def get_args(): parser.add_argument("--scenario", choices=["SingleStream", "Offline", "Server"], default="Offline", help="Scenario") parser.add_argument("--accuracy", action="store_true", help="enable accuracy pass") parser.add_argument("--mlperf_conf", default=str(MLPERF_CONF), help="mlperf rules config") - parser.add_argument("--user_conf", default="user.conf", help="mlperf rules config") + parser.add_argument("--user_conf", default="user.conf", help="user config for user LoadGen settings such as target QPS") parser.add_argument("--pytorch_config_toml", default="pytorch/configs/rnnt.toml") parser.add_argument("--pytorch_checkpoint", default="pytorch/work_dir/rnnt.pt") parser.add_argument("--dataset_dir", required=True) diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index 6aa8d1b2a..d9f9f5c15 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -195,7 +195,8 @@ If you want run with accuracy pass, try: ### Usage ``` usage: main.py [-h] - [--config ../mlperf.conf] + [--mlperf_conf ../../mlperf.conf] + [--user_conf user.conf] [--dataset {imagenet,imagenet_mobilenet,coco,coco-300,coco-1200,coco-1200-onnx,coco-1200-pt,coco-1200-tf}] --dataset-path DATASET_PATH [--dataset-list DATASET_LIST] [--data-format {NCHW,NHWC}] @@ -208,8 +209,11 @@ usage: main.py [-h] [--max-latency MAX_LATENCY] [--cache CACHE] [--accuracy] ``` -```--config``` -the mlperf config file to use, defaults to v0.5/mlperf.conf +```--mlperf_conf``` +the mlperf config file to use for rules compliant parameters, defaults to ../../mlperf.conf + +```--user_conf``` +the user config file to use for user LoadGen settings such as target QPS, defaults to user.conf ```--dataset``` use the specified dataset. Currently we only support ImageNet. diff --git a/vision/classification_and_detection/python/main.py b/vision/classification_and_detection/python/main.py index 677ca92c3..cd6825fff 100755 --- a/vision/classification_and_detection/python/main.py +++ b/vision/classification_and_detection/python/main.py @@ -193,7 +193,9 @@ def get_args(): parser.add_argument("--find-peak-performance", action="store_true", help="enable finding peak performance pass") # file to use mlperf rules compliant parameters - parser.add_argument("--config", default="../mlperf.conf", help="mlperf rules config") + parser.add_argument("--mlperf_conf", default="../../mlperf.conf", help="mlperf rules config") + # file for user LoadGen settings such as target QPS + parser.add_argument("--user_conf", default="user.conf", help="user config for user LoadGen settings such as target QPS") # below will override mlperf rules compliant settings - don't use for official submission parser.add_argument("--time", type=int, help="time to scan in seconds") @@ -435,9 +437,14 @@ def main(): "cmdline": str(args), } - config = os.path.abspath(args.config) - if not os.path.exists(config): - log.error("{} not found".format(config)) + mlperf_conf = os.path.abspath(args.mlperf_conf) + if not os.path.exists(mlperf_conf): + log.error("{} not found".format(mlperf_conf)) + sys.exit(1) + + user_conf = os.path.abspath(args.user_conf) + if not os.path.exists(user_conf): + log.error("{} not found".format(user_conf)) sys.exit(1) if args.output: @@ -478,7 +485,8 @@ def process_latencies(latencies_ns): last_timeing = [t / NANO_SEC for t in latencies_ns] settings = lg.TestSettings() - settings.FromConfig(config, args.model_name, args.scenario) + settings.FromConfig(mlperf_conf, args.model_name, args.scenario) + settings.FromConfig(user_conf, args.model_name, args.scenario) settings.scenario = scenario settings.mode = lg.TestMode.PerformanceOnly if args.accuracy: diff --git a/vision/medical_imaging/3d-unet/run.py b/vision/medical_imaging/3d-unet/run.py index 7582e3fc1..e69f2b6e7 100644 --- a/vision/medical_imaging/3d-unet/run.py +++ b/vision/medical_imaging/3d-unet/run.py @@ -43,7 +43,7 @@ def get_args(): help="mlperf rules config") parser.add_argument("--user_conf", default="user.conf", - help="mlperf rules config") + help="user config for user LoadGen settings such as target QPS") parser.add_argument( "--model_dir", default=