From 20ddfacd767c3bd248be6a31625deacb37342849 Mon Sep 17 00:00:00 2001 From: Guenther Schmuelling Date: Wed, 26 Aug 2020 09:27:23 -0700 Subject: [PATCH 1/4] ignore certain loadgen errors --- tools/submission/submission-checker.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index f1e5cc31e..0f9c9cdfe 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -286,7 +286,7 @@ def split_path(m): return m.replace("\\", "/").split("/") -def ignore_errors_for_v0_5(line): +def ignore_errors(line): if "check for ERROR in detailed" in line: return True if "Loadgen built with uncommitted changes" in line: @@ -325,9 +325,8 @@ def check_accuracy_dir(config, model, path): for line in f: # look for: ERROR if "ERROR" in line: - if config.version in ["v0.5"] and ignore_errors_for_v0_5(line): + if ignore_errors(line): continue - # TODO: should this be a failed run? log.error("%s contains error: %s", fname, line) is_valid = False return is_valid, acc @@ -359,7 +358,7 @@ def check_performance_dir(config, model, path): for line in f: # look for: ERROR if "ERROR" in line: - if config.version in ["v0.5"] and ignore_errors_for_v0_5(line): + if ignore_errors(line): continue log.error("%s contains error: %s", fname, line) is_valid = False From 915c223048641db00159d02412f8066a3ccdbb5c Mon Sep 17 00:00:00 2001 From: Guenther Schmuelling Date: Thu, 27 Aug 2020 08:40:15 -0700 Subject: [PATCH 2/4] hang ignored errors on the mlperf version --- tools/submission/submission-checker.py | 31 ++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index 2b91e48a7..a4a3e3c45 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -60,6 +60,12 @@ "sample_index_rng_seed": 665484352860916858, "schedule_rng_seed": 3622009729038561421, }, + "ignore_errors": [ + "check for ERROR in detailed", + "Loadgen built with uncommitted changes", + "Ran out of generated queries to issue before the minimum query count and test duration were reached", + "CAS failed", + ], }, "v0.7": { "models": [ @@ -131,6 +137,9 @@ "sample_index_rng_seed": 665484352860916858, "schedule_rng_seed": 3622009729038561421, }, + "ignore_errors": [ + "CAS failed", + ], }, } @@ -261,6 +270,12 @@ def get_performance_sample_count(self, model): raise ValueError("model not known: " + model) return self.performance_sample_count[model] + def ignore_errors(self, line): + for error in self.base["ignore_errors"]: + if error in line: + return True + return False + def get_args(): """Parse commandline.""" @@ -287,18 +302,6 @@ def split_path(m): return m.replace("\\", "/").split("/") -def ignore_errors(line): - if "check for ERROR in detailed" in line: - return True - if "Loadgen built with uncommitted changes" in line: - return True - if "Ran out of generated queries to issue before the minimum query count and test duration were reached" in line: - return True - if "CAS failed" in line: - return True - return False - - def check_accuracy_dir(config, model, path): is_valid = False acc = None @@ -345,7 +348,7 @@ def check_accuracy_dir(config, model, path): for line in f: # look for: ERROR if "ERROR" in line: - if ignore_errors(line): + if config.ignore_errors(line): continue log.error("%s contains error: %s", fname, line) is_valid = False @@ -379,7 +382,7 @@ def check_performance_dir(config, model, path): for line in f: # look for: ERROR if "ERROR" in line: - if ignore_errors(line): + if config.ignore_errors(line): continue log.error("%s contains error: %s", fname, line) is_valid = False From 5f00f9491064f041dc7ede4c09ff8387861ada4f Mon Sep 17 00:00:00 2001 From: Guenther Schmuelling Date: Thu, 27 Aug 2020 09:54:24 -0700 Subject: [PATCH 3/4] apply mapping before checking valid model --- tools/submission/submission-checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index a4a3e3c45..650eecab8 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -497,15 +497,15 @@ def check_results_dir(config, filter_submitter, csv): # we are looking at ./$division/$submitter/$system_desc/$model, # ie ./closed/mlperf_org/t4-ort/bert name = os.path.join(results_path, system_desc, model_name) + mlperf_model = config.get_mlperf_model(model_name) - if is_closed and model_name not in config.models: + if is_closed and mlperf_model not in config.models: # for closed division we want the model name to match. # for open division the model_name might be different than the task - log.error("%s has a invalid model (%s) for closed division", name, model_name) + log.error("%s has a invalid model %s for closed division", name, model_name) results[name] = None continue - mlperf_model = config.get_mlperf_model(model_name) # # Look at each scenario From b5fb63946fa732cc2f51bc0202c3f906fb35c605 Mon Sep 17 00:00:00 2001 From: Guenther Schmuelling Date: Thu, 27 Aug 2020 10:01:48 -0700 Subject: [PATCH 4/4] fix for https://github.com/mlperf/inference/issues/705 --- tools/submission/submission-checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/submission/submission-checker.py b/tools/submission/submission-checker.py index 650eecab8..cb187669d 100755 --- a/tools/submission/submission-checker.py +++ b/tools/submission/submission-checker.py @@ -71,8 +71,8 @@ "models": [ "ssd-small", "ssd-large", "resnet", "rnnt", "bert-99", "bert-99.9", - "dlrm-99", "dlrm-99.9" - "3d-unet-99", "3d-unet-99.9" + "dlrm-99", "dlrm-99.9", + "3d-unet-99", "3d-unet-99.9", ], "required-scenarios-datacenter": { "resnet": ["Server", "Offline"],