[Scripts] Add subcommand stress

scripts/bash_completion/stress.options

@@ -0,0 +1,16 @@
+--help
+--sensitive
+--warning-sensitive
+--hack-sensitive
+--model-solution=
+--rounds=
+--import
+--seed=
+--no-val
+--no-sol-compile
+--no-model
+--no-check
+--no-tle
+--time-limit=
+--hard-time-limit=
+--min-score=

scripts/internal/get_global_validators.py

@@ -0,0 +1,20 @@
+import os
+
+from util import load_json, log_warning, unify_list
+
+
+SUBTASKS_JSON = os.environ.get('SUBTASKS_JSON')
+
+
+def get_global_validators():
+    data = load_json(SUBTASKS_JSON)
+
+    validators = list(data.get('global_validators', []))
+    if len(validators) == 0:
+        log_warning("There is no global validator.")
+    return unify_list(validators)
+
+
+if __name__ == '__main__':
+    for validator in get_global_validators():
+        print(validator)

scripts/internal/problem_util.sh

@@ -83,3 +83,12 @@ function get_test_validator_commands {
 	test_name="$1"; shift
 	get_test_validators "${test_name}" "${tests_dir}" | convert_validator_names_to_commands || return $?
 }
+
+
+function get_global_validators {
+	"${PYTHON}" "${INTERNALS}/get_global_validators.py"
+}
+
+function get_global_validator_commands {
+	get_global_validators | convert_validator_names_to_commands || return $?
+}

scripts/internal/stress.py

@@ -0,0 +1,119 @@
+import sys
+import os
+import shlex
+import random
+import subprocess
+import importlib.util
+
+from util import simple_usage_message, wait_process_success
+from color_util import cprint, colors
+
+
+INTERNALS_DIR = os.environ.get('INTERNALS')
+GEN_STR_RAND_SEED = os.environ.get('GEN_STR_RAND_SEED')
+
+
+def _main():
+    if len(sys.argv) != 3:
+        simple_usage_message("<verify|number-of-rounds> <test-case-generation-file>")
+
+    verification_mode = False
+    if sys.argv[1] == "verify":
+        verification_mode = True
+    else:
+        num_rounds = int(sys.argv[1])
+    test_gen_python_file = sys.argv[2]
+
+    GEN_COMMAND_FUNC_NAME = "gen_command"
+
+    try:
+
+        test_gen_spec = importlib.util.spec_from_file_location("test_gen_module", test_gen_python_file)
+        if test_gen_spec is None:
+            raise Exception("Not a valid spec.")
+        test_gen_module = importlib.util.module_from_spec(test_gen_spec)
+        sys.modules["test_gen_module"] = test_gen_module
+        test_gen_spec.loader.exec_module(test_gen_module)
+        if not hasattr(test_gen_module, GEN_COMMAND_FUNC_NAME):
+            raise Exception("The module does not have a member named '{}'.".format(GEN_COMMAND_FUNC_NAME))
+        gen_command_func = getattr(test_gen_module, GEN_COMMAND_FUNC_NAME)
+        if not callable(gen_command_func):
+            raise Exception("The object '{}' defined in the module is not callable.".format(GEN_COMMAND_FUNC_NAME))
+
+    except Exception as ex:
+        sys.stderr.write("""\
+Error:
+Could not load/use the test case generation module from '{fpath}'.
+{error}
+""".format(
+                fpath=test_gen_python_file,
+                error=ex,
+            )
+        )
+        exit(2)
+
+    def get_gen_command_line():
+        command_line_str = gen_command_func()
+        if not isinstance(command_line_str, str):
+            raise Exception("The return value of {}() is not a string.".format(GEN_COMMAND_FUNC_NAME))
+        try:
+            command_line_list = shlex.split(command_line_str)
+            if not command_line_list:
+                raise Exception("The string must have at least one element.".format(GEN_COMMAND_FUNC_NAME))
+        except Exception as ex:
+            raise Exception("Error in parsing the return value of {}() '{}': {}".format(GEN_COMMAND_FUNC_NAME, command_line_str, ex))
+        return (command_line_str, command_line_list,)
+
+
+    if GEN_STR_RAND_SEED is not None:
+        random.seed(GEN_STR_RAND_SEED)
+
+    if verification_mode:
+        # Calling gen_command_func once to check for the correctness of the program.
+        try:
+            get_gen_command_line()
+        except Exception as ex:
+            sys.stderr.write("""\
+Error:
+Could not successfully call '{func}' from '{fpath}'.
+{error}
+""".format(
+                    func=GEN_COMMAND_FUNC_NAME,
+                    fpath=test_gen_python_file,
+                    error=ex,
+                )
+            )
+            exit(2)
+
+        exit(0)
+
+    round_index = 0
+    while True:
+        round_index += 1
+        if round_index > num_rounds >= 0:
+            break
+
+        cprint(colors.YELLOW, "Round {}:".format(round_index))
+        try:
+            (test_gen_cmd_str, test_gen_cmd_list,) = get_gen_command_line()
+        except Exception as ex:
+            cprint(
+                colors.ERROR,
+                """\
+Error: Could not successfully create the test generation command line.
+{error}""".format(error=ex),
+            )
+            continue
+
+        print(test_gen_cmd_str)
+
+        command = [
+            'bash',
+            os.path.join(INTERNALS_DIR, 'stress_single_test.sh'),
+            str(round_index),
+        ] + test_gen_cmd_list
+        wait_process_success(subprocess.Popen(command))
+
+
+if __name__ == '__main__':
+    _main()

scripts/internal/stress_single_test.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source "${INTERNALS}/util.sh"
+source "${INTERNALS}/problem_util.sh"
+source "${INTERNALS}/gen_util.sh"
+source "${INTERNALS}/invoke_util.sh"
+
+round_index="$1"; shift
+gen_command_line=("$@")
+
+test_name="testcase"
+input_file_name="${test_name}.in"
+input_file_path="${SANDBOX_ROOT}/${input_file_name}"
+model_output="${SANDBOX_MODEL}/${test_name}.out"
+stressed_stdout="${SANDBOX_STRESSED}/${test_name}.out"
+stressed_stderr="${SANDBOX_STRESSED}/${test_name}.err"
+
+# Remove any remaining files from previous runs
+rm -f "${input_file_path}" "${model_output}" "${stressed_stdout}" "${stressed_stderr}"
+
+initialize_failed_job_list
+
+
+export BOX_PADDING=6
+
+echo -n "gen"
+gen_job="${test_name}.gen"
+echo "${gen_command_line[@]}" > "${LOGS_DIR}/${gen_job}.args"
+insensitive guard "${gen_job}" gen_input "${input_file_path}" "${gen_command_line[@]}"
+verify_job_failure "${gen_job}"
+gen_status="$(job_status "${gen_job}")"
+echo_status "${gen_status}"
+
+
+function validate {
+	if [ ! -f "${input_file_path}" ]; then
+		errcho "input file '${input_file_name}' is not available"
+		return 4
+	fi
+	local validator_commands
+	validator_commands="$(get_global_validator_commands)" || return $?
+	run_validator_commands_on_input "${input_file_path}" <<< "${validator_commands}" || return $?
+}
+
+echo -n "val"
+val_job="${test_name}.val"
+if ! "${SKIP_VAL}" && ! is_in "${gen_status}" "FAIL"; then
+	insensitive guard "${val_job}" validate
+fi
+verify_job_failure "${val_job}"
+val_status="$(job_status "${val_job}")"
+echo_status "${val_status}"
+
+
+function create_model_output {
+	if [ ! -f "${input_file_path}" ]; then
+		errcho "input file '${input_file_name}' is not available"
+		return 4
+	fi
+	local -r temp_output="${model_output}.tmp"
+	export SANDBOX="${SANDBOX_MODEL}"
+	local ret=0
+	bash "${SCRIPTS}/run.sh" < "${input_file_path}" > "${temp_output}" || ret=$?
+	unset SANDBOX
+	[ "${ret}" -eq "0" ] ||
+		return "${ret}"
+	mv "${temp_output}" "${model_output}"
+}
+
+echo -n "model"
+model_job="${test_name}.model"
+if ! "${SKIP_MODEL}" && ! is_in "${gen_status}" "FAIL"; then
+	insensitive guard "${model_job}" create_model_output
+fi
+verify_job_failure "${model_job}"
+model_status="$(job_status "${model_job}")"
+echo_status "${model_status}"
+
+
+
+export BOX_PADDING=4
+
+unset execution_time score verdict reason
+
+echo -n "stressed"
+stressed_job="${test_name}.stressed"
+if is_in "${gen_status}" "FAIL"; then
+	execution_time=""
+	score="?"
+	verdict="${VERDICT__UNKNOWN}"
+	reason="Failure in generating input"
+else
+	export SANDBOX="${SANDBOX_STRESSED}"
+	invoke_solution "${stressed_job}" "${test_name}" "${input_file_path}" "${stressed_stdout}" "${stressed_stderr}"
+	unset SANDBOX
+fi
+if variable_exists "verdict" && is_verdict_judge_failure "${verdict}"; then
+	verify_job_failure "${stressed_job}"
+fi
+stressed_status="$(job_status "${stressed_job}")"
+echo_status "${stressed_status}"
+printf "%7s" "${execution_time}"
+hspace 4
+
+
+export BOX_PADDING=5
+
+echo -n "check"
+check_job="${test_name}.check"
+if ! is_in "${stressed_status}" "FAIL" "SKIP"; then
+	export SANDBOX="${SANDBOX_STRESSED}"
+	run_checker_if_needed "${check_job}" "${test_name}" "${input_file_path}" "${model_output}" "${stressed_stdout}" "${stressed_stderr}"
+	unset SANDBOX
+fi
+verify_job_failure "${check_job}"
+check_status="$(job_status "${check_job}")"
+echo_status "${check_status}"
+
+print_score "${score}" "6"
+hspace 2
+
+export BOX_PADDING=20
+echo_verdict "${verdict}"
+
+echo "${score}" > "${LOGS_DIR}/${test_name}.score"
+echo "${verdict}" > "${LOGS_DIR}/${test_name}.verdict"
+echo "${reason}" > "${LOGS_DIR}/${test_name}.reason"
+
+echo
+
+if should_stop_for_failed_jobs; then
+	echo
+	echo "Failure on gen command line:"
+	echo "${gen_command_line[@]}"
+	stop_for_failed_jobs
+fi
+
+if is_signed_decimal_format "${score}" && py_test "${score} < ${MIN_SCORE}"; then
+	cecho "error" "Hacked!"
+	echo "${gen_command_line[@]}" >> "${LOGS_DIR}/hacked.txt"
+	if "${HACK_SENSITIVE}"; then
+		echo
+		cecho "yellow" "Gen command line:"
+		echo "${gen_command_line[@]}"
+		exit 100
+	fi
+fi