diff --git a/tools/ci/commands.json b/tools/ci/commands.json index 590d243abfb89b..79c51560c54dc8 100644 --- a/tools/ci/commands.json +++ b/tools/ci/commands.json @@ -80,6 +80,14 @@ "requirements_tc.txt" ] }, + "tc-run": { + "path": "run_tc.py", + "script": "run_tc", + "parser": "get_parser", + "parse_known": true, + "help": "Run command in Taskcluster CI environment (for use in Docker image)", + "virtualenv": false + }, "update-codeowners": { "path": "update_codeowners.py", "parser": "get_parser", diff --git a/tools/ci/run_tc.py b/tools/ci/run_tc.py index a729bc8878cf4f..aa95618615d589 100755 --- a/tools/ci/run_tc.py +++ b/tools/ci/run_tc.py @@ -55,6 +55,9 @@ os.pardir)) +started_processes = [] + + def run(cmd, return_stdout=False, **kwargs): print(" ".join(cmd)) if return_stdout: @@ -68,7 +71,7 @@ def run(cmd, return_stdout=False, **kwargs): def start(cmd): print(" ".join(cmd)) - subprocess.Popen(cmd) + started_processes.append(subprocess.Popen(cmd)) def get_parser(): @@ -99,6 +102,8 @@ def get_parser(): help="Install web-platform.test certificates to UA store") p.add_argument("--no-install-certificates", action="store_false", default=None, help="Don't install web-platform.test certificates to UA store") + p.add_argument("--setup-repository", action="store_true", default=None, dest="setup_repository", + help="Run any repository setup steps, instead use the existing worktree") p.add_argument("--no-setup-repository", action="store_false", dest="setup_repository", help="Don't run any repository setup steps, instead use the existing worktree. " "This is useful for local testing.") @@ -256,33 +261,33 @@ def unpack(path): raise Exception -def setup_environment(args): +def setup_environment(**kwargs): if "TASK_ARTIFACTS" in os.environ: artifacts = json.loads(os.environ["TASK_ARTIFACTS"]) download_artifacts(artifacts) - if args.hosts_file: + if kwargs["hosts_file"]: make_hosts_file() - if args.install_certificates: + if kwargs["install_certificates"]: install_certificates() - if "chrome" in args.browser: - assert args.channel is not None - install_chrome(args.channel) + if "chrome" in kwargs["browser"]: + assert kwargs["channel"] is not None + install_chrome(kwargs["channel"]) # These browsers use dbus for various features. - if any(b in args.browser for b in ["chrome", "webkitgtk_minibrowser", "wpewebkit_minibrowser"]): + if any(b in kwargs["browser"] for b in ["chrome", "webkitgtk_minibrowser", "wpewebkit_minibrowser"]): start_dbus() - if args.xvfb: + if kwargs["xvfb"]: start_xvfb() - if args.oom_killer: + if kwargs["oom_killer"]: start_userspace_oom_killer() -def setup_repository(args): +def setup_repository(**kwargs): is_pr = os.environ.get("GITHUB_PULL_REQUEST", "false") != "false" # Initially task_head points at the same commit as the ref we want to test. @@ -303,12 +308,12 @@ def setup_repository(args): # resources. In the latter case we assume it's OK to use the current merge # instead of the one at the time the decision task ran. - if args.ref: + if kwargs["ref"]: if is_pr: - assert args.ref.endswith("/merge") - expected_head = args.merge_rev + assert kwargs["ref"].endswith("/merge") + expected_head = kwargs["merge_rev"] else: - expected_head = args.head_rev + expected_head = kwargs["head_rev"] task_head = run(["git", "rev-parse", "task_head"], return_stdout=True).strip() @@ -324,14 +329,14 @@ def setup_repository(args): sys.exit(1) else: # Convert the refs/pulls//merge to refs/pulls//head - head_ref = args.ref.rsplit("/", 1)[0] + "/head" + head_ref = kwargs["ref"].rsplit("/", 1)[0] + "/head" try: remote_head = run(["git", "ls-remote", "origin", head_ref], return_stdout=True).split("\t")[0] except subprocess.CalledProcessError: print("CRITICAL: Failed to read remote ref %s" % head_ref) sys.exit(1) - if remote_head != args.head_rev: + if remote_head != kwargs["head_rev"]: print("CRITICAL: task_head points at %s, expected %s. " "This may be because the branch was updated" % (task_head, expected_head)) sys.exit(1) @@ -362,7 +367,7 @@ def setup_repository(args): # TODO: move this somewhere earlier in the task run(["git", "fetch", "--quiet", "origin", "%s:%s" % (branch, branch)]) - checkout_rev = args.checkout if args.checkout is not None else "task_head" + checkout_rev = kwargs["checkout"] if kwargs["checkout"] is not None else "task_head" checkout_revision(checkout_rev) refs = run(["git", "for-each-ref", "refs/heads"], return_stdout=True) @@ -405,36 +410,47 @@ def include_job(job): return job in set(jobs_str.splitlines()) +def run_tc(*args, **kwargs): + try: + is_ci = "TASKCLUSTER_ROOT_URL" in os.environ + + if "TASK_EVENT" in os.environ: + event = json.loads(os.environ["TASK_EVENT"]) + elif is_ci: + event = fetch_event_data() + else: + event = None + + if event: + set_variables(event) + + if kwargs["setup_repository"] or (kwargs["setup_repository"] is None and is_ci): + setup_repository(**kwargs) + + # Hack for backwards compatibility + if kwargs["script"] in ["run-all", "lint", "update_built", "tools_unittest", + "wpt_integration", "resources_unittest", + "wptrunner_infrastructure", "stability", "affected_tests"]: + job = kwargs["script"] + if not include_job(job): + return + kwargs["script"] = kwargs["script_args"][0] + kwargs["script_args"] = kwargs["script_args"][1:] + + # Run the job + setup_environment(**kwargs) + os.chdir(root) + cmd = [kwargs["script"]] + kwargs["script_args"] + print(" ".join(cmd)) + sys.exit(subprocess.call(cmd)) + finally: + for process in started_processes: + process.kill() + + def main(): args = get_parser().parse_args() - - if "TASK_EVENT" in os.environ: - event = json.loads(os.environ["TASK_EVENT"]) - else: - event = fetch_event_data() - - if event: - set_variables(event) - - if args.setup_repository: - setup_repository(args) - - # Hack for backwards compatibility - if args.script in ["run-all", "lint", "update_built", "tools_unittest", - "wpt_integration", "resources_unittest", - "wptrunner_infrastructure", "stability", "affected_tests"]: - job = args.script - if not include_job(job): - return - args.script = args.script_args[0] - args.script_args = args.script_args[1:] - - # Run the job - setup_environment(args) - os.chdir(root) - cmd = [args.script] + args.script_args - print(" ".join(cmd)) - sys.exit(subprocess.call(cmd)) + run_tc(**vars(args)) if __name__ == "__main__": diff --git a/tools/ci/tc/decision.py b/tools/ci/tc/decision.py index d00ba6ba194ee1..d98f54811295a9 100644 --- a/tools/ci/tc/decision.py +++ b/tools/ci/tc/decision.py @@ -227,7 +227,7 @@ def build_full_command(event, task): %(fetch_ref)s; %(install_str)s cd web-platform-tests; -./tools/ci/run_tc.py %(options_str)s -- %(task_cmd)s; +./wpt tc-run %(options_str)s -- %(task_cmd)s; """ % cmd_args]