build: share Saucelabs browsers between karma test targets using background Saucelabs daemon and custom karma launcher (angular#49200)

This upgrades the Saucelabs Bazel step on CI to use the more efficient Saucelabs daemon

PR Close angular#49200

Author: gregmagolan

Diff for: .circleci/config.yml

@@ -308,18 +308,21 @@ jobs:
       # container for this job. This is necessary because we launch a lot of browsers concurrently
       # and therefore the tunnel and Karma need to process a lot of file requests and tests.
       resource_class: xlarge
+    environment:
+      NUMBER_OF_PARALLEL_BROWSERS: 2
     steps:
       - custom_attach_workspace
       - init_environment
       - init_saucelabs_environment
+      - run:
+          name: Start Saucelabs daemon service
+          command: yarn bazel run //tools/saucelabs-daemon/background-service -- ${NUMBER_OF_PARALLEL_BROWSERS}
+          background: true
       - run:
           name: Run Bazel tests on Saucelabs
-          # See /tools/saucelabs/README.md for more info
           command: |
-            yarn bazel run //tools/saucelabs:sauce_service_setup
             TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)')
-            yarn bazel test --config=saucelabs ${TESTS}
-            yarn bazel run //tools/saucelabs:sauce_service_stop
+            yarn bazel test --config=saucelabs --jobs=${NUMBER_OF_PARALLEL_BROWSERS} ${TESTS}
           no_output_timeout: 40m
       - notify_webhook_on_fail:
           webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL
@@ -674,6 +677,9 @@ workflows:
       - build-npm-packages:
           requires:
             - setup
+      - saucelabs:
+          requires:
+            - setup
       - legacy-unit-tests-saucelabs:
           requires:
             - setup

Diff for: .pullapprove.yml

@@ -1166,6 +1166,7 @@ groups:
           'tools/legacy-saucelabs/**/{*,.*}',
           'tools/rxjs/**/{*,.*}',
           'tools/saucelabs/**/{*,.*}',
+          'tools/saucelabs-daemon/**/{*,.*}',
           'tools/symbol-extractor/**/{*,.*}',
           'tools/testing/**/{*,.*}',
           'tools/tslint/**/{*,.*}',

Diff for: BUILD.bazel

@@ -1,4 +1,5 @@
 load("//tools:defaults.bzl", "nodejs_binary")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
 load("//:yarn.bzl", "YARN_PATH")
 
 package(default_visibility = ["//visibility:public"])
@@ -26,6 +27,14 @@ alias(
     actual = "//packages:tsconfig-build.json",
 )
 
+js_library(
+    name = "browser-providers",
+    srcs = [
+        "browser-providers.conf.d.ts",
+        "browser-providers.conf.js",
+    ],
+)
+
 filegroup(
     name = "angularjs_scripts",
     srcs = [
@@ -61,3 +70,12 @@ nodejs_binary(
         "//integration:__subpackages__",
     ],
 )
+
+alias(
+    name = "sauce_connect",
+    actual = select({
+        "@npm//@angular/build-tooling/bazel/constraints:linux_x64": "@sauce_connect_linux_amd64//:bin/sc",
+        "@npm//@angular/build-tooling/bazel/constraints:macos_x64": "@sauce_connect_mac//:bin/sc",
+        "@npm//@angular/build-tooling/bazel/constraints:macos_arm64": "@sauce_connect_mac//:bin/sc",
+    }),
+)

Diff for: WORKSPACE

@@ -215,3 +215,20 @@ register_toolchains(
     "@npm//@angular/build-tooling/bazel/git-toolchain:git_macos_arm64_toolchain",
     "@npm//@angular/build-tooling/bazel/git-toolchain:git_windows_toolchain",
 )
+
+# Fetch sauce connect (tool to open Saucelabs tunnel for Saucelabs browser tests)
+http_archive(
+    name = "sauce_connect_linux_amd64",
+    build_file_content = """exports_files(["bin/sc"], visibility = ["//visibility:public"])""",
+    sha256 = "26b9c3630f441b47854b6032f7eca6f1d88d3f62e50ee44c27015d71a5155c36",
+    strip_prefix = "sc-4.8.2-linux",
+    url = "https://saucelabs.com/downloads/sc-4.8.2-linux.tar.gz",
+)
+
+http_archive(
+    name = "sauce_connect_mac",
+    build_file_content = """exports_files(["bin/sc"], visibility = ["//visibility:public"])""",
+    sha256 = "28277ce81ef9ab84f5b87b526258920a8ead44789a5034346e872629bbf38089",
+    strip_prefix = "sc-4.8.2-osx",
+    url = "https://saucelabs.com/downloads/sc-4.8.2-osx.zip",
+)

Diff for: browser-providers.conf.d.ts

@@ -0,0 +1,20 @@
+type CustomLauncher = {
+  base: string;
+  browserName: string;
+  platformName: string;
+  platformVersion: string;
+  deviceName: string;
+  appiumVersion: string;
+  extendedDebugging: boolean;
+}
+
+type CustomLaunchers = {
+  [string]: CustomLauncher;
+};
+
+type SauceAliases = {
+  [string]: string[];
+};
+
+export const customLaunchers: CustomLaunchers;
+export const sauceAliases: SauceAliases;

Diff for: karma-js.conf.js

@@ -12,6 +12,13 @@ const {hostname} = require('os');
 const seed = process.env.JASMINE_RANDOM_SEED || String(Math.random()).slice(-5);
 console.info(`Jasmine random seed: ${seed}`);
 
+const isBazel = !!process.env.TEST_TARGET;
+
+if (!process.env.KARMA_WEB_TEST_MODE && isBazel && process.env.TEST_TARGET.includes('_saucelabs')) {
+  console.info(`Saucelabs target detected: ${process.env.TEST_TARGET}`);
+  process.env.KARMA_WEB_TEST_MODE = 'SL_REQUIRED';
+}
+
 module.exports = function(config) {
   const conf = {
     frameworks: ['jasmine'],
@@ -50,7 +57,6 @@ module.exports = function(config) {
 
     plugins: [
       'karma-jasmine',
-      'karma-sauce-launcher',
       'karma-chrome-launcher',
       'karma-sourcemap-loader',
     ],
@@ -99,20 +105,29 @@ module.exports = function(config) {
     set: () => {},
   });
 
-  if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
-    console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);
+  if (isBazel) {
+    // Add the custom Saucelabs daemon to the plugins
+    const saucelabsDaemonLauncher = require('./tools/saucelabs-daemon/launcher/index.cjs').default;
+    conf.plugins.push(saucelabsDaemonLauncher);
+  } else {
+    conf.plugins.push('karma-sauce-launcher');
 
-    const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER'];
+    if (process.env['SAUCE_TUNNEL_IDENTIFIER']) {
+      console.log(`SAUCE_TUNNEL_IDENTIFIER: ${process.env.SAUCE_TUNNEL_IDENTIFIER}`);
 
-    // Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel.
-    conf.sauceLabs.build = tunnelIdentifier;
-    conf.sauceLabs.tunnelIdentifier = tunnelIdentifier;
+      const tunnelIdentifier = process.env['SAUCE_TUNNEL_IDENTIFIER'];
 
-    // Patch the `saucelabs` package so that `karma-sauce-launcher` does not attempt downloading
-    // the test logs from upstream and tries re-uploading them with the Karma enhanced details.
-    // This slows-down tests/browser restarting and can decrease stability.
-    // https://github.com/karma-runner/karma-sauce-launcher/blob/59b0c5c877448e064ad56449cd906743721c6b62/src/launcher/launcher.ts#L72-L79.
-    require('saucelabs').default.prototype.downloadJobAsset = () => Promise.resolve('<FAKE-LOGS>');
+      // Setup the Saucelabs plugin so that it can launch browsers using the proper tunnel.
+      conf.sauceLabs.build = tunnelIdentifier;
+      conf.sauceLabs.tunnelIdentifier = tunnelIdentifier;
+
+      // Patch the `saucelabs` package so that `karma-sauce-launcher` does not attempt downloading
+      // the test logs from upstream and tries re-uploading them with the Karma enhanced details.
+      // This slows-down tests/browser restarting and can decrease stability.
+      // https://github.com/karma-runner/karma-sauce-launcher/blob/59b0c5c877448e064ad56449cd906743721c6b62/src/launcher/launcher.ts#L72-L79.
+      require('saucelabs').default.prototype.downloadJobAsset = () =>
+          Promise.resolve('<FAKE-LOGS>');
+    }
   }
 
   // For SauceLabs jobs, we set up a domain which resolves to the machine which launched

Diff for: package.json

@@ -88,6 +88,7 @@
     "@types/jasminewd2": "^2.0.8",
     "@types/node": "^16.11.7",
     "@types/selenium-webdriver": "3.0.7",
+    "@types/selenium-webdriver4": "npm:@types/[email protected]",
     "@types/semver": "^7.3.4",
     "@types/shelljs": "^0.8.6",
     "@types/systemjs": "0.19.32",
@@ -135,6 +136,7 @@
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rxjs": "^6.6.7",
     "selenium-webdriver": "3.5.0",
+    "selenium-webdriver4": "npm:[email protected]",
     "semver-dsl": "^1.0.1",
     "shelljs": "^0.8.5",
     "source-map": "0.7.4",

Diff for: packages/zone.js/test/karma_test.bzl

@@ -100,6 +100,7 @@ def karma_test(name, env_srcs, env_deps, env_entry_point, test_srcs, test_deps,
                 configuration_env_vars = ["KARMA_WEB_TEST_MODE"],
                 data = [
                     "//:browser-providers.conf.js",
+                    "//tools/saucelabs-daemon/launcher:launcher_cjs",
                 ],
                 static_files = [
                     ":assets/sample.json",

Diff for: tools/defaults.bzl

@@ -321,23 +321,21 @@ def karma_web_test_suite(
     # Add a saucelabs target for Karma tests in `//packages/`.
     if native.package_name().startswith("packages/"):
         _karma_web_test(
-            name = "saucelabs_%s" % name,
+            name = "{}_saucelabs".format(name),
             # Default timeout is moderate (5min). This causes the test to be terminated while
             # Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
             # unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
             # Bazel's long timeout (15min). This ensures that Karma can shut down properly.
             timeout = "long",
             config_file = "//:karma-js.conf.js",
             deps = [
-                "@npm//karma-sauce-launcher",
                 ":%s_bundle" % name,
             ],
             data = data + [
                 "//:browser-providers.conf.js",
+                "//tools/saucelabs-daemon/launcher:launcher_cjs",
             ],
-            karma = "//tools/saucelabs:karma-saucelabs",
             tags = tags + [
-                "exclusive",
                 "manual",
                 "no-remote-exec",
                 "saucelabs",

Diff for: tools/saucelabs-daemon/BUILD.bazel

@@ -0,0 +1,12 @@
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "saucelabs-daemon",
+    srcs = [
+        "browser.ts",
+        "ipc-defaults.ts",
+        "ipc-messages.ts",
+    ],
+)

Diff for: tools/saucelabs-daemon/README.md

@@ -0,0 +1,88 @@
+# Saucelabs testing with Bazel
+
+## Local testing
+
+1. Set up your `SAUCE_USERNAME`, `SAUCE_ACCESS_KEY` & `SAUCE_TUNNEL_IDENTIFIER` environment variables.
+These are required. You can find the values for `SAUCE_USERNAME` and `SAUCE_ACCESS_KEY` in `/.circleci/env.sh`. `SAUCE_TUNNEL_IDENTIFIER` can be set to any unique value.
+
+If you are having trouble running Saucelabs tests locally you can contact [Joey Perrott](https://github.com/josephperrott) or [Paul Gschwendtner](https://github.com/devversion) for support.
+
+2. Once you have your environment variables set up, run the setup task in the root of the repo:
+
+``` bash
+yarn bazel run //tools/saucelabs-daemon/background-service -- <number_of_browsers>
+```
+
+This will start a daemon process that will connect to Saucelabs and provision browsers
+once you start running your first test target.
+
+3. In another terminal, you can run a particular test target through SauceLabs by suffixing the target name with "_saucelabs".
+
+For example, `packages/core/test:test_web` becomes `packages/core/test:test_web_saucelabs`.
+
+```
+yarn bazel test //packages/core/test:test_web_saucelabs
+```
+
+## Additional test features
+
+To see the test output while the tests are running (as these are long tests), add the `--test_output=streamed` option. 
+Note, this option will also prevent bazel from using the test cache and will force the test to run.
+
+For running all Saucelabs tests in the project, `bazel query` is used to gather up all karma Saucelabs test labels because they are otherwise hidden by the `manual` tag.
+
+Running all karma tests in Saucelabs:
+
+Start the saucelabs-daemon background service in one terminal window:
+
+``` bash
+yarn bazel run //tools/saucelabs-daemon/background-service -- <number_of_browsers>
+```
+
+In a second terminal window, run all of the saucelabs test targets:
+
+``` bash
+TESTS=$(./node_modules/.bin/bazelisk query --output label '(kind(karma_web_test, ...) intersect attr("tags", "saucelabs", ...)) except attr("tags", "fixme-saucelabs", ...)')
+yarn bazel test --config=saucelabs --jobs=<number_of_browsers> ${TESTS}
+```
+
+NB: The number of parallel Bazel tests specified by `--jobs=<number_of_browsers>` must not exceed the number parallel browsers requested when starting the daemon.
+
+## Under the hood
+
+The `//tools/saucelabs-daemon/background-service` target does not start the Sauce Connect proxy at start-up, but instead listens for the start signal from the saucelabs karma launcher. 
+This signal is sent by saucelabs-daemon custom karma launcher `tools/saucelabs-daemon/launcher/launcher.ts`. 
+This is necessary as the Sauce Connect Proxy process must be started outside of `bazel test` as Bazel will automatically kill any processes spawned during a test when that tests completes, which would prevent the tunnel & provisioned browsers from being shared by multiple tests.
+
+The karma_web_test rule for saucelabs must have a few important tags:
+
+*  `no-remote-exec` as they cannot be executed remotely since tests need to communicate with the daemon.
+*  `manual` so they are not automatically tested with `//...`
+*  `saucelabs` so that they can be easily gathered up for testing in a `bazel query`
+
+These are added automatically the by `karma_web_test_suite` macro in `tools/defaults.bzl`.
+
+## Debugging
+
+**Q: How do I get the tests to run on IE? I only see Chromium.**
+
+If you see something like this at the end of your test output, it means you're not actually running SauceLabs: 
+
+```
+INFO: Build completed successfully, 43 total actions
+/packages/core/test:test_web_chromium
+``` 
+
+This is a common error caused by forgetting to suffix your test target with "_saucelabs".
+
+For example, `/packages/core/test:test_web` becomes `/packages/core/test:test_web_saucelabs`.
+
+**Q: How can I tell that the SauceLabs connection was successfully made?**
+
+There is a dashboard at saucelabs.com where you can see active tunnel connections (Angular has an account).
+As soon as you actually run the test target (not after the setup task), you should see an active tunnel connection under the SAUCE_TUNNEL_IDENTIFICATION_KEY you entered. 
+If a tunnel connection is not there, you are not actually connecting with SauceLabs.
+
+Note: It may *look* like the tests are running because of the Bazel output. 
+The progress Bazel is showing does not mean that SauceLabs is connected.
+If the tests are actually running, you should see the "..." test report for passing tests.

Diff for: tools/saucelabs-daemon/background-service/BUILD.bazel

@@ -0,0 +1,28 @@
+load("//tools:defaults.bzl", "nodejs_binary", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "background-service_lib",
+    srcs = glob(["*.ts"]),
+    deps = [
+        "//:browser-providers",
+        "//tools/saucelabs-daemon",
+        "@npm//@types/node",
+        "@npm//@types/selenium-webdriver4",
+        "@npm//chalk",
+        "@npm//selenium-webdriver4",
+    ],
+)
+
+nodejs_binary(
+    name = "background-service",
+    data = [
+        ":background-service_lib",
+        "//:sauce_connect",
+    ],
+    entry_point = ":cli.ts",
+    templated_args = [
+        "$(rootpath //:sauce_connect)",
+    ],
+)