Disable firefox browserstack tests temporarily (#7935)

* Disable firefox browserstack tests temporarily * Remove unused WebGL scripts * Update test-ci.sh * Always use port 9876 for karma * Always use port 9200 for karma Browserstack supports ports 9200-9400 for safari testing (https://www.browserstack.com/question/39572), and when karma launches and its port is in use, it automatically increments the port. Starting at 9200 ensures the port is always supported by BrowserStack. * Fix bash npm-run-all script * Fix tfjs-vis test script * Update karma.conf.js * Update automl macos test version --------- Co-authored-by: Ping Yu <[email protected]>
tensorflow · Aug 26, 2023 · d500a0d · d500a0d
1 parent 20ceb6f
commit d500a0d
Show file tree

Hide file tree

Showing 27 changed files with 51 additions and 95 deletions.
diff --git a/e2e/benchmarks/browserstack-benchmark/karma.conf.js b/e2e/benchmarks/browserstack-benchmark/karma.conf.js
@@ -26,7 +26,7 @@ function getBrowserStackConfig() {
     hostname: 'bs-local.com',
     plugins: ['karma-jasmine', 'karma-browserstack-launcher'],
     reporters: ['progress', 'BrowserStack'],
-    port: 9812,
+    port: 9200,
     browserStack: {
       username: process.env.BROWSERSTACK_USERNAME,
       accessKey: process.env.BROWSERSTACK_ACCESS_KEY,

diff --git a/e2e/karma.conf.js b/e2e/karma.conf.js
@@ -108,7 +108,7 @@ const browserstackConfig = {
   // breaks WASM file serving.
   // See https://www.browserstack.com/question/39574
   singleRun: true,
-  port: 9876
+  port: 9200
 };
 
 const chromeWebgpuFlags = [

diff --git a/e2e/script_tag_tests/tfjs-core-cpu/karma.conf.js b/e2e/script_tag_tests/tfjs-core-cpu/karma.conf.js
@@ -43,7 +43,7 @@ module.exports = function(config) {
   };
 
   const browserstackConfig =
-      {...devConfig, hostname: 'bs-local.com', singleRun: true, port: 9811};
+      {...devConfig, hostname: 'bs-local.com', singleRun: true, port: 9200};
 
   if (config.grep) {
     args.push('--grep', config.grep);

diff --git a/e2e/script_tag_tests/tfjs/karma.conf.js b/e2e/script_tag_tests/tfjs/karma.conf.js
@@ -35,7 +35,7 @@ module.exports = function(config) {
   };
 
   const browserstackConfig =
-      {...devConfig, hostname: 'bs-local.com', singleRun: true, port: 9811};
+      {...devConfig, hostname: 'bs-local.com', singleRun: true, port: 9200};
 
   if (config.grep) {
     args.push('--grep', config.grep);

diff --git a/e2e/scripts/run-browserstack-tests.sh b/e2e/scripts/run-browserstack-tests.sh
@@ -39,7 +39,8 @@ if [[ "$NIGHTLY" = true || "$RELEASE" = true ]]; then
   COMMANDS+=(
     "yarn run-browserstack --browsers=bs_ios_12 --tags '$TAGS' --testEnv webgl --flags '{\"\\"\"WEBGL_VERSION\"\\"\": 1, \"\\"\"WEBGL_CPU_FORWARD\"\\"\": false, \"\\"\"WEBGL_SIZE_UPLOAD_UNIFORM\"\\"\": 0}'"
     "yarn run-browserstack --browsers=bs_safari_mac --tags '$TAGS' --testEnv webgl --flags '{\"\\"\"WEBGL_VERSION\"\\"\": 1, \"\\"\"WEBGL_CPU_FORWARD\"\\"\": false, \"\\"\"WEBGL_SIZE_UPLOAD_UNIFORM\"\\"\": 0}'"
-    "yarn run-browserstack --browsers=bs_firefox_mac --tags '$TAGS'"
+    # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+    # "yarn run-browserstack --browsers=bs_firefox_mac --tags '$TAGS'"
     "yarn run-browserstack --browsers=bs_android_10 --tags '$TAGS'"
     # Test script tag bundles
     "karma start ./script_tag_tests/tfjs-core-cpu/karma.conf.js --browserstack --browsers=bs_chrome_mac"

diff --git a/tfjs-automl/karma.conf.js b/tfjs-automl/karma.conf.js
@@ -71,7 +71,7 @@ module.exports = function(config) {
     preprocessors: {'**/*.ts': ['karma-typescript']},
     karmaTypescriptConfig,
     reporters: ['progress', 'karma-typescript'],
-    port: 9866,
+    port: 9200,
     colors: true,
     browsers: ['Chrome'],
     client: {jasmine: {random: false}, args: args},
@@ -82,7 +82,7 @@ module.exports = function(config) {
       tunnelIdentifier:
           `tfjs_automl_${Date.now()}_${Math.floor(Math.random() * 1000)}`
     },
-    captureTimeout: 3e5,
+    captureTimeout: 10 * 1000,
     reportSlowerThan: 500,
     browserNoActivityTimeout: 3e5,
     browserDisconnectTimeout: 3e5,
@@ -94,21 +94,21 @@ module.exports = function(config) {
         browser: 'chrome',
         browser_version: 'latest',
         os: 'OS X',
-        os_version: 'High Sierra'
+        os_version: 'Ventura'
       },
       bs_firefox_mac: {
         base: 'BrowserStack',
         browser: 'firefox',
         browser_version: 'latest',
         os: 'OS X',
-        os_version: 'High Sierra'
+        os_version: 'Ventura'
       },
       bs_safari_mac: {
         base: 'BrowserStack',
         browser: 'safari',
         browser_version: 'latest',
         os: 'OS X',
-        os_version: 'Big Sur'
+        os_version: 'Ventura'
       },
       bs_ios_12: {
         base: 'BrowserStack',

diff --git a/tfjs-automl/package.json b/tfjs-automl/package.json
@@ -15,7 +15,7 @@
     "test-node": "tsc && ts-node --transpile-only --skip-ignore --project tsconfig.test.json src/test_node.ts",
     "coverage": "KARMA_COVERAGE=1 karma start --singleRun",
     "run-flaky": "node ../scripts/run_flaky.js",
-    "run-browserstack": "karma start --singleRun --reporters='dots,karma-typescript,BrowserStack' --hostname='bs-local.com'",
+    "run-browserstack": "karma start --singleRun --reporters='dots,karma-typescript,BrowserStack'",
     "test-ci": "./scripts/test-ci.sh",
     "build-npm": "./scripts/build-npm.sh"
   },

diff --git a/tfjs-automl/scripts/test-ci.sh b/tfjs-automl/scripts/test-ci.sh
@@ -27,5 +27,7 @@ yarn run-flaky "yarn run-browserstack --browsers=bs_chrome_mac"
 # Run the rest of the karma tests in parallel. These runs will reuse the
 # already downloaded binary.
 npm-run-all -p -c --aggregate-output \
-  "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
-  "run-flaky \"yarn run-browserstack --browsers=bs_safari_mac  --testEnv webgl1\""
+  "run-flaky \"yarn run-browserstack --browsers=bs_safari_mac --testEnv webgl1\""
+# TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+# "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
+
diff --git a/tfjs-backend-wasm/BUILD.bazel b/tfjs-backend-wasm/BUILD.bazel
@@ -151,7 +151,8 @@ tfjs_web_test(
     ],
     browsers = [
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         "bs_safari_mac",
         "bs_ios_12",
         # TODO(mattsoulanille): Fix clipByValue on Android.

diff --git a/tfjs-backend-webgl/BUILD.bazel b/tfjs-backend-webgl/BUILD.bazel
@@ -107,7 +107,8 @@ tfjs_web_test(
     ],
     browsers = [
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         "bs_android_10",
         "win_10_chrome",
     ],

diff --git a/tfjs-backend-webgl/scripts/build-npm.sh b/tfjs-backend-webgl/scripts/build-npm.sh
diff --git a/tfjs-backend-webgl/scripts/test-ci.sh b/tfjs-backend-webgl/scripts/test-ci.sh
diff --git a/tfjs-backend-webgl/scripts/tsconfig.json b/tfjs-backend-webgl/scripts/tsconfig.json
diff --git a/tfjs-backend-webgpu/karma.conf.js b/tfjs-backend-webgpu/karma.conf.js
@@ -67,7 +67,7 @@ module.exports = function(config) {
       require('karma-jasmine-html-reporter'),
     ],
     exclude,
-    port: 9876,
+    port: 9200,
     colors: true,
     autoWatch: false,
     browsers: ['Chrome', 'chrome_webgpu'],

diff --git a/tfjs-core/BUILD.bazel b/tfjs-core/BUILD.bazel
@@ -111,7 +111,8 @@ tfjs_web_test(
     ],
     browsers = [
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         "bs_safari_mac",
         "bs_ios_12",
         "bs_android_10",

diff --git a/tfjs-core/src/BUILD.bazel b/tfjs-core/src/BUILD.bazel
@@ -222,7 +222,8 @@ tfjs_web_test(
     name = "worker_test",
     browsers = [
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         "bs_safari_mac",
         # Temporarily disabled because BrowserStack does not support loading
         # absolute paths in iOS, which is required for loading the worker.

diff --git a/tfjs-data/karma.conf.js b/tfjs-data/karma.conf.js
@@ -27,7 +27,7 @@ module.exports = function(config) {
         {tsconfig: 'tsconfig.json', compilerOptions: {module: 'commonjs'}},
     reporters: ['progress', 'karma-typescript'],
     browsers: ['Chrome'],
-    port: 9876,
+    port: 9200,
     browserStack: {
       username: process.env.BROWSERSTACK_USERNAME,
       accessKey: process.env.BROWSERSTACK_KEY,

diff --git a/tfjs-layers/BUILD.bazel b/tfjs-layers/BUILD.bazel
@@ -51,7 +51,8 @@ tfjs_web_test(
     ],
     browsers = [
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         # disabled android test due to training flakiness
         # "bs_android_10",
         "win_10_chrome",

diff --git a/tfjs-layers/karma.conf.js b/tfjs-layers/karma.conf.js
@@ -56,7 +56,7 @@ module.exports = function(config) {
     karmaTypescriptConfig,
     reporters: ['progress', 'karma-typescript'],
     browsers: ['Chrome'],
-    port: 9846,
+    port: 9200,
     browserStack: {
       username: process.env.BROWSERSTACK_USERNAME,
       accessKey: process.env.BROWSERSTACK_KEY,

diff --git a/tfjs-tfdf/BUILD.bazel b/tfjs-tfdf/BUILD.bazel
@@ -107,7 +107,8 @@ tfjs_web_test(
     browsers = [
         # TODO: Support Safari.
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         "bs_android_10",
         "win_10_chrome",
     ],

diff --git a/tfjs-tflite/src/BUILD.bazel b/tfjs-tflite/src/BUILD.bazel
@@ -104,7 +104,8 @@ tfjs_web_test(
     name = "worker_test",
     browsers = [
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         "bs_safari_mac",
         # Temporarily disabled because BrowserStack does not support loading
         # absolute paths in iOS, which is required for loading the worker.

diff --git a/tfjs-vis/karma.conf.js b/tfjs-vis/karma.conf.js
@@ -68,7 +68,7 @@ module.exports = function(config) {
     karmaTypescriptConfig,
     reporters: ['progress', 'karma-typescript'],
     browsers: ['Chrome'],
-    port: 9836,
+    port: 9200,
     browserStack: {
       username: process.env.BROWSERSTACK_USERNAME,
       accessKey: process.env.BROWSERSTACK_KEY,

diff --git a/tfjs-vis/scripts/test-ci.sh b/tfjs-vis/scripts/test-ci.sh
@@ -26,6 +26,9 @@ yarn run-flaky "yarn run-browserstack --browsers=bs_chrome_mac"
 
 # Run the rest of the karma tests in parallel. These runs will reuse the
 # already downloaded binary.
-npm-run-all -p -c --aggregate-output \
-  "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
-  "run-flaky \"yarn run-browserstack --browsers=bs_safari_mac\""
+yarn run-flaky "yarn run-browserstack --browsers=bs_safari_mac"
+#npm-run-all -p -c --aggregate-output \
+  #"run-flaky \"yarn run-browserstack --browsers=bs_safari_mac\""
+  # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+  # "run-flaky \"yarn run-browserstack --browsers=bs_firefox_mac\"" \
+
diff --git a/tfjs/karma.conf.js b/tfjs/karma.conf.js
@@ -54,7 +54,7 @@ module.exports = function(config) {
     karmaTypescriptConfig,
     reporters: ['progress', 'karma-typescript'],
     browsers: ['Chrome'],
-    port: 9806,
+    port: 9200,
     browserStack: {
       username: process.env.BROWSERSTACK_USERNAME,
       accessKey: process.env.BROWSERSTACK_KEY,

diff --git a/tfjs/scripts/test-ci.sh b/tfjs/scripts/test-ci.sh
@@ -17,7 +17,8 @@
 # Exit immediately if a command exits with a non-zero status.
 set -e
 
-node ../scripts/run_flaky.js "yarn karma start --browsers='bs_firefox_mac' --singleRun"
+# TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+# node ../scripts/run_flaky.js "yarn karma start --browsers='bs_firefox_mac' --singleRun"
 node ../scripts/run_flaky.js "yarn karma start --browsers='bs_chrome_mac' --singleRun"
 yarn test-tools
 

diff --git a/tools/karma_template.conf.js b/tools/karma_template.conf.js
@@ -16,7 +16,12 @@
  */
 
 const browserstackConfig = {
-  port: 9876,
+  // Browserstack only supports a certain range of ports for safari.
+  // Karma will automatically use the next available port if the
+  // chosen one is in use. Starting at 9200 gives us the largest
+  // range of ports (9200 - 9400).
+  // https://www.browserstack.com/question/39572
+  port: 9200,
 };
 
 // Select Chrome or ChromeHeadless based on the value of the --//:headless flag.

diff --git a/tools/tfjs_web_test.bzl b/tools/tfjs_web_test.bzl
@@ -122,7 +122,8 @@ def tfjs_web_test(name, ci = True, args = [], **kwargs):
 
     browsers = kwargs.pop("browsers", [
         "bs_chrome_mac",
-        "bs_firefox_mac",
+        # TODO(mattSoulanille): Re-enable firefox once it works on browserstack.
+        # "bs_firefox_mac",
         "bs_safari_mac",
         "bs_ios_12",
         "bs_android_10",