Run benchmarks in test mode when not specifying verification functions in CI

flow-typed/npm/tinybench_v3.1.x.js

@@ -0,0 +1,121 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow strict
+ * @format
+ * @oncall react_native
+ */
+
+declare module 'tinybench' {
+  declare export class Task extends EventTarget {
+    name: string;
+    result: void | $ReadOnly<TaskResult>;
+    runs: number;
+
+    reset(): void;
+    run(): Promise<Task>;
+    runSync(): Task;
+    warmup(): Promise<void>;
+  }
+
+  export type Hook = (
+    task: Task,
+    mode: 'run' | 'warmup',
+  ) => Promise<void> | void;
+
+  export type BenchOptions = {
+    iterations?: number,
+    name?: string,
+    now?: () => number,
+    setup?: Hook,
+    signal?: AbortSignal,
+    teardown?: Hook,
+    throws?: boolean,
+    time?: number,
+    warmup?: boolean,
+    warmupIterations?: number,
+    warmupTime?: number,
+  };
+
+  export interface Statistics {
+    aad: void | number;
+    critical: number;
+    df: number;
+    mad: void | number;
+    max: number;
+    mean: number;
+    min: number;
+    moe: number;
+    p50: void | number;
+    p75: void | number;
+    p99: void | number;
+    p995: void | number;
+    p999: void | number;
+    rme: number;
+    samples: number[];
+    sd: number;
+    sem: number;
+    variance: number;
+  }
+
+  export interface TaskResult {
+    critical: number;
+    df: number;
+    error?: Error;
+    hz: number;
+    latency: Statistics;
+    max: number;
+    mean: number;
+    min: number;
+    moe: number;
+    p75: number;
+    p99: number;
+    p995: number;
+    p999: number;
+    period: number;
+    rme: number;
+    samples: number[];
+    sd: number;
+    sem: number;
+    throughput: Statistics;
+    totalTime: number;
+    variance: number;
+  }
+
+  export type FnOptions = {
+    afterAll?: (this: Task) => void | Promise<void>,
+    afterEach?: (this: Task) => void | Promise<void>,
+    beforeAll?: (this: Task) => void | Promise<void>,
+    beforeEach?: (this: Task) => void | Promise<void>,
+  };
+
+  export type Fn = () => Promise<mixed> | mixed;
+
+  declare export class Bench extends EventTarget {
+    concurrency: null | 'task' | 'bench';
+    name?: string;
+    opts: $ReadOnly<BenchOptions>;
+    threshold: number;
+
+    constructor(options?: BenchOptions): this;
+
+    // $FlowExpectedError[unsafe-getters-setters]
+    get results(): Array<$ReadOnly<TaskResult>>;
+
+    // $FlowExpectedError[unsafe-getters-setters]
+    get tasks(): Array<Task>;
+
+    add(name: string, fn: Fn, fnOpts?: FnOptions): this;
+    getTask(name: string): void | Task;
+    remove(name: string): this;
+    reset(): void;
+    run(): Promise<Array<Task>>;
+    runSync(): Array<Task>;
+    table(
+      convert?: (task: Task) => Record<string, void | string | number>,
+    ): void | Array<Record<string, void | string | number>>;
+  }
+}

package.json

@@ -102,6 +102,7 @@
     "shelljs": "^0.8.5",
     "signedsource": "^1.0.0",
     "supports-color": "^7.1.0",
+    "tinybench": "^3.1.0",
     "typescript": "5.0.4",
     "ws": "^6.2.3"
   },

packages/react-native-fantom/runner/entrypoint-template.js

@@ -18,12 +18,14 @@ module.exports = function entrypointTemplate({
   featureFlagsModulePath,
   featureFlags,
   snapshotConfig,
+  isRunningFromCI,
 }: {
   testPath: string,
   setupModulePath: string,
   featureFlagsModulePath: string,
   featureFlags: FantomTestConfigJsOnlyFeatureFlags,
   snapshotConfig: SnapshotConfig,
+  isRunningFromCI: boolean,
 }): string {
   return `/**
  * Copyright (c) Meta Platforms, Inc. and affiliates.
@@ -38,6 +40,7 @@ module.exports = function entrypointTemplate({
  */
 
 import {registerTest} from '${setupModulePath}';
+import {setConstants} from '@react-native/fantom';
 ${
   Object.keys(featureFlags).length > 0
     ? `import * as ReactNativeFeatureFlags from '${featureFlagsModulePath}';
@@ -50,6 +53,10 @@ ${Object.entries(featureFlags)
     : ''
 }
 
+setConstants({
+  isRunningFromCI: ${String(isRunningFromCI)},
+});
+
 registerTest(() => require('${testPath}'), ${JSON.stringify(snapshotConfig)});
 `;
 };

packages/react-native-fantom/runner/getFantomTestConfig.js

@@ -46,6 +46,8 @@ const DEFAULT_MODE: FantomTestConfigMode =
 
 const FANTOM_FLAG_FORMAT = /^(\w+):(\w+)$/;
 
+const FANTOM_BENCHMARK_SUITE_RE = /\nbenchmark(\s*)\.suite\(/g;
+
 /**
  * Extracts the Fantom configuration from the test file, specified as part of
  * the docblock comment. E.g.:
@@ -69,7 +71,9 @@ const FANTOM_FLAG_FORMAT = /^(\w+):(\w+)$/;
 export default function getFantomTestConfig(
   testPath: string,
 ): FantomTestConfig {
-  const docblock = extract(fs.readFileSync(testPath, 'utf8'));
+  const testContents = fs.readFileSync(testPath, 'utf8');
+
+  const docblock = extract(testContents);
   const pragmas = parse(docblock) as DocblockPragmas;
 
   const config: FantomTestConfig = {
@@ -102,6 +106,10 @@ export default function getFantomTestConfig(
       default:
         throw new Error(`Invalid Fantom mode: ${mode}`);
     }
+  } else {
+    if (FANTOM_BENCHMARK_SUITE_RE.test(testContents)) {
+      config.mode = FantomTestConfigMode.Optimized;
+    }
   }
 
   const maybeRawFlagConfig = pragmas.fantom_flags;

packages/react-native-fantom/runner/runner.js

@@ -24,6 +24,7 @@ import {
   getBuckModesForPlatform,
   getDebugInfoFromCommandResult,
   getShortHash,
+  isRunningFromCI,
   printConsoleLog,
   runBuck2,
   runBuck2Sync,
@@ -198,6 +199,7 @@ module.exports = async function runTest(
       updateSnapshot: snapshotState._updateSnapshot,
       data: getInitialSnapshotData(snapshotState),
     },
+    isRunningFromCI: isRunningFromCI(),
   });
 
   const entrypointPath = path.join(

packages/react-native-fantom/runner/utils.js

@@ -67,6 +67,16 @@ export type SyncCommandResult = {
   stderr: string,
 };
 
+function isEmpty(value: ?string): boolean {
+  return value == null || value === '';
+}
+
+export function isRunningFromCI(): boolean {
+  return (
+    !isEmpty(process.env.SANDCASTLE) || !isEmpty(process.env.GITHUB_ACTIONS)
+  );
+}
+
 function maybeLogCommand(command: string, args: Array<string>): void {
   if (EnvironmentOptions.logCommands) {
     console.log(`RUNNING \`${command} ${args.join(' ')}\``);

packages/react-native-fantom/src/Benchmark.js

@@ -0,0 +1,129 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow strict-local
+ * @format
+ */
+
+import {getConstants} from './index';
+import nullthrows from 'nullthrows';
+import NativeCPUTime from 'react-native/src/private/specs/modules/NativeCPUTime';
+import {
+  Bench,
+  type BenchOptions,
+  type FnOptions,
+  type TaskResult,
+} from 'tinybench';
+
+type SuiteOptions = $ReadOnly<{
+  ...Pick<
+    BenchOptions,
+    'iterations' | 'time' | 'warmup' | 'warmupIterations' | 'warmupTime',
+  >,
+  disableOptimizedBuildCheck?: boolean,
+}>;
+
+type SuiteResults = Array<$ReadOnly<TaskResult>>;
+
+interface SuiteAPI {
+  add(name: string, fn: () => void, options?: FnOptions): SuiteAPI;
+  verify(fn: (results: SuiteResults) => void): SuiteAPI;
+}
+
+export function suite(
+  suiteName: string,
+  suiteOptions?: ?SuiteOptions,
+): SuiteAPI {
+  const tasks: Array<{
+    name: string,
+    fn: () => void,
+    options: FnOptions | void,
+  }> = [];
+  const verifyFns = [];
+
+  global.it(suiteName, () => {
+    if (tasks.length === 0) {
+      throw new Error('No benchmark tests defined');
+    }
+
+    const {isRunningFromCI} = getConstants();
+
+    // If we're running from CI and there's no verification function, there's
+    // no point in running the benchmark.
+    // We still run a single iteration of each test just to make sure that the
+    // logic in the benchmark doesn't break.
+    const isTestOnly = isRunningFromCI && verifyFns.length === 0;
+
+    const overriddenOptions: BenchOptions = isTestOnly
+      ? {
+          warmupIterations: 1,
+          warmupTime: 0,
+          iterations: 1,
+          time: 0,
+        }
+      : {};
+
+    const {disableOptimizedBuildCheck, ...benchOptions} = suiteOptions ?? {};
+
+    const bench = new Bench({
+      ...benchOptions,
+      ...overriddenOptions,
+      name: suiteName,
+      throws: true,
+      now: () => NativeCPUTime.getCPUTimeNanos() / 1000000,
+    });
+
+    for (const task of tasks) {
+      bench.add(task.name, task.fn, task.options);
+    }
+
+    bench.runSync();
+
+    if (!isTestOnly) {
+      printBenchmarkResults(bench);
+    }
+
+    for (const verify of verifyFns) {
+      verify(bench.results);
+    }
+
+    if (!isTestOnly && !NativeCPUTime.hasAccurateCPUTimeNanosForBenchmarks()) {
+      throw new Error(
+        '`NativeCPUTime` module does not provide accurate CPU time information in this environment. Please run the benchmarks in an environment where it does.',
+      );
+    }
+
+    if (__DEV__ && disableOptimizedBuildCheck !== true) {
+      throw new Error('Benchmarks should not be run in development mode');
+    }
+  });
+
+  const suiteAPI = {
+    add(name: string, fn: () => void, options?: FnOptions): SuiteAPI {
+      tasks.push({name, fn, options});
+      return suiteAPI;
+    },
+    verify(fn: (results: SuiteResults) => void): SuiteAPI {
+      verifyFns.push(fn);
+      return suiteAPI;
+    },
+  };
+
+  return suiteAPI;
+}
+
+function printBenchmarkResults(bench: Bench) {
+  const longestTaskNameLength = bench.tasks.reduce(
+    (maxLength, task) => Math.max(maxLength, task.name.length),
+    0,
+  );
+  const separatorWidth = 121 + longestTaskNameLength - 'Task name'.length;
+
+  console.log('-'.repeat(separatorWidth));
+  console.log(bench.name);
+  console.table(nullthrows(bench.table()));
+  console.log('-'.repeat(separatorWidth) + '\n');
+}