[ci] Parallelize yarn build

ghstack-source-id: 78cf0304633fc31f2372c683258b543ddd20a7af
Pull Request resolved: facebook#30071

.circleci/config.yml

@@ -97,7 +97,7 @@ jobs:
     steps:
       - checkout
       - setup_node_modules
-      - run: yarn build
+      - run: yarn build --ci=circleci
       - persist_to_workspace:
           root: .
           paths:

.github/workflows/runtime_build.yml

@@ -11,6 +11,11 @@ jobs:
   build_and_lint:
     name: yarn build and lint
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # yml is dumb. update the --total arg to yarn build if you change the number of workers
+        worker_id: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
+        release_channel: [stable, experimental]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -25,11 +30,18 @@ jobs:
           path: "**/node_modules"
           key: ${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
       - run: yarn install --frozen-lockfile
-      - run: yarn build
-      - run: yarn lint-build
-      - name: Cache build
-        uses: actions/cache@v4
-        id: build_cache
+      - run: yarn build --index=${{ matrix.worker_id }} --total=20 --r=${{ matrix.release_channel }} --ci=github
+        env:
+          CI: github
+          RELEASE_CHANNEL: ${{ matrix.release_channel }}
+          NODE_INDEX: ${{ matrix.worker_id }}
+      - name: Lint build
+        run: yarn lint-build
+      - name: Display structure of build
+        run: ls -R build
+      - name: Archive build
+        uses: actions/upload-artifact@v4
         with:
-          path: "build/**"
-          key: yarn-build-${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
+          name: build_${{ matrix.worker_id }}_${{ matrix.release_channel }}
+          path: |
+            build

scripts/rollup/build-all-release-channels.js

@@ -15,6 +15,8 @@ const {
   canaryChannelLabel,
   rcNumber,
 } = require('../../ReactVersions');
+const yargs = require('yargs');
+const {buildEverything} = require('./build-ghaction');
 
 // Runs the build script for both stable and experimental release channels,
 // by configuring an environment variable.
@@ -53,44 +55,88 @@ fs.writeFileSync(
   `export default '${PLACEHOLDER_REACT_VERSION}';\n`
 );
 
-if (process.env.CIRCLE_NODE_TOTAL) {
-  // In CI, we use multiple concurrent processes. Allocate half the processes to
-  // build the stable channel, and the other half for experimental. Override
-  // the environment variables to "trick" the underlying build script.
-  const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
-  const halfTotal = Math.floor(total / 2);
-  const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
-  if (index < halfTotal) {
-    const nodeTotal = halfTotal;
-    const nodeIndex = index;
-    buildForChannel('stable', nodeTotal, nodeIndex);
-    processStable('./build');
+const argv = yargs.wrap(yargs.terminalWidth()).options({
+  releaseChannel: {
+    alias: 'r',
+    describe: 'Build the given release channel.',
+    requiresArg: true,
+    type: 'string',
+    choices: ['experimental', 'stable'],
+  },
+  index: {
+    alias: 'i',
+    describe: 'Worker id.',
+    requiresArg: true,
+    type: 'number',
+  },
+  total: {
+    alias: 't',
+    describe: 'Total number of workers.',
+    requiresArg: true,
+    type: 'number',
+  },
+  ci: {
+    describe: 'Run tests in CI',
+    requiresArg: false,
+    type: 'choices',
+    choices: ['circleci', 'github'],
+  },
+}).argv;
+
+async function main() {
+  if (argv.ci === 'github') {
+    await buildEverything(argv.index, argv.total);
+    switch (argv.releaseChannel) {
+      case 'stable': {
+        processStable('./build');
+        break;
+      }
+      case 'experimental': {
+        processExperimental('./build');
+        break;
+      }
+      default:
+        throw new Error(`Unknown release channel ${argv.releaseChannel}`);
+    }
+  } else if (argv.ci === 'circleci') {
+    // In CI, we use multiple concurrent processes. Allocate half the processes to
+    // build the stable channel, and the other half for experimental. Override
+    // the environment variables to "trick" the underlying build script.
+    const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
+    const halfTotal = Math.floor(total / 2);
+    const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
+    if (index < halfTotal) {
+      const nodeTotal = halfTotal;
+      const nodeIndex = index;
+      buildForChannel('stable', nodeTotal, nodeIndex);
+      processStable('./build');
+    } else {
+      const nodeTotal = total - halfTotal;
+      const nodeIndex = index - halfTotal;
+      buildForChannel('experimental', nodeTotal, nodeIndex);
+      processExperimental('./build');
+    }
   } else {
-    const nodeTotal = total - halfTotal;
-    const nodeIndex = index - halfTotal;
-    buildForChannel('experimental', nodeTotal, nodeIndex);
-    processExperimental('./build');
+    // Running locally, no concurrency. Move each channel's build artifacts into
+    // a temporary directory so that they don't conflict.
+    buildForChannel('stable', '', '');
+    const stableDir = tmp.dirSync().name;
+    crossDeviceRenameSync('./build', stableDir);
+    processStable(stableDir);
+    buildForChannel('experimental', '', '');
+    const experimentalDir = tmp.dirSync().name;
+    crossDeviceRenameSync('./build', experimentalDir);
+    processExperimental(experimentalDir);
+
+    // Then merge the experimental folder into the stable one. processExperimental
+    // will have already removed conflicting files.
+    //
+    // In CI, merging is handled automatically by CircleCI's workspace feature.
+    mergeDirsSync(experimentalDir + '/', stableDir + '/');
+
+    // Now restore the combined directory back to its original name
+    crossDeviceRenameSync(stableDir, './build');
   }
-} else {
-  // Running locally, no concurrency. Move each channel's build artifacts into
-  // a temporary directory so that they don't conflict.
-  buildForChannel('stable', '', '');
-  const stableDir = tmp.dirSync().name;
-  crossDeviceRenameSync('./build', stableDir);
-  processStable(stableDir);
-  buildForChannel('experimental', '', '');
-  const experimentalDir = tmp.dirSync().name;
-  crossDeviceRenameSync('./build', experimentalDir);
-  processExperimental(experimentalDir);
-
-  // Then merge the experimental folder into the stable one. processExperimental
-  // will have already removed conflicting files.
-  //
-  // In CI, merging is handled automatically by CircleCI's workspace feature.
-  mergeDirsSync(experimentalDir + '/', stableDir + '/');
-
-  // Now restore the combined directory back to its original name
-  crossDeviceRenameSync(stableDir, './build');
 }
 
 function buildForChannel(channel, nodeTotal, nodeIndex) {
@@ -457,3 +503,5 @@ function mergeDirsSync(source, destination) {
     }
   }
 }
+
+main();