[ci] Parallelize yarn build and yarn lint-build

ghstack-source-id: fc61ba266b164d9bf3831d4d6e37da261481a916 Pull Request resolved: #30071
facebook · Jun 24, 2024 · c46e49e · c46e49e
1 parent b991643
commit c46e49e
Show file tree

Hide file tree

Showing 4 changed files with 995 additions and 49 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -97,7 +97,7 @@ jobs:
     steps:
       - checkout
       - setup_node_modules
-      - run: yarn build
+      - run: yarn build --ci=circleci
       - persist_to_workspace:
           root: .
           paths:

diff --git a/.github/workflows/runtime_build.yml b/.github/workflows/runtime_build.yml
@@ -8,9 +8,34 @@ on:
       - 'compiler/**'
 
 jobs:
+  define_build_params:
+    name: Build build params
+    runs-on: ubuntu-latest
+    outputs:
+      bundle_type: ${{ steps.define_bundle_types.outputs.result }}
+      release_channel:  ${{ steps.define_release_channels.outputs.result }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
+        id: define_bundle_types
+        with:
+          script: |
+            const {bundleTypes} = require('./scripts/rollup/bundles');
+            return Object.values(bundleTypes);
+      - uses: actions/github-script@v7
+        id: define_release_channels
+        with:
+          script: |
+            return ['stable', 'experimental'];
+
   build:
     name: yarn build
     runs-on: ubuntu-latest
+    needs: define_build_params
+    strategy:
+      matrix:
+        bundle_type: ${{ fromJSON(needs.define_build_params.outputs.bundle_type) }}
+        release_channel: ${{ fromJSON(needs.define_build_params.outputs.release_channel) }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -25,17 +50,21 @@ jobs:
           path: "**/node_modules"
           key: ${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
       - run: yarn install --frozen-lockfile
-      - run: yarn build
-      - name: Cache build
-        uses: actions/cache@v4
-        id: build_cache
+      - run: yarn build --b=${{ matrix.bundle_type }} --r=${{ matrix.release_channel }} --ci=github
+      - name: Archive build
+        uses: actions/upload-artifact@v4
         with:
-          path: "build/**"
-          key: yarn-build-${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
+          name: ${{ matrix.bundle_type }}-${{ matrix.release_channel }}
+          path: |
+            build/**
 
   lint_build:
     name: yarn lint-build
-    needs: build
+    needs: [define_build_params, build]
+    strategy:
+      matrix:
+        bundle_type: ${{ fromJSON(needs.define_build_params.outputs.bundle_type) }}
+        release_channel: ${{ fromJSON(needs.define_build_params.outputs.release_channel) }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -50,11 +79,9 @@ jobs:
         with:
           path: "**/node_modules"
           key: ${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
-      - name: Restore cached build
-        uses: actions/cache@v4
-        id: build_cache
+      - name: Restore archived build
+        uses: actions/download-artifact@v4
         with:
-          path: "build/**"
-          key: yarn-build-${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
+          name: ${{ matrix.bundle_type }}-${{ matrix.release_channel }}
       - run: yarn install --frozen-lockfile
       - run: yarn lint-build
diff --git a/scripts/rollup/build-all-release-channels.js b/scripts/rollup/build-all-release-channels.js
@@ -15,6 +15,9 @@ const {
   canaryChannelLabel,
   rcNumber,
 } = require('../../ReactVersions');
+const yargs = require('yargs');
+const Bundles = require('./bundles');
+const {buildEverything} = require('./build-ghaction');
 
 // Runs the build script for both stable and experimental release channels,
 // by configuring an environment variable.
@@ -53,44 +56,87 @@ fs.writeFileSync(
   `export default '${PLACEHOLDER_REACT_VERSION}';\n`
 );
 
-if (process.env.CIRCLE_NODE_TOTAL) {
-  // In CI, we use multiple concurrent processes. Allocate half the processes to
-  // build the stable channel, and the other half for experimental. Override
-  // the environment variables to "trick" the underlying build script.
-  const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
-  const halfTotal = Math.floor(total / 2);
-  const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
-  if (index < halfTotal) {
-    const nodeTotal = halfTotal;
-    const nodeIndex = index;
-    buildForChannel('stable', nodeTotal, nodeIndex);
-    processStable('./build');
+const argv = yargs.wrap(yargs.terminalWidth()).options({
+  releaseChannel: {
+    alias: 'r',
+    describe: 'Build the given release channel.',
+    requiresArg: true,
+    type: 'string',
+    default: 'experimental',
+    choices: ['experimental', 'stable'],
+  },
+  bundleType: {
+    alias: 'b',
+    describe: 'Build the given bundle type.',
+    requiresArg: true,
+    type: 'string',
+    choices: Object.values(Bundles.bundleTypes),
+  },
+  ci: {
+    describe: 'Run tests in CI',
+    requiresArg: false,
+    type: 'choices',
+    choices: ['circleci', 'github'],
+  },
+}).argv;
+
+async function main() {
+  if (argv.ci === 'github') {
+    // ./scripts/rollup/build was being used by spawning a new process and passing via ENV variables
+    // so let's just preserve this for now and rewrite it later to just take a function arg
+    process.env.RELEASE_CHANNEL = argv.releaseChannel;
+    await buildEverything(argv.bundleType);
+    switch (argv.releaseChannel) {
+      case 'stable': {
+        processStable('./build');
+        break;
+      }
+      case 'experimental': {
+        processExperimental('./build');
+        break;
+      }
+      default:
+        throw new Error(`Unknown release channel ${argv.releaseChannel}`);
+    }
+  } else if (argv.ci === 'circleci') {
+    // In CI, we use multiple concurrent processes. Allocate half the processes to
+    // build the stable channel, and the other half for experimental. Override
+    // the environment variables to "trick" the underlying build script.
+    const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
+    const halfTotal = Math.floor(total / 2);
+    const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
+    if (index < halfTotal) {
+      const nodeTotal = halfTotal;
+      const nodeIndex = index;
+      buildForChannel('stable', nodeTotal, nodeIndex);
+      processStable('./build');
+    } else {
+      const nodeTotal = total - halfTotal;
+      const nodeIndex = index - halfTotal;
+      buildForChannel('experimental', nodeTotal, nodeIndex);
+      processExperimental('./build');
+    }
   } else {
-    const nodeTotal = total - halfTotal;
-    const nodeIndex = index - halfTotal;
-    buildForChannel('experimental', nodeTotal, nodeIndex);
-    processExperimental('./build');
+    // Running locally, no concurrency. Move each channel's build artifacts into
+    // a temporary directory so that they don't conflict.
+    buildForChannel('stable', '', '');
+    const stableDir = tmp.dirSync().name;
+    crossDeviceRenameSync('./build', stableDir);
+    processStable(stableDir);
+    buildForChannel('experimental', '', '');
+    const experimentalDir = tmp.dirSync().name;
+    crossDeviceRenameSync('./build', experimentalDir);
+    processExperimental(experimentalDir);
+
+    // Then merge the experimental folder into the stable one. processExperimental
+    // will have already removed conflicting files.
+    //
+    // In CI, merging is handled automatically by CircleCI's workspace feature.
+    mergeDirsSync(experimentalDir + '/', stableDir + '/');
+
+    // Now restore the combined directory back to its original name
+    crossDeviceRenameSync(stableDir, './build');
   }
-} else {
-  // Running locally, no concurrency. Move each channel's build artifacts into
-  // a temporary directory so that they don't conflict.
-  buildForChannel('stable', '', '');
-  const stableDir = tmp.dirSync().name;
-  crossDeviceRenameSync('./build', stableDir);
-  processStable(stableDir);
-  buildForChannel('experimental', '', '');
-  const experimentalDir = tmp.dirSync().name;
-  crossDeviceRenameSync('./build', experimentalDir);
-  processExperimental(experimentalDir);
-
-  // Then merge the experimental folder into the stable one. processExperimental
-  // will have already removed conflicting files.
-  //
-  // In CI, merging is handled automatically by CircleCI's workspace feature.
-  mergeDirsSync(experimentalDir + '/', stableDir + '/');
-
-  // Now restore the combined directory back to its original name
-  crossDeviceRenameSync(stableDir, './build');
 }
 
 function buildForChannel(channel, nodeTotal, nodeIndex) {
@@ -457,3 +503,5 @@ function mergeDirsSync(source, destination) {
     }
   }
 }
+
+main();