Merge branch 'vercel:canary' into canary

X-oss-byte · Aug 15, 2023 · c9ce304 · c9ce304
2 parents b0ad169 + e02397d
commit c9ce304
Show file tree

Hide file tree

Showing 199 changed files with 2,997 additions and 1,602 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -2,4 +2,4 @@ packages/next/bundles/** -text
 packages/next/compiled/** -text
 
 # Make next/src/build folder indexable for github search
-packages/next/src/build/** linguist-generated=false
+build/** linguist-generated=false
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -20,6 +20,7 @@
 
 /**/*image*                                   @timneutkens @ijjk @shuding @styfle @huozhi @vercel/devex
 /**/*image*/**                                @timneutkens @ijjk @shuding @styfle @huozhi @vercel/devex
+/**/*img*                                     @timneutkens @ijjk @shuding @styfle @huozhi @vercel/devex
 /packages/next/client/use-intersection.tsx    @timneutkens @ijjk @shuding @styfle
 /packages/next/server/lib/squoosh/            @timneutkens @ijjk @shuding @styfle
 /packages/next/server/serve-static.ts         @timneutkens @ijjk @shuding @styfle @huozhi
@@ -33,4 +34,4 @@
 Cargo.toml                                    @timneutkens @ijjk @shuding @vercel/web-tooling
 Cargo.lock                                    @timneutkens @ijjk @shuding @vercel/web-tooling
 /.cargo/config.toml                           @timneutkens @ijjk @shuding @vercel/web-tooling
-/.config/nextest.toml                         @timneutkens @ijjk @shuding @vercel/web-tooling
+/.config/nextest.toml                         @timneutkens @ijjk @shuding @vercel/web-tooling
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
@@ -6,11 +6,13 @@ on:
   workflow_dispatch:
 
 env:
-  NAPI_CLI_VERSION: 2.14.7
+  NAPI_CLI_VERSION: 2.16.2
   TURBO_VERSION: 1.10.9
   NODE_MAINTENANCE_VERSION: 16
   NODE_LTS_VERSION: 18.16.1
   CARGO_PROFILE_RELEASE_LTO: 'true'
+  TURBO_TEAM: 'vercel'
+  TURBO_REMOTE_ONLY: 'true'
 
 jobs:
   build:
@@ -20,7 +22,6 @@ jobs:
       # we build a dev binary for use in CI so skip downloading
       # canary next-swc binaries in the monorepo
       NEXT_SKIP_NATIVE_POSTINSTALL: 1
-      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
     outputs:
       isRelease: ${{ github.event_name != 'workflow_dispatch' && steps.check-release.outputs.IS_RELEASE }}
     steps:
@@ -70,107 +71,157 @@ jobs:
 
   # Build binaries for publishing
   build-native:
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
     strategy:
       fail-fast: false
       matrix:
         settings:
-          # pnpm is aliased here temporarily until the build docker
-          # image is updated past Node.js v14.19 (current 14.18.1)
-          - host: macos-latest
+          - host:
+              - 'self-hosted'
+              - 'macos'
+              - 'arm64'
+
             target: 'x86_64-apple-darwin'
             build: |
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable
               turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-apple-darwin --release
               strip -x packages/next-swc/native/next-swc.*.node
-          - host: windows-latest
+
+          - host:
+              - 'self-hosted'
+              - 'macos'
+              - 'arm64'
+
+            target: 'aarch64-apple-darwin'
+            build: |
+              export CC=$(xcrun -f clang);
+              export CXX=$(xcrun -f clang++);
+              SYSROOT=$(xcrun --sdk macosx --show-sdk-path);
+              export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT";
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable
+              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-apple-darwin
+              strip -x packages/next-swc/native/next-swc.*.node
+
+          - host:
+              - 'self-hosted'
+              - 'windows'
+              - 'x64'
+
             build: |
               corepack enable
               npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
               turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-pc-windows-msvc
             target: 'x86_64-pc-windows-msvc'
-          - host: windows-latest
+
+          - host:
+              - 'self-hosted'
+              - 'windows'
+              - 'x64'
+
             build: |
               corepack enable
               npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
               turbo run build-native-no-plugin --remote-cache-timeout 90 --summarize -- --release --target i686-pc-windows-msvc
             target: 'i686-pc-windows-msvc'
-          - host: ubuntu-latest
+
+          - host:
+              - 'self-hosted'
+              - 'windows'
+              - 'x64'
+
+            target: 'aarch64-pc-windows-msvc'
+            build: |
+              corepack enable
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
+              turbo run build-native-no-plugin-woa-release --remote-cache-timeout 90 --summarize -- --target aarch64-pc-windows-msvc
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+
             target: 'x86_64-unknown-linux-gnu'
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
             build: >-
               set -e &&
               apt update &&
-              apt install -y pkg-config &&
+              apt install -y pkg-config xz-utils &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add x86_64-unknown-linux-gnu &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
               unset CC_x86_64_unknown_linux_gnu && unset CC &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-gnu &&
-              strip packages/next-swc/native/next-swc.*.node
-          - host: ubuntu-latest
+              cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-gnu &&
+              strip native/next-swc.*.node &&
+              objdump -T native/next-swc.*.node | grep GLIBC_
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+
             target: 'x86_64-unknown-linux-musl'
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
             build: >-
               set -e &&
+              apk update &&
               apk add --no-cache libc6-compat pkgconfig &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add x86_64-unknown-linux-musl &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-musl &&
-              strip packages/next-swc/native/next-swc.*.node
-          - host: macos-latest
-            target: 'aarch64-apple-darwin'
-            build: |
-              sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*;
-              export CC=$(xcrun -f clang);
-              export CXX=$(xcrun -f clang++);
-              SYSROOT=$(xcrun --sdk macosx --show-sdk-path);
-              export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT";
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-apple-darwin
-              strip -x packages/next-swc/native/next-swc.*.node
-          - host: ubuntu-latest
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
+              cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-musl &&
+              strip native/next-swc.*.node
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+
             target: 'aarch64-unknown-linux-gnu'
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-aarch64
             build: >-
               set -e &&
               apt update &&
-              apt install -y pkg-config &&
+              apt install -y pkg-config xz-utils &&
               export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add aarch64-unknown-linux-gnu &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
               export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
-              llvm-strip -x packages/next-swc/native/next-swc.*.node
-          - host: ubuntu-latest
+              cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
+              llvm-strip -x native/next-swc.*.node &&
+              objdump -T native/next-swc.*.node | grep GLIBC_
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+
             target: 'aarch64-unknown-linux-musl'
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
             build: >-
               set -e &&
+              apk update &&
               apk add --no-cache libc6-compat pkgconfig &&
               export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add aarch64-unknown-linux-musl &&
-              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-musl &&
-              llvm-strip -x packages/next-swc/native/next-swc.*.node
-          - host: windows-latest
-            target: 'aarch64-pc-windows-msvc'
-            build: |
-              corepack enable
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
-              turbo run build-native-no-plugin-woa-release --remote-cache-timeout 90 --summarize -- --target aarch64-pc-windows-msvc
+              cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-musl &&
+              llvm-strip -x native/next-swc.*.node
+
     name: stable - ${{ matrix.settings.target }} - node@16
     runs-on: ${{ matrix.settings.host }}
-    env:
-      TURBO_TEAM: 'vercel'
-      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
-      TURBO_REMOTE_ONLY: 'true'
     steps:
       # https://github.com/actions/virtual-environments/issues/1187
       - name: tune linux network
@@ -202,65 +253,44 @@ jobs:
           targets: ${{ matrix.settings.target }}
           skip-install: ${{ matrix.settings.docker }}
 
-      - name: Cache cargo registry
-        uses: actions/cache@v3
-        timeout-minutes: 5
-        with:
-          path: ~/.cargo/registry
-          key: ${{ matrix.settings.target }}-cargo-registry
-
-      - name: Cache cargo index
-        uses: actions/cache@v3
-        timeout-minutes: 5
-        with:
-          path: ~/.cargo/git
-          key: ${{ matrix.settings.target }}-cargo-index
-
       - name: normalize versions
         run: node scripts/normalize-version-bump.js
 
       - name: Setup toolchain
         run: ${{ matrix.settings.setup }}
         if: ${{ matrix.settings.setup }}
-        shell: bash
 
-      - name: Build in docker
-        uses: addnab/docker-run-action@v3
+      # we only need custom caching for docker builds
+      # as they are on an older Node.js version and have
+      # issues with turbo caching
+      - name: pull build cache
         if: ${{ matrix.settings.docker }}
-        with:
-          image: ${{ matrix.settings.docker }}
-          options: >-
-            -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
-            -e CARGO_INCREMENTAL=${{ env.CARGO_INCREMENTAL }}
-            -e CARGO_PROFILE_RELEASE_LTO=${{ env.CARGO_PROFILE_RELEASE_LTO }}
-            -e CARGO_TERM_COLOR=${{ env.CARGO_TERM_COLOR }}
-            -e RUST_BACKTRACE=${{ env.RUST_BACKTRACE }}
-            -e CARGO_REGISTRIES_CRATES_IO_PROTOCOL=${{ env.CARGO_REGISTRIES_CRATES_IO_PROTOCOL }}
-            -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }}
-            -e TURBO_VERSION=${{ env.TURBO_VERSION }}
-            -e TURBO_TEAM=vercel
-            -e TURBO_TOKEN=${{ secrets.TURBO_TOKEN }}
-            -e TURBO_REMOTE_ONLY=true
-            -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git
-            -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry
-            -v ${{ github.workspace }}:/build
-            -w /build
-          run: ${{ matrix.settings.build }}
+        run: node ./scripts/pull-turbo-cache.js ${{ matrix.settings.target }}
+
+      - name: check build exists
+        if: ${{ matrix.settings.docker }}
+        run: if [ -f packages/next-swc/native/next-swc.*.node ]; then echo "BUILD_EXISTS=yes" >> $GITHUB_OUTPUT; else echo "BUILD_EXISTS=no" >> $GITHUB_OUTPUT; fi
+        id: build-exists
+
+      - name: Build in docker
+        if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
+        run: docker run -v "/var/run/docker.sock":"/var/run/docker.sock"  -e RUST_TOOLCHAIN -e RUST_BACKTRACE -e NAPI_CLI_VERSION -e CARGO_TERM_COLOR -e CARGO_INCREMENTAL -e CARGO_PROFILE_RELEASE_LTO -e CARGO_REGISTRIES_CRATES_IO_PROTOCOL -e TURBO_API -e TURBO_TEAM -e TURBO_TOKEN -e TURBO_VERSION -e TURBO_REMOTE_ONLY -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build --entrypoint=bash ${{ matrix.settings.docker }} -c "${{ matrix.settings.build }}"
+
+      - name: cache build
+        if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
+        run: pnpm turbo run cache-build-native --force -- ${{ matrix.settings.target }}
 
       - name: 'Build'
         run: ${{ matrix.settings.build }}
         if: ${{ !matrix.settings.docker }}
-        shell: bash
 
       - name: 'check build cache status'
         id: check-did-build
-        shell: bash
-        run: if [[ ! -z $(ls .turbo/runs) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi
+        run: if [[ ! -z $(ls packages/next-swc/native) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi
 
       # Trying to upload metrics for the Turbopack to datadog's CI pipeline execution
       - name: 'Collect turbopack build metrics'
         id: check-turbopack-bytesize
-        shell: bash
         if: ${{ steps.check-did-build.outputs.DID_BUILD == 'yup' }}
         continue-on-error: true
         run: |
@@ -299,11 +329,13 @@ jobs:
     strategy:
       matrix:
         target: [web, nodejs]
-    runs-on: macos-latest
-    env:
-      TURBO_TEAM: 'vercel'
-      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
-      TURBO_REMOTE_ONLY: 'true'
+
+    runs-on:
+      - 'self-hosted'
+      - 'linux'
+      - 'x64'
+      - 'metal'
+
     steps:
       - uses: actions/checkout@v3