Update split chunk handling for edge/node (vercel#62205)

While investigating OOMs noticed that our split chunks handling for edge runtime isn't optimized properly causing a lot of duplicate transpiling/minification which causes exponential memory/cache usage. On a minimal app before this change the memory usage and cache size were over `2GB`s after this change they are under `200MB`. **Before** ![CleanShot 2024-02-17 at 16 08 46@2x](https://github.com/vercel/next.js/assets/22380829/5286e5fb-5e48-4296-a6be-d2ed1455eb95) **After** ![CleanShot 2024-02-18 at 08 58 51@2x](https://github.com/vercel/next.js/assets/22380829/f813a185-51a7-40e0-b44b-e1ab95649194) Closes NEXT-2521
NOUIY · Feb 18, 2024 · 0b5f7d9 · 0b5f7d9
1 parent 7bf8257
commit 0b5f7d9
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 46 deletions.
diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts
@@ -715,11 +715,15 @@ export default async function getBaseWebpackConfig(
   // Packages which will be split into the 'framework' chunk.
   // Only top-level packages are included, e.g. nested copies like
   // 'node_modules/meow/node_modules/object-assign' are not included.
+  const nextFrameworkPaths: string[] = []
   const topLevelFrameworkPaths: string[] = []
   const visitedFrameworkPackages = new Set<string>()
-
   // Adds package-paths of dependencies recursively
-  const addPackagePath = (packageName: string, relativeToPath: string) => {
+  const addPackagePath = (
+    packageName: string,
+    relativeToPath: string,
+    paths: string[]
+  ) => {
     try {
       if (visitedFrameworkPackages.has(packageName)) {
         return
@@ -738,11 +742,11 @@ export default async function getBaseWebpackConfig(
       const directory = path.join(packageJsonPath, '../')
 
       // Returning from the function in case the directory has already been added and traversed
-      if (topLevelFrameworkPaths.includes(directory)) return
-      topLevelFrameworkPaths.push(directory)
+      if (paths.includes(directory)) return
+      paths.push(directory)
       const dependencies = require(packageJsonPath).dependencies || {}
       for (const name of Object.keys(dependencies)) {
-        addPackagePath(name, directory)
+        addPackagePath(name, directory, paths)
       }
     } catch (_) {
       // don't error on failing to resolve framework packages
@@ -759,8 +763,9 @@ export default async function getBaseWebpackConfig(
         ]
       : []),
   ]) {
-    addPackagePath(packageName, dir)
+    addPackagePath(packageName, dir, topLevelFrameworkPaths)
   }
+  addPackagePath('next', dir, nextFrameworkPaths)
 
   const crossOrigin = config.crossOrigin
 
@@ -913,6 +918,7 @@ export default async function getBaseWebpackConfig(
       splitChunks: (():
         | Required<webpack.Configuration>['optimization']['splitChunks']
         | false => {
+        // server chunking
         if (dev) {
           if (isNodeServer) {
             /*
@@ -963,21 +969,6 @@ export default async function getBaseWebpackConfig(
           return false
         }
 
-        if (isNodeServer) {
-          return {
-            filename: '[name].js',
-            chunks: 'all',
-            minChunks: 2,
-          }
-        }
-
-        if (isEdgeServer) {
-          return {
-            filename: 'edge-chunks/[name].js',
-            minChunks: 2,
-          }
-        }
-
         const frameworkCacheGroup = {
           chunks: 'all' as const,
           name: 'framework',
@@ -996,6 +987,22 @@ export default async function getBaseWebpackConfig(
           // becoming a part of the commons chunk)
           enforce: true,
         }
+
+        const nextRuntimeCacheGroup = {
+          chunks: 'all' as const,
+          name: 'next-runtime',
+          test(module: any) {
+            const resource = module.nameForCondition?.()
+            return resource
+              ? nextFrameworkPaths.some((pkgPath) =>
+                  resource.startsWith(pkgPath)
+                )
+              : false
+          },
+          priority: 30,
+          enforce: true,
+        }
+
         const libCacheGroup = {
           test(module: {
             size: Function
@@ -1037,6 +1044,20 @@ export default async function getBaseWebpackConfig(
           minChunks: 1,
           reuseExistingChunk: true,
         }
+
+        if (isNodeServer || isEdgeServer) {
+          return {
+            filename: `${isEdgeServer ? 'edge-chunks/' : ''}[name].js`,
+            cacheGroups: {
+              nextRuntime: nextRuntimeCacheGroup,
+              framework: frameworkCacheGroup,
+              lib: libCacheGroup,
+            },
+            minChunks: 2,
+          }
+        }
+
+        // client chunking
         const cssCacheGroup = {
           test: /\.(css|sass|scss)$/i,
           chunks: 'all' as const,

diff --git a/packages/next/src/build/webpack/plugins/terser-webpack-plugin/src/index.ts b/packages/next/src/build/webpack/plugins/terser-webpack-plugin/src/index.ts
@@ -98,16 +98,6 @@ export class TerserPlugin {
               return false
             }
 
-            // don't minify _middleware as it can break in some cases
-            // and doesn't provide too much of a benefit as it's server-side
-            if (
-              name.match(
-                /(edge-runtime-webpack\.js|edge-chunks|middleware\.js$)/
-              )
-            ) {
-              return false
-            }
-
             const { info } = res
 
             // Skip double minimize assets from child compilation

diff --git a/test/e2e/middleware-trailing-slash/test/index.test.ts b/test/e2e/middleware-trailing-slash/test/index.test.ts
@@ -111,11 +111,11 @@ describe('Middleware Runtime trailing slash', () => {
         )
         expect(manifest.middleware).toEqual({
           '/': {
-            files: [
+            files: expect.arrayContaining([
               'prerender-manifest.js',
               'server/edge-runtime-webpack.js',
               'server/middleware.js',
-            ],
+            ]),
             name: 'middleware',
             page: '/',
             matchers: [{ regexp: '^/.*$', originalSource: '/:path*' }],

diff --git a/test/e2e/switchable-runtime/index.test.ts b/test/e2e/switchable-runtime/index.test.ts
@@ -187,10 +187,10 @@ describe('Switchable runtime', () => {
           expect(manifest).toMatchObject({
             functions: {
               '/api/hello': {
-                files: [
+                files: expect.arrayContaining([
                   'server/edge-runtime-webpack.js',
                   'server/pages/api/hello.js',
-                ],
+                ]),
                 name: 'pages/api/hello',
                 page: '/api/hello',
                 matchers: [
@@ -199,10 +199,10 @@ describe('Switchable runtime', () => {
                 wasm: [],
               },
               '/api/edge': {
-                files: [
+                files: expect.arrayContaining([
                   'server/edge-runtime-webpack.js',
                   'server/pages/api/edge.js',
-                ],
+                ]),
                 name: 'pages/api/edge',
                 page: '/api/edge',
                 matchers: [
@@ -621,11 +621,11 @@ describe('Switchable runtime', () => {
           expect(manifest).toMatchObject({
             functions: {
               '/api/hello': {
-                files: [
+                files: expect.arrayContaining([
                   'prerender-manifest.js',
                   'server/edge-runtime-webpack.js',
                   'server/pages/api/hello.js',
-                ],
+                ]),
                 name: 'pages/api/hello',
                 page: '/api/hello',
                 matchers: [
@@ -634,11 +634,11 @@ describe('Switchable runtime', () => {
                 wasm: [],
               },
               '/api/edge': {
-                files: [
+                files: expect.arrayContaining([
                   'prerender-manifest.js',
                   'server/edge-runtime-webpack.js',
                   'server/pages/api/edge.js',
-                ],
+                ]),
                 name: 'pages/api/edge',
                 page: '/api/edge',
                 matchers: [

diff --git a/test/integration/build-trace-extra-entries/test/index.test.js b/test/integration/build-trace-extra-entries/test/index.test.js
@@ -50,11 +50,6 @@ describe('build trace with extra entries', () => {
       expect(appTrace.files.some((file) => file.endsWith('hello.json'))).toBe(
         true
       )
-      expect(
-        appTrace.files.filter(
-          (file) => file.includes('chunks') && file.endsWith('.js')
-        ).length
-      ).toBe(0)
 
       expect(
         indexTrace.files.filter(