diff --git a/code/frameworks/nextjs/package.json b/code/frameworks/nextjs/package.json
index 4a6c60c0ba62..afc8d9ce1f63 100644
--- a/code/frameworks/nextjs/package.json
+++ b/code/frameworks/nextjs/package.json
@@ -88,7 +88,6 @@
     "@babel/preset-react": "^7.22.15",
     "@babel/preset-typescript": "^7.23.2",
     "@babel/runtime": "^7.23.2",
-    "@pmmmwh/react-refresh-webpack-plugin": "^0.5.11",
     "@storybook/addon-actions": "workspace:*",
     "@storybook/builder-webpack5": "workspace:*",
     "@storybook/core-common": "workspace:*",
diff --git a/code/frameworks/nextjs/src/swc/loader.ts b/code/frameworks/nextjs/src/swc/loader.ts
index 687f60d8aefe..08bb292efae4 100644
--- a/code/frameworks/nextjs/src/swc/loader.ts
+++ b/code/frameworks/nextjs/src/swc/loader.ts
@@ -1,7 +1,6 @@
 import { getProjectRoot } from '@storybook/core-common';
 import { getVirtualModuleMapping } from '@storybook/core-webpack';
-import type { Options } from '@storybook/types';
-import ReactRefreshWebpackPlugin from '@pmmmwh/react-refresh-webpack-plugin';
+import type { Options, Preset } from '@storybook/types';
 import type { NextConfig } from 'next';
 import path from 'path';
 import type { RuleSetRule } from 'webpack';
@@ -9,6 +8,13 @@ import semver from 'semver';
 import { NextjsSWCNotSupportedError } from '@storybook/core-events/server-errors';
 import { getNextjsVersion } from '../utils';
 
+const applyFastRefresh = async (options: Options) => {
+  const isDevelopment = options.configType === 'DEVELOPMENT';
+  const framework = await options.presets.apply<Preset>('framework');
+  const reactOptions = typeof framework === 'object' ? framework.options : {};
+  return isDevelopment && (reactOptions.fastRefresh || process.env.FAST_REFRESH === 'true');
+};
+
 export const configureSWCLoader = async (
   baseConfig: any,
   options: Options,
@@ -23,15 +29,6 @@ export const configureSWCLoader = async (
 
   const dir = getProjectRoot();
 
-  baseConfig.plugins = [
-    ...baseConfig.plugins,
-    new ReactRefreshWebpackPlugin({
-      overlay: {
-        sockIntegration: 'whm',
-      },
-    }),
-  ];
-
   const virtualModules = await getVirtualModuleMapping(options);
 
   baseConfig.module.rules = [
@@ -53,7 +50,7 @@ export const configureSWCLoader = async (
           rootDir: dir,
           pagesDir: `${dir}/pages`,
           appDir: `${dir}/apps`,
-          hasReactRefresh: isDevelopment,
+          hasReactRefresh: applyFastRefresh(options),
           nextConfig,
           supportedBrowsers: require('next/dist/build/utils').getSupportedBrowsers(
             dir,
diff --git a/code/yarn.lock b/code/yarn.lock
index 44f7f1428778..88ea95a5e552 100644
--- a/code/yarn.lock
+++ b/code/yarn.lock
@@ -6854,7 +6854,6 @@ __metadata:
     "@babel/preset-typescript": "npm:^7.23.2"
     "@babel/runtime": "npm:^7.23.2"
     "@babel/types": "npm:^7.23.0"
-    "@pmmmwh/react-refresh-webpack-plugin": "npm:^0.5.11"
     "@storybook/addon-actions": "workspace:*"
     "@storybook/builder-webpack5": "workspace:*"
     "@storybook/core-common": "workspace:*"