diff --git a/spyder_kernels/console/kernel.py b/spyder_kernels/console/kernel.py
index da595371..925cf429 100644
--- a/spyder_kernels/console/kernel.py
+++ b/spyder_kernels/console/kernel.py
@@ -25,7 +25,7 @@
 # Third-party imports
 from ipykernel.ipkernel import IPythonKernel
 from ipykernel import get_connection_info
-from traitlets.config.loader import LazyConfigValue
+from traitlets.config.loader import Config, LazyConfigValue
 import zmq
 from zmq.utils.garbage import gc
 
@@ -554,67 +554,41 @@ def set_matplotlib_conf(self, conf):
         fontsize_n = 'pylab/inline/fontsize'
         bottom_n = 'pylab/inline/bottom'
         bbox_inches_n = 'pylab/inline/bbox_inches'
-        inline_backend = 'inline'
-
-        if pylab_autoload_n in conf or pylab_backend_n in conf:
-            self._set_mpl_backend(
-                conf.get(pylab_backend_n, inline_backend),
-                pylab=conf.get(pylab_autoload_n, False)
-            )
 
         if figure_format_n in conf:
-            self._set_config_option(
-                'InlineBackend.figure_format',
-                conf[figure_format_n]
+            self._set_inline_config_option(
+                'figure_formats', conf[figure_format_n]
             )
 
+        inline_rc = {}
         if resolution_n in conf:
-            self._set_mpl_inline_rc_config('figure.dpi', conf[resolution_n])
-
-        if width_n in conf and height_n in conf:
-            self._set_mpl_inline_rc_config(
-                'figure.figsize',
-                (conf[width_n], conf[height_n])
+            inline_rc.update({'figure.dpi': conf[resolution_n]})
+        if width_n in conf or height_n in conf:
+            inline_rc.update(
+                {'figure.figsize': (conf[width_n], conf[height_n])}
             )
-
         if fontsize_n in conf:
-            self._set_mpl_inline_rc_config('font.size', conf[fontsize_n])
-
+            inline_rc.update({'font.size': conf[fontsize_n]})
         if bottom_n in conf:
-            self._set_mpl_inline_rc_config(
-                'figure.subplot.bottom',
-                conf[bottom_n]
-            )
+            inline_rc.update({'figure.subplot.bottom': conf[bottom_n]})
 
-        if bbox_inches_n in conf:
-            self.set_mpl_inline_bbox_inches(conf[bbox_inches_n])
+        # Update Inline backend parameters, if available.
+        if inline_rc:
+            self._set_inline_config_option('rc', inline_rc)
 
+        if bbox_inches_n in conf:
+            bbox_inches = 'tight' if conf[bbox_inches_n] else None
+            self._set_inline_config_option(
+                'print_figure_kwargs', {'bbox_inches': bbox_inches}
+            )
 
-    def set_mpl_inline_bbox_inches(self, bbox_inches):
-        """
-        Set inline print figure bbox inches.
-
-        The change is done by updating the 'print_figure_kwargs' config dict.
-        """
-        config = self.config
-        inline_config = (
-            config['InlineBackend'] if 'InlineBackend' in config else {})
-        print_figure_kwargs = (
-            inline_config['print_figure_kwargs']
-            if 'print_figure_kwargs' in inline_config else {})
-        bbox_inches_dict = {
-            'bbox_inches': 'tight' if bbox_inches else None}
-        print_figure_kwargs.update(bbox_inches_dict)
-
-        # This seems to be necessary for newer versions of Traitlets because
-        # print_figure_kwargs doesn't return a dict.
-        if isinstance(print_figure_kwargs, LazyConfigValue):
-            figure_kwargs_dict = print_figure_kwargs.to_dict().get('update')
-            if figure_kwargs_dict:
-                print_figure_kwargs = figure_kwargs_dict
-
-        self._set_config_option(
-            'InlineBackend.print_figure_kwargs', print_figure_kwargs)
+        # Only update backend if it has changed or if autoloading pylab.
+        pylab_autoload_o = conf.get(pylab_autoload_n, False)
+        current_backend = self.get_matplotlib_backend()
+        pylab_backend_o = conf.get(pylab_backend_n, current_backend)
+        backend_changed = current_backend != pylab_backend_o
+        if pylab_autoload_o or backend_changed:
+            self._set_mpl_backend(pylab_backend_o, pylab_autoload_o)
 
     # -- For completions
     def set_jedi_completer(self, use_jedi):
@@ -901,9 +875,9 @@ def _set_mpl_backend(self, backend, pylab=False):
             return
 
         generic_error = (
-            "\n" + "="*73 + "\n"
+            "\n" + "=" * 73 + "\n"
             "NOTE: The following error appeared when setting "
-            "your Matplotlib backend!!\n" + "="*73 + "\n\n"
+            "your Matplotlib backend!!\n" + "=" * 73 + "\n\n"
             "{0}"
         )
 
@@ -925,7 +899,7 @@ def _set_mpl_backend(self, backend, pylab=False):
             # trying to set a backend. See issue 5541
             if "GUI eventloops" in str(err):
                 previous_backend = matplotlib.get_backend()
-                if not backend in previous_backend.lower():
+                if backend not in previous_backend.lower():
                     # Only inform about an error if the user selected backend
                     # and the one set by Matplotlib are different. Else this
                     # message is very confusing.
@@ -958,7 +932,7 @@ def _set_config_option(self, option, value):
         Set config options using the %config magic.
 
         As parameters:
-            option: config option, for example 'InlineBackend.figure_format'.
+            option: config option, for example 'InlineBackend.figure_formats'.
             value: value of the option, for example 'SVG', 'Retina', etc.
         """
         try:
@@ -972,16 +946,60 @@ def _set_config_option(self, option, value):
         except Exception:
             pass
 
-    def _set_mpl_inline_rc_config(self, option, value):
+    def _set_inline_config_option(self, option, value):
         """
-        Update any of the Matplolib rcParams given an option and value.
+        Update InlineBackend given an option and value.
+
+        Parameters
+        ----------
+        option: str
+            Configuration option. One of 'close_figures', 'figure_formats',
+            'print_figure_kwargs', or 'rc'.
+        value: str | dict
+            Value of the option.
         """
+        if (
+            'InlineBackend' in self.config
+            and option in self.config['InlineBackend']
+            and isinstance(value, dict)
+        ):
+            self.config['InlineBackend'][option].update(value)
+        elif 'InlineBackend' in self.config:
+            self.config['InlineBackend'].update({option: value})
+        else:
+            self.config.update({'InlineBackend': Config({option: value})})
+
+        value = self.config['InlineBackend'][option]
+
+        if isinstance(value, LazyConfigValue):
+            value = value.to_dict().get('update') or value
+
+        self._set_config_option(f'InlineBackend.{option}', value)
+
+        if option == 'rc' and self.get_matplotlib_backend() == 'inline':
+            # Explicitly update rcParams if already in inline mode so that
+            # new settings are effective immediately.
+            try:
+                import matplotlib
+                matplotlib.rcParams.update(value)
+            except Exception:
+                pass
+
+    def restore_rc_file_defaults(self):
+        """Restore inline rcParams to file defaults"""
         try:
-            from matplotlib import rcParams
-            rcParams[option] = value
+            import matplotlib
         except Exception:
-            # Needed in case matplolib isn't installed
-            pass
+            return
+
+        if (
+            'InlineBackend' in self.config
+            and 'rc' in self.config['InlineBackend']
+        ):
+            # Only restore keys that may have been set explicitly by
+            # _set_inline_config_option
+            for k in self.config['InlineBackend']['rc'].keys():
+                matplotlib.rcParams[k] = matplotlib.rcParamsOrig[k]
 
     def set_sympy_forecolor(self, background_color='dark'):
         """Set SymPy forecolor depending on console background."""
diff --git a/spyder_kernels/console/shell.py b/spyder_kernels/console/shell.py
index 5a9c1a29..d2d32a85 100644
--- a/spyder_kernels/console/shell.py
+++ b/spyder_kernels/console/shell.py
@@ -89,6 +89,10 @@ def enable_matplotlib(self, gui=None):
         if gui is None or gui.lower() == "auto":
             gui = automatic_backend()
 
+        # Before activating the backend, restore to file default those
+        # InlineBackend settings that may have been set explicitly.
+        self.kernel.restore_rc_file_defaults()
+
         enabled_gui, backend = super().enable_matplotlib(gui)
 
         # This is necessary for IPython 8.24+, which returns None after
diff --git a/spyder_kernels/console/start.py b/spyder_kernels/console/start.py
index eb910305..600b656c 100644
--- a/spyder_kernels/console/start.py
+++ b/spyder_kernels/console/start.py
@@ -93,28 +93,6 @@ def kernel_config():
             "del sys; del pdb"
         )
 
-    # Default inline backend configuration.
-    # This is useful to have when people doesn't
-    # use our config system to configure the
-    # inline backend but want to use
-    # '%matplotlib inline' at runtime
-    spy_cfg.InlineBackend.rc = {
-        # The typical default figure size is too large for inline use,
-        # so we shrink the figure size to 6x4, and tweak fonts to
-        # make that fit.
-        'figure.figsize': (6.0, 4.0),
-        # 72 dpi matches SVG/qtconsole.
-        # This only affects PNG export, as SVG has no dpi setting.
-        'figure.dpi': 72,
-        # 12pt labels get cutoff on 6x4 logplots, so use 10pt.
-        'font.size': 10,
-        # 10pt still needs a little more room on the xlabel
-        'figure.subplot.bottom': .125,
-        # Play nicely with any background color.
-        'figure.facecolor': 'white',
-        'figure.edgecolor': 'white'
-    }
-
     if is_module_installed('matplotlib'):
         spy_cfg.IPKernelApp.matplotlib = "inline"