diff --git a/nixos/doc/manual/configuration/profiles/minimal.section.md b/nixos/doc/manual/configuration/profiles/minimal.section.md
index 76d9585a0bd3c20..5b72112477f7110 100644
--- a/nixos/doc/manual/configuration/profiles/minimal.section.md
+++ b/nixos/doc/manual/configuration/profiles/minimal.section.md
@@ -1,8 +1,6 @@
 # Minimal {#sec-profile-minimal}
 
 This profile defines a small NixOS configuration. It does not contain any
-graphical stuff. It's a very short file that enables
-[noXlibs](#opt-environment.noXlibs), sets
-[](#opt-i18n.supportedLocales) to
-only support the user-selected locale,
-and [disables packages' documentation](#opt-documentation.enable).
+graphical stuff. It's a very short file that sets [](#opt-i18n.supportedLocales)
+to only support the user-selected locale, and
+[disables packages' documentation](#opt-documentation.enable).
diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md
index 4f773fccd23a14e..93c3f4cc18183ee 100644
--- a/nixos/doc/manual/release-notes/rl-2411.section.md
+++ b/nixos/doc/manual/release-notes/rl-2411.section.md
@@ -232,6 +232,9 @@
 
 - `pkgs.nextcloud27` has been removed since it's EOL.
 
+- The `environment.noXlibs` option has been removed. It was a common source of unexpected rebuilds and breakage that was often hard to diagnose.
+  If you need to disable certain libraries, you're encouraged to add your own overlay to your configuration that targets the packages you care about.
+
 - `frigate` was updated past 0.14.0. This release includes various breaking changes, so please go read the [release notes](https://github.com/blakeblackshear/frigate/releases/tag/v0.14.0).
    Most prominently access to the webinterface and API are now protected by authentication. Retrieve the auto-created
    admin account from the `frigate.service` journal after upgrading.
diff --git a/nixos/modules/config/no-x-libs.nix b/nixos/modules/config/no-x-libs.nix
deleted file mode 100644
index e3fd0b34ddf4b60..000000000000000
--- a/nixos/modules/config/no-x-libs.nix
+++ /dev/null
@@ -1,91 +0,0 @@
-# This module gets rid of all dependencies on X11 client libraries
-# (including fontconfig).
-{ config, lib, ... }:
-{
-  options = {
-    environment.noXlibs = lib.mkOption {
-      type = lib.types.bool;
-      default = false;
-      description = ''
-        Switch off the options in the default configuration that
-        require X11 libraries. This includes client-side font
-        configuration and SSH forwarding of X11 authentication
-        in. Thus, you probably do not want to enable this option if
-        you want to run X11 programs on this machine via SSH.
-      '';
-    };
-  };
-
-  config = lib.mkIf config.environment.noXlibs {
-    programs.ssh.setXAuthLocation = false;
-    security.pam.services.su.forwardXAuth = lib.mkForce false;
-
-    fonts.fontconfig.enable = false;
-
-    nixpkgs.overlays = lib.singleton (lib.const (super: {
-      beam = super.beam_nox;
-      cairo = super.cairo.override { x11Support = false; };
-      dbus = super.dbus.override { x11Support = false; };
-      fastfetch = super.fastfetch.override { vulkanSupport = false; waylandSupport = false; x11Support = false; };
-      ffmpeg = super.ffmpeg.override { ffmpegVariant = "headless"; };
-      ffmpeg_4 = super.ffmpeg_4.override { ffmpegVariant = "headless"; };
-      ffmpeg_6 = super.ffmpeg_6.override { ffmpegVariant = "headless"; };
-      ffmpeg_7 = super.ffmpeg_7.override { ffmpegVariant = "headless"; };
-      # dep of graphviz, libXpm is optional for Xpm support
-      gd = super.gd.override { withXorg = false; };
-      ghostscript = super.ghostscript.override { cupsSupport = false; x11Support = false; };
-      gjs = (super.gjs.override { installTests = false; }).overrideAttrs { doCheck = false; }; # avoid test dependency on gtk3
-      gobject-introspection = super.gobject-introspection.override { x11Support = false; };
-      gpg-tui = super.gpg-tui.override { x11Support = false; };
-      gpsd = super.gpsd.override { guiSupport = false; };
-      graphviz = super.graphviz-nox;
-      gst_all_1 = super.gst_all_1 // {
-        gst-plugins-bad = super.gst_all_1.gst-plugins-bad.override { guiSupport = false; };
-        gst-plugins-base = super.gst_all_1.gst-plugins-base.override { enableGl = false; enableWayland = false; enableX11 = false; };
-        gst-plugins-good = super.gst_all_1.gst-plugins-good.override { enableWayland = false; enableX11 = false; gtkSupport = false; qt5Support = false; qt6Support = false; };
-        gst-plugins-rs = super.gst_all_1.gst-plugins-rs.override { withGtkPlugins = false; };
-      };
-      imagemagick = super.imagemagick.override { libX11Support = false; libXtSupport = false; };
-      imagemagickBig = super.imagemagickBig.override { libX11Support = false; libXtSupport = false; };
-      intel-vaapi-driver = super.intel-vaapi-driver.override { enableGui = false; };
-      libdevil = super.libdevil-nox;
-      libextractor = super.libextractor.override { gtkSupport = false; };
-      libplacebo = super.libplacebo.override { vulkanSupport = false; };
-      libva = super.libva-minimal;
-      limesuite = super.limesuite.override { withGui = false; };
-      mc = super.mc.override { x11Support = false; };
-      mpv-unwrapped = super.mpv-unwrapped.override { drmSupport = false; screenSaverSupport = false; sdl2Support = false; vulkanSupport = false; waylandSupport = false; x11Support = false; };
-      msmtp = super.msmtp.override { withKeyring = false; };
-      mupdf = super.mupdf.override { enableGL = false; enableX11 = false; };
-      neofetch = super.neofetch.override { x11Support = false; };
-      networkmanager-fortisslvpn = super.networkmanager-fortisslvpn.override { withGnome = false; };
-      networkmanager-iodine = super.networkmanager-iodine.override { withGnome = false; };
-      networkmanager-l2tp = super.networkmanager-l2tp.override { withGnome = false; };
-      networkmanager-openconnect = super.networkmanager-openconnect.override { withGnome = false; };
-      networkmanager-openvpn = super.networkmanager-openvpn.override { withGnome = false; };
-      networkmanager-sstp = super.networkmanager-vpnc.override { withGnome = false; };
-      networkmanager-vpnc = super.networkmanager-vpnc.override { withGnome = false; };
-      pango = super.pango.override { x11Support = false; };
-      pinentry-curses = super.pinentry-curses.override { withLibsecret = false; };
-      pinentry-tty = super.pinentry-tty.override { withLibsecret = false; };
-      pipewire = super.pipewire.override { vulkanSupport = false; x11Support = false; };
-      pythonPackagesExtensions = super.pythonPackagesExtensions ++ [
-        (python-final: python-prev: {
-          # tk feature requires wayland which fails to compile
-          matplotlib = python-prev.matplotlib.override { enableTk = false; };
-        })
-      ];
-      qemu = super.qemu.override { gtkSupport = false; spiceSupport = false; sdlSupport = false; };
-      qrencode = super.qrencode.overrideAttrs (_: { doCheck = false; });
-      qt5 = super.qt5.overrideScope (lib.const (super': {
-        qtbase = super'.qtbase.override { withGtk3 = false; withQttranslation = false; };
-      }));
-      stoken = super.stoken.override { withGTK3 = false; };
-      # translateManpages -> perlPackages.po4a -> texlive-combined-basic -> texlive-core-big -> libX11
-      util-linux = super.util-linux.override { translateManpages = false; };
-      vim-full = super.vim-full.override { guiSupport = false; };
-      vte = super.vte.override { gtkVersion = null; };
-      zbar = super.zbar.override { enableVideo = false; withXorg = false; };
-    }));
-  };
-}
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 66d82f2ab2d27cf..dc79565496171c1 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -20,7 +20,6 @@
   ./config/nix-channel.nix
   ./config/nix-flakes.nix
   ./config/nix-remote-build.nix
-  ./config/no-x-libs.nix
   ./config/nsswitch.nix
   ./config/power-management.nix
   ./config/pulseaudio.nix
diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix
index 6d2d1acb8e15bbd..51de1b59f9566d9 100644
--- a/nixos/modules/rename.nix
+++ b/nixos/modules/rename.nix
@@ -21,6 +21,10 @@ in
 
     # Completely removed modules
     (mkRemovedOptionModule [ "environment" "blcr" "enable" ] "The BLCR module has been removed")
+    (mkRemovedOptionModule [ "environment" "noXlibs" ] ''
+      The environment.noXlibs option was removed, as it often caused surprising breakages for new users.
+      If you need its functionality, you can apply similar overlays in your own config.
+    '')
     (mkRemovedOptionModule [ "fonts" "fontconfig" "penultimate" ] "The corresponding package has removed from nixpkgs.")
     (mkRemovedOptionModule [ "hardware" "brightnessctl" ] ''
       The brightnessctl module was removed because newer versions of
diff --git a/pkgs/by-name/he/hello/package.nix b/pkgs/by-name/he/hello/package.nix
index 487bc033a23fc5b..a2e68bcefd457bb 100644
--- a/pkgs/by-name/he/hello/package.nix
+++ b/pkgs/by-name/he/hello/package.nix
@@ -31,12 +31,6 @@ stdenv.mkDerivation (finalAttrs: {
 
   passthru.tests = {
     version = testers.testVersion { package = hello; };
-
-    invariant-under-noXlibs =
-      testers.testEqualDerivation
-        "hello must not be rebuilt when environment.noXlibs is set."
-        hello
-        (nixos { environment.noXlibs = true; }).pkgs.hello;
   };
 
   passthru.tests.run = callPackage ./test.nix { hello = finalAttrs.finalPackage; };