diff --git a/src/core/generalized_first_order.jl b/src/core/generalized_first_order.jl
index 891e4751c..a485c7c65 100644
--- a/src/core/generalized_first_order.jl
+++ b/src/core/generalized_first_order.jl
@@ -202,6 +202,10 @@ function SciMLBase.__init(
             linesearch_ad = alg.forward_ad === nothing ?
                             (alg.reverse_ad === nothing ? alg.jacobian_ad :
                              alg.reverse_ad) : alg.forward_ad
+            if linesearch_ad !== nothing && iip && !DI.check_inplace(linesearch_ad)
+                @warn "$(linesearch_ad) doesn't support in-place problems."
+                linesearch_ad = nothing
+            end
             linesearch_ad = get_concrete_forward_ad(
                 linesearch_ad, prob, False; check_forward_mode = false)
             linesearch_cache = init(
diff --git a/test/gpu/core_tests.jl b/test/gpu/core_tests.jl
index d6d38ee1a..75087aa30 100644
--- a/test/gpu/core_tests.jl
+++ b/test/gpu/core_tests.jl
@@ -12,13 +12,19 @@
 
         prob = NonlinearProblem(linear_f, u0)
 
-        SOLVERS = (NewtonRaphson(), LevenbergMarquardt(; linsolve = QRFactorization()),
-            LevenbergMarquardt(; linsolve = KrylovJL_GMRES()), PseudoTransient(),
-            Klement(), Broyden(; linesearch = LiFukushimaLineSearch()),
+        SOLVERS = (
+            NewtonRaphson(),
+            LevenbergMarquardt(; linsolve = QRFactorization()),
+            LevenbergMarquardt(; linsolve = KrylovJL_GMRES()),
+            PseudoTransient(),
+            Klement(),
+            Broyden(; linesearch = LiFukushimaLineSearch()),
             LimitedMemoryBroyden(; threshold = 2, linesearch = LiFukushimaLineSearch()),
-            DFSane(), TrustRegion(; linsolve = QRFactorization()),
+            DFSane(),
+            TrustRegion(; linsolve = QRFactorization()),
             TrustRegion(; linsolve = KrylovJL_GMRES(), concrete_jac = true),  # Needed if Zygote not loaded
-            nothing)
+            nothing
+        )
 
         @testset "[IIP] GPU Solvers" begin
             for alg in SOLVERS