diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8ddf4f4ff..9eb146557 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -14,8 +14,36 @@ steps:
         Pkg.Registry.update();
         # Install packages present in subdirectories
         dev_pks = Pkg.PackageSpec[];
-        for path in ("lib/SciMLJacobianOperators",)
-            push!(dev_pks, Pkg.PackageSpec(; path));
+        for path in ("lib/SciMLJacobianOperators", "lib/BracketingNonlinearSolve", "lib/NonlinearSolveBase", "lib/SimpleNonlinearSolve")
+          push!(dev_pks, Pkg.PackageSpec(; path));
+        end
+        Pkg.develop(dev_pks);
+        Pkg.instantiate();
+        Pkg.test(; coverage=true)'
+    agents:
+      queue: "juliagpu"
+      cuda: "*"
+    timeout_in_minutes: 60
+    # Don't run Buildkite if the commit message includes the text [skip tests]
+    if: build.message !~ /\[skip tests\]/
+
+  - label: "Julia 1 (SimpleNonlinearSolve)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1"
+      - JuliaCI/julia-coverage#v1:
+          codecov: true
+          dirs:
+            - src
+            - ext
+    command: |
+      julia --color=yes --code-coverage=user --depwarn=yes --project=lib/SimpleNonlinearSolve -e '
+        import Pkg;
+        Pkg.Registry.update();
+        # Install packages present in subdirectories
+        dev_pks = Pkg.PackageSpec[];
+        for path in ("lib/NonlinearSolveBase", "lib/BracketingNonlinearSolve")
+          push!(dev_pks, Pkg.PackageSpec(; path))
         end
         Pkg.develop(dev_pks);
         Pkg.instantiate();
diff --git a/.github/workflows/CI_BracketingNonlinearSolve.yml b/.github/workflows/CI_BracketingNonlinearSolve.yml
new file mode 100644
index 000000000..6d78e5612
--- /dev/null
+++ b/.github/workflows/CI_BracketingNonlinearSolve.yml
@@ -0,0 +1,70 @@
+name: CI (BracketingNonlinearSolve)
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "lib/BracketingNonlinearSolve/**"
+      - "lib/NonlinearSolveBase/**"
+      - ".github/workflows/CI_BracketingNonlinearSolve.yml"
+  push:
+    branches:
+      - master
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "min"
+          - "1"
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+      - uses: actions/cache@v4
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - name: "Install Dependencies and Run Tests"
+        run: |
+          import Pkg
+          Pkg.Registry.update()
+          # Install packages present in subdirectories
+          dev_pks = Pkg.PackageSpec[]
+          for path in ("lib/NonlinearSolveBase",)
+              push!(dev_pks, Pkg.PackageSpec(; path))
+          end
+          Pkg.develop(dev_pks)
+          Pkg.instantiate()
+          Pkg.test(; coverage=true)
+        shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/BracketingNonlinearSolve {0}
+      - uses: julia-actions/julia-processcoverage@v1
+        with:
+          directories: lib/BracketingNonlinearSolve/src,lib/BracketingNonlinearSolve/ext
+      - uses: codecov/codecov-action@v4
+        with:
+          file: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          fail_ci_if_error: true
diff --git a/.github/workflows/CI_NonlinearSolve.yml b/.github/workflows/CI_NonlinearSolve.yml
index a4c14820e..843a04b54 100644
--- a/.github/workflows/CI_NonlinearSolve.yml
+++ b/.github/workflows/CI_NonlinearSolve.yml
@@ -11,6 +11,9 @@ on:
       - "Project.toml"
       - ".github/workflows/CI_NonlinearSolve.yml"
       - "lib/SciMLNonlinearOperators/**"
+      - "lib/BracketingNonlinearSolve/**"
+      - "lib/NonlinearSolveBase/**"
+      - "lib/SimpleNonlinearSolve/**"
   push:
     branches:
       - master
@@ -60,7 +63,7 @@ jobs:
           Pkg.Registry.update()
           # Install packages present in subdirectories
           dev_pks = Pkg.PackageSpec[]
-          for path in ("lib/SciMLJacobianOperators",)
+          for path in ("lib/SciMLJacobianOperators", "lib/BracketingNonlinearSolve", "lib/NonlinearSolveBase", "lib/SimpleNonlinearSolve")
               push!(dev_pks, Pkg.PackageSpec(; path))
           end
           Pkg.develop(dev_pks)
diff --git a/.github/workflows/CI_NonlinearSolveBase.yml b/.github/workflows/CI_NonlinearSolveBase.yml
new file mode 100644
index 000000000..f3878acef
--- /dev/null
+++ b/.github/workflows/CI_NonlinearSolveBase.yml
@@ -0,0 +1,63 @@
+name: CI (NonlinearSolveBase)
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "lib/NonlinearSolveBase/**"
+      - ".github/workflows/CI_NonlinearSolveBase.yml"
+  push:
+    branches:
+      - master
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "min"
+          - "1"
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+      - uses: actions/cache@v4
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - name: "Install Dependencies and Run Tests"
+        run: |
+          import Pkg
+          Pkg.Registry.update()
+          Pkg.instantiate()
+          Pkg.test(; coverage=true)
+        shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/NonlinearSolveBase {0}
+      - uses: julia-actions/julia-processcoverage@v1
+        with:
+          directories: lib/NonlinearSolveBase/src,lib/NonlinearSolveBase/ext
+      - uses: codecov/codecov-action@v4
+        with:
+          file: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          fail_ci_if_error: true
diff --git a/.github/workflows/CI_SimpleNonlinearSolve.yml b/.github/workflows/CI_SimpleNonlinearSolve.yml
new file mode 100644
index 000000000..11c3ef7c2
--- /dev/null
+++ b/.github/workflows/CI_SimpleNonlinearSolve.yml
@@ -0,0 +1,77 @@
+name: CI (SimpleNonlinearSolve)
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "lib/SimpleNonlinearSolve/**"
+      - "lib/BracketingNonlinearSolve/**"
+      - "lib/NonlinearSolveBase/**"
+      - ".github/workflows/CI_SimpleNonlinearSolve.yml"
+  push:
+    branches:
+      - master
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "min"
+          - "1"
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+        group:
+          - core
+          - adjoint
+          - alloc_check
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+      - uses: actions/cache@v4
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - name: "Install Dependencies and Run Tests"
+        run: |
+          import Pkg
+          Pkg.Registry.update()
+          # Install packages present in subdirectories
+          dev_pks = Pkg.PackageSpec[]
+          for path in ("lib/NonlinearSolveBase", "lib/BracketingNonlinearSolve")
+              push!(dev_pks, Pkg.PackageSpec(; path))
+          end
+          Pkg.develop(dev_pks)
+          Pkg.instantiate()
+          Pkg.test(; coverage=true)
+        shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/SimpleNonlinearSolve {0}
+        env:
+          GROUP: ${{ matrix.group }}
+      - uses: julia-actions/julia-processcoverage@v1
+        with:
+          directories: lib/SimpleNonlinearSolve/src,lib/SimpleNonlinearSolve/ext
+      - uses: codecov/codecov-action@v4
+        with:
+          file: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          fail_ci_if_error: true
diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
index 73494545f..805a5fe4c 100644
--- a/.github/workflows/CompatHelper.yml
+++ b/.github/workflows/CompatHelper.yml
@@ -23,4 +23,10 @@ jobs:
       - name: CompatHelper.main()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: julia -e 'using CompatHelper; CompatHelper.main(;subdirs=["", "docs"])'
+        run: |
+          import CompatHelper
+          subdirs = ["", "docs"]
+          append!(subdirs, joinpath.(("lib",), filter(p -> isdir(joinpath("lib", p)), readdir("lib"))))
+          CompatHelper.main(; subdirs)
+        shell: julia --color=yes {0}
+        working-directory: "./"
diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml
index 008cd511e..9dc416799 100644
--- a/.github/workflows/Documentation.yml
+++ b/.github/workflows/Documentation.yml
@@ -15,14 +15,14 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1'
+          version: '1.10'
       - name: Install dependencies
         run: |
           import Pkg
           Pkg.Registry.update()
           # Install packages present in subdirectories
           dev_pks = Pkg.PackageSpec[]
-          for path in ("lib/SciMLJacobianOperators", ".")
+          for path in ("lib/SciMLJacobianOperators", ".", "lib/SimpleNonlinearSolve", "lib/NonlinearSolveBase", "lib/BracketingNonlinearSolve")
               push!(dev_pks, Pkg.PackageSpec(; path))
           end
           Pkg.develop(dev_pks)
diff --git a/.github/workflows/Downgrade.yml b/.github/workflows/Downgrade.yml
index 83001be29..172457df8 100644
--- a/.github/workflows/Downgrade.yml
+++ b/.github/workflows/Downgrade.yml
@@ -14,15 +14,42 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        version: ["1.10"]
+        version:
+          - "1.10"
+        group:
+          - Core
+          - Downstream
+          - Misc
+          - Wrappers
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
       - uses: julia-actions/julia-downgrade-compat@v1
+      - name: "Install Dependencies and Run Tests"
+        run: |
+          import Pkg
+          Pkg.Registry.update()
+          # Install packages present in subdirectories
+          dev_pks = Pkg.PackageSpec[]
+          for path in ("lib/SciMLJacobianOperators", "lib/BracketingNonlinearSolve", "lib/NonlinearSolveBase", "lib/SimpleNonlinearSolve")
+              push!(dev_pks, Pkg.PackageSpec(; path))
+          end
+          Pkg.develop(dev_pks)
+          Pkg.instantiate()
+          Pkg.test(; coverage=true)
+        shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
+        env:
+          GROUP: ${{ matrix.group }}
+      - uses: julia-actions/julia-processcoverage@v1
         with:
-          skip: Pkg,TOML
-      - uses: julia-actions/julia-buildpkg@v1
-      - uses: julia-actions/julia-runtest@v1
+          directories: src,ext,lib/SciMLJacobianOperators/src
+      - uses: codecov/codecov-action@v4
+        with:
+          file: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          fail_ci_if_error: true
diff --git a/.typos.toml b/.typos.toml
index d78350bfb..4bde74f7c 100644
--- a/.typos.toml
+++ b/.typos.toml
@@ -1,2 +1,3 @@
 [default.extend-words]
-SER = "SER"
\ No newline at end of file
+SER = "SER"
+fo = "fo"
diff --git a/Project.toml b/Project.toml
index 725aa5c9d..03e6519dd 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,24 +1,24 @@
 name = "NonlinearSolve"
 uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec"
 authors = ["SciML"]
-version = "3.15.2"
+version = "4.0.0"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
+CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
 ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
-FastBroadcast = "7034ab61-46d4-4ed7-9d0f-46aef9175898"
 FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
 FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02"
 LineSearch = "87fe0de2-c867-4266-b59a-2f0a94fc965b"
-LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
 MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb"
+NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
@@ -27,10 +27,8 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
 SciMLJacobianOperators = "19f34311-ddf3-4b8b-af20-060888a46c0e"
 SciMLOperators = "c0aeaf25-5076-4817-a8d5-81caf7dfa961"
-Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
 SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
-SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
 SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35"
 StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 SymbolicIndexingInterface = "2efcf032-c050-4f8e-a9bb-153293bab1f5"
@@ -41,11 +39,13 @@ BandedMatrices = "aae01518-5342-5314-be14-df237901396f"
 FastLevenbergMarquardt = "7a0df574-e128-4d35-8cbd-3d84502bf7ce"
 FixedPointAcceleration = "817d07cb-a79a-5c30-9a31-890123675176"
 LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891"
+LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
 MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9"
 NLSolvers = "337daf1e-9722-11e9-073e-8b9effe078ba"
 NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56"
 SIAMFANLEquations = "084e46ad-d928-497d-ad5e-07fa361a48c4"
 SpeedMapping = "f1835b91-879b-4a3f-a438-e4baacf14412"
+Sundials = "c3572dad-4567-51f8-b174-8c6c989267f4"
 
 [extensions]
 NonlinearSolveBandedMatricesExt = "BandedMatrices"
@@ -54,9 +54,10 @@ NonlinearSolveFixedPointAccelerationExt = "FixedPointAcceleration"
 NonlinearSolveLeastSquaresOptimExt = "LeastSquaresOptim"
 NonlinearSolveMINPACKExt = "MINPACK"
 NonlinearSolveNLSolversExt = "NLSolvers"
-NonlinearSolveNLsolveExt = "NLsolve"
+NonlinearSolveNLsolveExt = ["NLsolve", "LineSearches"]
 NonlinearSolveSIAMFANLEquationsExt = "SIAMFANLEquations"
 NonlinearSolveSpeedMappingExt = "SpeedMapping"
+NonlinearSolveSundialsExt = "Sundials"
 
 [compat]
 ADTypes = "1.9"
@@ -65,12 +66,12 @@ ArrayInterface = "7.16"
 BandedMatrices = "1.5"
 BenchmarkTools = "1.4"
 CUDA = "5.5"
+CommonSolve = "0.2.4"
 ConcreteStructs = "0.2.3"
 DiffEqBase = "6.155.3"
-DifferentiationInterface = "0.6.1"
-Enzyme = "0.13.2"
+DifferentiationInterface = "0.6.16"
+Enzyme = "0.13.11"
 ExplicitImports = "1.5"
-FastBroadcast = "0.3.5"
 FastClosures = "0.3.2"
 FastLevenbergMarquardt = "0.1"
 FiniteDiff = "2.24"
@@ -86,11 +87,11 @@ LinearAlgebra = "1.10"
 LinearSolve = "2.35"
 MINPACK = "1.2"
 MaybeInplace = "0.1.4"
-ModelingToolkit = "9.41.0"
 NLSolvers = "0.5"
 NLsolve = "4.5"
 NaNMath = "1"
 NonlinearProblemLibrary = "0.1.2"
+NonlinearSolveBase = "1"
 OrdinaryDiffEqTsit5 = "1.1.0"
 Pkg = "1.10"
 PrecompileTools = "1.2"
@@ -104,11 +105,10 @@ SIAMFANLEquations = "1.0.1"
 SciMLBase = "2.54.0"
 SciMLJacobianOperators = "0.1"
 SciMLOperators = "0.3.10"
-Setfield = "1.1.1"
-SimpleNonlinearSolve = "1.12.3"
+SimpleNonlinearSolve = "2"
 SparseArrays = "1.10"
 SparseConnectivityTracer = "0.6.5"
-SparseMatrixColorings = "0.4.2"
+SparseMatrixColorings = "0.4.5"
 SpeedMapping = "0.3"
 StableRNGs = "1"
 StaticArrays = "1.9"
@@ -132,8 +132,8 @@ FixedPointAcceleration = "817d07cb-a79a-5c30-9a31-890123675176"
 Hwloc = "0e44f5e4-bd66-52a0-8798-143a42290a1d"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891"
+LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255"
 MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9"
-ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
 NLSolvers = "337daf1e-9722-11e9-073e-8b9effe078ba"
 NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56"
 NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
@@ -143,6 +143,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
 SIAMFANLEquations = "084e46ad-d928-497d-ad5e-07fa361a48c4"
+SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5"
 SpeedMapping = "f1835b91-879b-4a3f-a438-e4baacf14412"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
@@ -151,4 +152,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "MINPACK", "ModelingToolkit", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"]
+test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "LineSearches", "MINPACK", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SparseConnectivityTracer", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"]
diff --git a/docs/Project.toml b/docs/Project.toml
index ab35d42ae..4e72b257d 100644
--- a/docs/Project.toml
+++ b/docs/Project.toml
@@ -3,6 +3,7 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 AlgebraicMultigrid = "2169fc97-5a83-5252-b627-83903c6c433c"
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
@@ -11,8 +12,8 @@ DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656"
 IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae"
-ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
 NonlinearSolve = "8913a72c-1f9b-4ce2-8d82-65094dcecaec"
+NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0"
 OrdinaryDiffEqTsit5 = "b1df2697-797e-41e3-8120-5422d3b24e4a"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -29,22 +30,22 @@ ADTypes = "1.9.0"
 AlgebraicMultigrid = "0.5, 0.6"
 ArrayInterface = "6, 7"
 BenchmarkTools = "1"
-DiffEqBase = "6.136"
-DifferentiationInterface = "0.6"
+BracketingNonlinearSolve = "1"
+DifferentiationInterface = "0.6.16"
 Documenter = "1"
 DocumenterCitations = "1"
 DocumenterInterLinks = "1.0.0"
 IncompleteLU = "0.2"
 InteractiveUtils = "<0.0.1, 1"
 LinearSolve = "2"
-ModelingToolkit = "8, 9"
-NonlinearSolve = "3"
+NonlinearSolve = "4"
+NonlinearSolveBase = "1"
 OrdinaryDiffEqTsit5 = "1.1.0"
 Plots = "1"
-Random = "<0.0.1, 1"
+Random = "1.10"
 SciMLBase = "2.4"
 SciMLJacobianOperators = "0.1"
-SimpleNonlinearSolve = "1"
+SimpleNonlinearSolve = "2"
 SparseConnectivityTracer = "0.6.5"
 StaticArrays = "1"
 SteadyStateDiffEq = "2"
diff --git a/docs/make.jl b/docs/make.jl
index eec25f4f7..bdc2b7fe9 100644
--- a/docs/make.jl
+++ b/docs/make.jl
@@ -1,7 +1,8 @@
 using Documenter, DocumenterCitations, DocumenterInterLinks
 using NonlinearSolve, SimpleNonlinearSolve, Sundials, SteadyStateDiffEq, SciMLBase,
-      DiffEqBase
+      BracketingNonlinearSolve, NonlinearSolveBase
 using SciMLJacobianOperators
+import DiffEqBase
 
 cp(joinpath(@__DIR__, "Manifest.toml"),
     joinpath(@__DIR__, "src/assets/Manifest.toml"), force = true)
@@ -20,8 +21,9 @@ interlinks = InterLinks(
 makedocs(;
     sitename = "NonlinearSolve.jl",
     authors = "Chris Rackauckas",
-    modules = [NonlinearSolve, SimpleNonlinearSolve, SteadyStateDiffEq,
-        Sundials, DiffEqBase, SciMLBase, SciMLJacobianOperators],
+    modules = [NonlinearSolve, SimpleNonlinearSolve, SteadyStateDiffEq, DiffEqBase,
+        Sundials, NonlinearSolveBase, SciMLBase, SciMLJacobianOperators,
+        BracketingNonlinearSolve],
     clean = true,
     doctest = false,
     linkcheck = true,
diff --git a/docs/pages.jl b/docs/pages.jl
index cfb2754c0..8da01762a 100644
--- a/docs/pages.jl
+++ b/docs/pages.jl
@@ -1,6 +1,7 @@
 # Put in a separate page so it can be used by SciMLDocs.jl
 
-pages = ["index.md",
+pages = [
+    "index.md",
     "Getting Started with Nonlinear Rootfinding in Julia" => "tutorials/getting_started.md",
     "Tutorials" => Any[
         "tutorials/code_optimization.md",
@@ -31,6 +32,7 @@ pages = ["index.md",
     "Native Functionalities" => Any[
         "native/solvers.md",
         "native/simplenonlinearsolve.md",
+        "native/bracketingnonlinearsolve.md",
         "native/steadystatediffeq.md",
         "native/descent.md",
         "native/globalization.md",
diff --git a/docs/src/basics/diagnostics_api.md b/docs/src/basics/diagnostics_api.md
index c8b207544..795348bd6 100644
--- a/docs/src/basics/diagnostics_api.md
+++ b/docs/src/basics/diagnostics_api.md
@@ -27,20 +27,16 @@ Note that you will have to restart Julia to disable the timer outputs once enabl
 ## Example Usage
 
 ```@example diagnostics_example
-using ModelingToolkit, NonlinearSolve
+using NonlinearSolve
 
-@variables x y z
-@parameters σ ρ β
+function nlfunc(resid, u0, p)
+    resid[1] = u0[1] * u0[1] - p
+    resid[2] = u0[2] * u0[2] - p
+    resid[3] = u0[3] * u0[3] - p
+    nothing
+end
 
-# Define a nonlinear system
-eqs = [0 ~ σ * (y - x), 0 ~ x * (ρ - z) - y, 0 ~ x * y - β * z]
-@mtkbuild ns = NonlinearSystem(eqs, [x, y, z], [σ, ρ, β])
-
-u0 = [x => 1.0, y => 0.0, z => 0.0]
-
-ps = [σ => 10.0 ρ => 26.0 β => 8 / 3]
-
-prob = NonlinearProblem(ns, u0, ps)
+prob = NonlinearProblem(nlfunc, [1.0, 3.0, 5.0], 2.0)
 
 solve(prob)
 ```
diff --git a/docs/src/basics/solve.md b/docs/src/basics/solve.md
index 8ceeaa5de..3bc7df056 100644
--- a/docs/src/basics/solve.md
+++ b/docs/src/basics/solve.md
@@ -1,7 +1,7 @@
 # [Common Solver Options (Solve Keyword Arguments)](@id solver_options)
 
 ```@docs
-solve(prob::SciMLBase.NonlinearProblem, args...; kwargs...)
+solve(::NonlinearProblem, args...; kwargs...)
 ```
 
 ## General Controls
@@ -21,7 +21,7 @@ solve(prob::SciMLBase.NonlinearProblem, args...; kwargs...)
     `real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)`.
   - `reltol::Number`: The relative tolerance. Defaults to
     `real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)`.
-  - `termination_condition`: Termination Condition from DiffEqBase. Defaults to
+  - `termination_condition`: Termination Condition from NonlinearSolveBase. Defaults to
     `AbsSafeBestTerminationMode()` for `NonlinearSolve.jl` and `AbsTerminateMode()` for
     `SimpleNonlinearSolve.jl`.
 
diff --git a/docs/src/basics/sparsity_detection.md b/docs/src/basics/sparsity_detection.md
index de924c398..b7782c437 100644
--- a/docs/src/basics/sparsity_detection.md
+++ b/docs/src/basics/sparsity_detection.md
@@ -76,22 +76,3 @@ prob = NonlinearProblem(
 
 Refer to the documentation of DifferentiationInterface.jl and SparseConnectivityTracer.jl
 for more information on sparsity detection algorithms.
-
-## Case III: Sparse AD Type is being Used
-
-!!! warning
-    
-    This is now deprecated. Please use the previous two cases instead.
-
-If you constructed a Nonlinear Solver with a sparse AD type, for example
-
-```julia
-NewtonRaphson(; autodiff = AutoSparse(AutoForwardDiff()))
-# OR
-TrustRegion(; autodiff = AutoSparse(AutoZygote()))
-```
-
-then NonlinearSolve will automatically perform matrix coloring and use sparse
-differentiation if none of `sparsity` or `jac_prototype` is provided. We default to using
-`TracerSparsityDetector()`. `Case I/II` take precedence for sparsity detection and we
-perform sparse AD based on those options if those are provided.
diff --git a/docs/src/basics/termination_condition.md b/docs/src/basics/termination_condition.md
index a87f157aa..4ea98b4a5 100644
--- a/docs/src/basics/termination_condition.md
+++ b/docs/src/basics/termination_condition.md
@@ -14,9 +14,6 @@ cache = init(du, u, AbsSafeBestTerminationMode(); abstol = 1e-9, reltol = 1e-9)
 If `abstol` and `reltol` are not supplied, then we choose a default based on the element
 types of `du` and `u`.
 
-We can query the `cache` using `DiffEqBase.get_termination_mode`, `DiffEqBase.get_abstol`
-and `DiffEqBase.get_reltol`.
-
 To test for termination simply call the `cache`:
 
 ```julia
@@ -26,46 +23,23 @@ terminated = cache(du, u, uprev)
 ### Absolute Tolerance
 
 ```@docs
-AbsTerminationMode
-AbsNormTerminationMode
-AbsSafeTerminationMode
-AbsSafeBestTerminationMode
+NonlinearSolveBase.AbsTerminationMode
+NonlinearSolveBase.AbsNormTerminationMode
+NonlinearSolveBase.AbsNormSafeTerminationMode
+NonlinearSolveBase.AbsNormSafeBestTerminationMode
 ```
 
 ### Relative Tolerance
 
 ```@docs
-RelTerminationMode
-RelNormTerminationMode
-RelSafeTerminationMode
-RelSafeBestTerminationMode
-```
-
-### Both Absolute and Relative Tolerance
-
-```@docs
-NormTerminationMode
-SteadyStateDiffEqTerminationMode
+NonlinearSolveBase.RelTerminationMode
+NonlinearSolveBase.RelNormTerminationMode
+NonlinearSolveBase.RelNormSafeTerminationMode
+NonlinearSolveBase.RelNormSafeBestTerminationMode
 ```
 
-The following was named to match an older version of SimpleNonlinearSolve. It is currently
-not used as a default anywhere.
-
-```@docs
-SimpleNonlinearSolveTerminationMode
-```
-
-### Return Codes (Deprecated)
-
-These are deprecated and will be removed in a future release. Use the
-`use_deprecated_retcodes = Val(false)` option to `SciMLBase.init` to use the new return
-`ReturnCode` versions.
+### Both Tolerances
 
 ```@docs
-DiffEqBase.NonlinearSafeTerminationReturnCode
-DiffEqBase.NonlinearSafeTerminationReturnCode.Success
-DiffEqBase.NonlinearSafeTerminationReturnCode.Default
-DiffEqBase.NonlinearSafeTerminationReturnCode.Failure
-DiffEqBase.NonlinearSafeTerminationReturnCode.PatienceTermination
-DiffEqBase.NonlinearSafeTerminationReturnCode.ProtectiveTermination
+NonlinearSolveBase.NormTerminationMode
 ```
diff --git a/docs/src/devdocs/algorithm_helpers.md b/docs/src/devdocs/algorithm_helpers.md
index 7b0f91a9f..c945b6003 100644
--- a/docs/src/devdocs/algorithm_helpers.md
+++ b/docs/src/devdocs/algorithm_helpers.md
@@ -60,9 +60,6 @@ NonlinearSolve.GenericTrustRegionScheme
 ## Miscellaneous
 
 ```@docs
-SimpleNonlinearSolve.__nextfloat_tdir
-SimpleNonlinearSolve.__prevfloat_tdir
-SimpleNonlinearSolve.__max_tdir
 NonlinearSolve.callback_into_cache!
 NonlinearSolve.concrete_jac
 ```
diff --git a/docs/src/native/bracketingnonlinearsolve.md b/docs/src/native/bracketingnonlinearsolve.md
new file mode 100644
index 000000000..2201378e1
--- /dev/null
+++ b/docs/src/native/bracketingnonlinearsolve.md
@@ -0,0 +1,21 @@
+# BracketingNonlinearSolve.jl
+
+These methods can be used independently of the rest of NonlinearSolve.jl
+
+```@index
+Pages = ["bracketingnonlinearsolve.md"]
+```
+
+## Interval Methods
+
+These methods are suited for interval (scalar) root-finding problems,
+i.e. [`IntervalNonlinearProblem`](@ref).
+
+```@docs
+ITP
+Alefeld
+Bisection
+Falsi
+Ridder
+Brent
+```
diff --git a/docs/src/native/simplenonlinearsolve.md b/docs/src/native/simplenonlinearsolve.md
index 0ff386898..777a81ef8 100644
--- a/docs/src/native/simplenonlinearsolve.md
+++ b/docs/src/native/simplenonlinearsolve.md
@@ -6,20 +6,6 @@ These methods can be used independently of the rest of NonlinearSolve.jl
 Pages = ["simplenonlinearsolve.md"]
 ```
 
-## Interval Methods
-
-These methods are suited for interval (scalar) root-finding problems,
-i.e. `IntervalNonlinearProblem`.
-
-```@docs
-ITP
-Alefeld
-Bisection
-Falsi
-Ridder
-Brent
-```
-
 ## General Methods
 
 These methods are suited for any general nonlinear root-finding problem, i.e.
@@ -54,6 +40,6 @@ Squares problems.
 [^1]: Needs [`StaticArrays.jl`](https://github.com/JuliaArrays/StaticArrays.jl) to be
     installed and loaded for the non-allocating version.
 [^2]: This method is non-allocating if the termination condition is set to either `nothing`
-    (default) or [`AbsNormTerminationMode`](@ref).
+    (default) or [`NonlinearSolveBase.AbsNormTerminationMode`](@ref).
 [^3]: Only the defaults are guaranteed to work inside kernels. We try to provide warnings
     if the used version is not non-allocating.
diff --git a/docs/src/native/solvers.md b/docs/src/native/solvers.md
index a5deca141..5653f1fab 100644
--- a/docs/src/native/solvers.md
+++ b/docs/src/native/solvers.md
@@ -22,18 +22,16 @@ documentation.
     preconditioners. For more information on specifying preconditioners for LinearSolve
     algorithms, consult the
     [LinearSolve.jl documentation](https://docs.sciml.ai/LinearSolve/stable/).
-  - `linesearch`: the line search algorithm to use. Defaults to [`NoLineSearch()`](@extref LineSearch.NoLineSearch),
-    which means that no line search is performed.  Algorithms from
-    [`LineSearches.jl`](https://github.com/JuliaNLSolvers/LineSearches.jl/) must be
-    wrapped in [`LineSearchesJL`](@ref) before being supplied. For a detailed documentation
-    refer to [Line Search Algorithms](@ref line-search).
-  - `autodiff`/`jacobian_ad`: etermines the backend used for the Jacobian. Note that this
+  - `linesearch`: the line search algorithm to use. Defaults to
+    [`NoLineSearch()`](@extref LineSearch.NoLineSearch), which means that no line search is
+    performed.
+  - `autodiff`: etermines the backend used for the Jacobian. Note that this
     argument is ignored if an analytical Jacobian is passed, as that will be used instead.
     Defaults to `nothing` which means that a default is selected according to the problem
     specification! Valid choices are types from ADTypes.jl.
-  - `forward_ad`/`vjp_autodiff`: similar to `autodiff`, but is used to compute Jacobian
+  - `vjp_autodiff`: similar to `autodiff`, but is used to compute Jacobian
     Vector Products. Ignored if the NonlinearFunction contains the `jvp` function.
-  - `reverse_ad`/`vjp_autodiff`: similar to `autodiff`, but is used to compute Vector
+  - `vjp_autodiff`: similar to `autodiff`, but is used to compute Vector
     Jacobian Products. Ignored if the NonlinearFunction contains the `vjp` function.
   - `concrete_jac`: whether to build a concrete Jacobian. If a Krylov-subspace method is
     used, then the Jacobian will not be constructed and instead direct Jacobian-Vector
diff --git a/docs/src/release_notes.md b/docs/src/release_notes.md
index 9d517267e..1dc3d9433 100644
--- a/docs/src/release_notes.md
+++ b/docs/src/release_notes.md
@@ -1,6 +1,27 @@
 # Release Notes
 
-## Breaking Changes in `NonlinearSolve.jl` v3
+## Oct '24
+
+### Breaking Changes in `NonlinearSolve.jl` v4
+
+  - See [common breaking changes](@ref common-breaking-changes-v4v2) below.
+
+### Breaking Changes in `SimpleNonlinearSolve.jl` v2
+
+  - See [common breaking changes](@ref common-breaking-changes-v4v2) below.
+
+### [Common Breaking Changes](@id common-breaking-changes-v4v2)
+
+  - Use of termination conditions from `DiffEqBase` has been removed. Use the termination
+    conditions from `NonlinearSolveBase` instead.
+  - If no autodiff is provided, we now choose from a list of autodiffs based on the packages
+    loaded. For example, if `Enzyme` is loaded, we will default to that (for reverse mode).
+    In general, we don't guarantee the exact autodiff selected if `autodiff` is not provided
+    (i.e. `nothing`).
+
+## Dec '23
+
+### Breaking Changes in `NonlinearSolve.jl` v3
 
   - `GeneralBroyden` and `GeneralKlement` have been renamed to `Broyden` and `Klement`
     respectively.
@@ -8,7 +29,7 @@
   - The old style of specifying autodiff with `chunksize`, `standardtag`, etc. has been
     deprecated in favor of directly specifying the autodiff type, like `AutoForwardDiff`.
 
-## Breaking Changes in `SimpleNonlinearSolve.jl` v1
+### Breaking Changes in `SimpleNonlinearSolve.jl` v1
 
   - Batched solvers have been removed in favor of `BatchedArrays.jl`. Stay tuned for detailed
     tutorials on how to use `BatchedArrays.jl` with `NonlinearSolve` & `SimpleNonlinearSolve`
diff --git a/docs/src/tutorials/modelingtoolkit.md b/docs/src/tutorials/modelingtoolkit.md
index 87b0a4405..e2016dde0 100644
--- a/docs/src/tutorials/modelingtoolkit.md
+++ b/docs/src/tutorials/modelingtoolkit.md
@@ -5,7 +5,7 @@ modeling system for the Julia SciML ecosystem. It adds a high-level interactive
 for the numerical solvers which can make it easy to symbolically modify and generate
 equations to be solved. The basic form of using ModelingToolkit looks as follows:
 
-```@example mtk
+```julia
 using ModelingToolkit, NonlinearSolve
 
 @variables x y z
@@ -30,20 +30,20 @@ sol = solve(prob, NewtonRaphson())
 As a symbolic system, ModelingToolkit can be used to represent the equations and derive new
 forms. For example, let's look at the equations:
 
-```@example mtk
+```julia
 equations(ns)
 ```
 
 We can ask it what the Jacobian of our system is via `calculate_jacobian`:
 
-```@example mtk
+```julia
 calculate_jacobian(ns)
 ```
 
 We can tell MTK to generate a computable form of this analytical Jacobian via `jac = true`
 to help the solver use efficient forms:
 
-```@example mtk
+```julia
 prob = NonlinearProblem(ns, u0, ps, jac = true)
 sol = solve(prob, NewtonRaphson())
 ```
@@ -54,7 +54,7 @@ One of the major reasons for using ModelingToolkit is to allow structural simpli
 the systems. It's very easy to write down a mathematical model that, in theory, could be
 solved more simply. Let's take a look at a quick system:
 
-```@example mtk
+```julia
 @variables u1 u2 u3 u4 u5
 eqs = [0 ~ u1 - sin(u5), 0 ~ u2 - cos(u1), 0 ~ u3 - hypot(u1, u2),
     0 ~ u4 - hypot(u2, u3), 0 ~ u5 - hypot(u4, u1)]
@@ -63,23 +63,23 @@ eqs = [0 ~ u1 - sin(u5), 0 ~ u2 - cos(u1), 0 ~ u3 - hypot(u1, u2),
 
 If we run structural simplification, we receive the following form:
 
-```@example mtk
+```julia
 sys = structural_simplify(sys)
 ```
 
-```@example mtk
+```julia
 equations(sys)
 ```
 
 How did it do this? Let's look at the `observed` to see the relationships that it found:
 
-```@example mtk
+```julia
 observed(sys)
 ```
 
 Using ModelingToolkit, we can build and solve the simplified system:
 
-```@example mtk
+```julia
 u0 = [u5 .=> 1.0]
 prob = NonlinearProblem(sys, u0)
 sol = solve(prob, NewtonRaphson())
@@ -87,23 +87,23 @@ sol = solve(prob, NewtonRaphson())
 
 We can then use symbolic indexing to retrieve any variable:
 
-```@example mtk
+```julia
 sol[u1]
 ```
 
-```@example mtk
+```julia
 sol[u2]
 ```
 
-```@example mtk
+```julia
 sol[u3]
 ```
 
-```@example mtk
+```julia
 sol[u4]
 ```
 
-```@example mtk
+```julia
 sol[u5]
 ```
 
diff --git a/ext/NonlinearSolveFastLevenbergMarquardtExt.jl b/ext/NonlinearSolveFastLevenbergMarquardtExt.jl
index 262f811a7..e772043b3 100644
--- a/ext/NonlinearSolveFastLevenbergMarquardtExt.jl
+++ b/ext/NonlinearSolveFastLevenbergMarquardtExt.jl
@@ -3,6 +3,7 @@ module NonlinearSolveFastLevenbergMarquardtExt
 using ArrayInterface: ArrayInterface
 using FastClosures: @closure
 using FastLevenbergMarquardt: FastLevenbergMarquardt
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, FastLevenbergMarquardtJL
 using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode
 using StaticArraysCore: SArray
@@ -33,8 +34,8 @@ function SciMLBase.__solve(prob::Union{NonlinearLeastSquaresProblem, NonlinearPr
     else
         @closure (du, u, p) -> fn(du, u)
     end
-    abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u))
-    reltol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, eltype(u))
+    abstol = get_tolerance(abstol, eltype(u))
+    reltol = get_tolerance(reltol, eltype(u))
 
     _jac_fn = NonlinearSolve.__construct_extension_jac(
         prob, alg, u, resid; alg.autodiff, can_handle_oop = Val(prob.u0 isa SArray))
diff --git a/ext/NonlinearSolveFixedPointAccelerationExt.jl b/ext/NonlinearSolveFixedPointAccelerationExt.jl
index 6e26e5351..2d36d7f56 100644
--- a/ext/NonlinearSolveFixedPointAccelerationExt.jl
+++ b/ext/NonlinearSolveFixedPointAccelerationExt.jl
@@ -1,5 +1,6 @@
 module NonlinearSolveFixedPointAccelerationExt
 
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, FixedPointAccelerationJL
 using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode
 using FixedPointAcceleration: FixedPointAcceleration, fixed_point
@@ -13,7 +14,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::FixedPointAccelerationJL
 
     f, u0, resid = NonlinearSolve.__construct_extension_f(
         prob; alias_u0, make_fixed_point = Val(true), force_oop = Val(true))
-    tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0))
+    tol = get_tolerance(abstol, eltype(u0))
 
     sol = fixed_point(f, u0; Algorithm = alg.algorithm, MaxIter = maxiters, MaxM = alg.m,
         ConvergenceMetricThreshold = tol, ExtrapolationPeriod = alg.extrapolation_period,
diff --git a/ext/NonlinearSolveLeastSquaresOptimExt.jl b/ext/NonlinearSolveLeastSquaresOptimExt.jl
index 6abe13a9c..20dac092d 100644
--- a/ext/NonlinearSolveLeastSquaresOptimExt.jl
+++ b/ext/NonlinearSolveLeastSquaresOptimExt.jl
@@ -2,6 +2,7 @@ module NonlinearSolveLeastSquaresOptimExt
 
 using ConcreteStructs: @concrete
 using LeastSquaresOptim: LeastSquaresOptim
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, LeastSquaresOptimJL, TraceMinimal
 using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode
 
@@ -42,8 +43,8 @@ function SciMLBase.__init(prob::Union{NonlinearLeastSquaresProblem, NonlinearPro
     NonlinearSolve.__test_termination_condition(termination_condition, :LeastSquaresOptim)
 
     f!, u, resid = NonlinearSolve.__construct_extension_f(prob; alias_u0)
-    abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u))
-    reltol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, eltype(u))
+    abstol = get_tolerance(abstol, eltype(u))
+    reltol = get_tolerance(reltol, eltype(u))
 
     if prob.f.jac === nothing && alg.autodiff isa Symbol
         lsoprob = LSO.LeastSquaresProblem(; x = u, f!, y = resid, alg.autodiff,
diff --git a/ext/NonlinearSolveMINPACKExt.jl b/ext/NonlinearSolveMINPACKExt.jl
index a7be409d4..88adf5753 100644
--- a/ext/NonlinearSolveMINPACKExt.jl
+++ b/ext/NonlinearSolveMINPACKExt.jl
@@ -1,6 +1,7 @@
 module NonlinearSolveMINPACKExt
 
 using MINPACK: MINPACK
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, CMINPACK
 using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode
 using FastClosures: @closure
@@ -19,15 +20,16 @@ function SciMLBase.__solve(
     method = ifelse(alg.method === :auto,
         ifelse(prob isa NonlinearLeastSquaresProblem, :lm, :hybr), alg.method)
 
-    show_trace = alg.show_trace || ShT
-    tracing = alg.tracing || StT
-    tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0))
+    show_trace = ShT
+    tracing = StT
+    tol = get_tolerance(abstol, eltype(u0))
 
     if alg.autodiff === missing && prob.f.jac === nothing
         original = MINPACK.fsolve(
             f!, u0, m; tol, show_trace, tracing, method, iterations = maxiters)
     else
-        _jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; alg.autodiff)
+        autodiff = alg.autodiff === missing ? nothing : alg.autodiff
+        _jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; autodiff)
         jac! = @closure (J, u) -> (_jac!(J, u); Cint(0))
         original = MINPACK.fsolve(
             f!, jac!, u0, m; tol, show_trace, tracing, method, iterations = maxiters)
diff --git a/ext/NonlinearSolveNLSolversExt.jl b/ext/NonlinearSolveNLSolversExt.jl
index e78dab947..a9f41c87c 100644
--- a/ext/NonlinearSolveNLSolversExt.jl
+++ b/ext/NonlinearSolveNLSolversExt.jl
@@ -6,6 +6,7 @@ using FiniteDiff: FiniteDiff
 using ForwardDiff: ForwardDiff
 using LinearAlgebra: norm
 using NLSolvers: NLSolvers, NEqOptions, NEqProblem
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, NLSolversJL
 using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode
 
@@ -14,8 +15,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::NLSolversJL, args...;
         alias_u0::Bool = false, termination_condition = nothing, kwargs...)
     NonlinearSolve.__test_termination_condition(termination_condition, :NLSolversJL)
 
-    abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(prob.u0))
-    reltol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, eltype(prob.u0))
+    abstol = get_tolerance(abstol, eltype(prob.u0))
+    reltol = get_tolerance(reltol, eltype(prob.u0))
 
     options = NEqOptions(; maxiter = maxiters, f_abstol = abstol, f_reltol = reltol)
 
diff --git a/ext/NonlinearSolveNLsolveExt.jl b/ext/NonlinearSolveNLsolveExt.jl
index 77ed4a56f..73d98c062 100644
--- a/ext/NonlinearSolveNLsolveExt.jl
+++ b/ext/NonlinearSolveNLsolveExt.jl
@@ -1,5 +1,7 @@
 module NonlinearSolveNLsolveExt
 
+using LineSearches: Static
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, NLsolveJL, TraceMinimal
 using NLsolve: NLsolve, OnceDifferentiable, nlsolve
 using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode
@@ -16,7 +18,8 @@ function SciMLBase.__solve(
     if prob.f.jac === nothing && alg.autodiff isa Symbol
         df = OnceDifferentiable(f!, u0, resid; alg.autodiff)
     else
-        jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; alg.autodiff)
+        autodiff = alg.autodiff isa Symbol ? nothing : alg.autodiff
+        jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; autodiff)
         if prob.f.jac_prototype === nothing
             J = similar(
                 u0, promote_type(eltype(u0), eltype(resid)), length(u0), length(resid))
@@ -26,14 +29,16 @@ function SciMLBase.__solve(
         df = OnceDifferentiable(f!, jac!, vec(u0), vec(resid), J)
     end
 
-    abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0))
-    show_trace = ShT || alg.show_trace
-    store_trace = StT || alg.store_trace
-    extended_trace = !(trace_level isa TraceMinimal) || alg.extended_trace
+    abstol = get_tolerance(abstol, eltype(u0))
+    show_trace = ShT
+    store_trace = StT
+    extended_trace = !(trace_level isa TraceMinimal)
+
+    linesearch = alg.linesearch === missing ? Static() : alg.linesearch
 
     original = nlsolve(df, vec(u0); ftol = abstol, iterations = maxiters, alg.method,
-        store_trace, extended_trace, alg.linesearch, alg.linsolve,
-        alg.factor, alg.autoscale, alg.m, alg.beta, show_trace)
+        store_trace, extended_trace, linesearch, alg.linsolve, alg.factor,
+        alg.autoscale, alg.m, alg.beta, show_trace)
 
     f!(vec(resid), original.zero)
     u = prob.u0 isa Number ? original.zero[1] : reshape(original.zero, size(prob.u0))
diff --git a/ext/NonlinearSolveSIAMFANLEquationsExt.jl b/ext/NonlinearSolveSIAMFANLEquationsExt.jl
index 6ec3e8393..2468064fb 100644
--- a/ext/NonlinearSolveSIAMFANLEquationsExt.jl
+++ b/ext/NonlinearSolveSIAMFANLEquationsExt.jl
@@ -1,6 +1,7 @@
 module NonlinearSolveSIAMFANLEquationsExt
 
 using FastClosures: @closure
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, SIAMFANLEquationsJL
 using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode
 using SIAMFANLEquations: SIAMFANLEquations, aasol, nsol, nsoli, nsolsc, ptcsol, ptcsoli,
@@ -40,8 +41,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SIAMFANLEquationsJL, arg
 
     (; method, delta, linsolve, m, beta) = alg
     T = eltype(prob.u0)
-    atol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, T)
-    rtol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, T)
+    atol = get_tolerance(abstol, T)
+    rtol = get_tolerance(reltol, T)
 
     if prob.u0 isa Number
         f = @closure u -> prob.f(u, prob.p)
@@ -90,10 +91,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SIAMFANLEquationsJL, arg
                         f, u, m, zeros(T, N, 2 * m + 4); atol, rtol, maxit = maxiters, beta)
                 end
             else
+                autodiff = alg.autodiff === missing ? nothing : alg.autodiff
                 FPS = prob.f.jac_prototype !== nothing ? zero(prob.f.jac_prototype) :
                       __zeros_like(u, N, N)
                 jac = NonlinearSolve.__construct_extension_jac(
-                    prob, alg, u, resid; alg.autodiff)
+                    prob, alg, u, resid; autodiff)
                 AJ! = @closure (J, u, x) -> jac(J, x)
                 if method == :newton
                     sol = nsol(f, u, FS, FPS, AJ!; sham = 1, atol,
diff --git a/ext/NonlinearSolveSpeedMappingExt.jl b/ext/NonlinearSolveSpeedMappingExt.jl
index 2813e3e58..ff9b4683b 100644
--- a/ext/NonlinearSolveSpeedMappingExt.jl
+++ b/ext/NonlinearSolveSpeedMappingExt.jl
@@ -1,5 +1,6 @@
 module NonlinearSolveSpeedMappingExt
 
+using NonlinearSolveBase: NonlinearSolveBase, get_tolerance
 using NonlinearSolve: NonlinearSolve, SpeedMappingJL
 using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode
 using SpeedMapping: speedmapping
@@ -12,9 +13,9 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SpeedMappingJL, args...;
 
     m!, u, resid = NonlinearSolve.__construct_extension_f(
         prob; alias_u0, make_fixed_point = Val(true))
-    tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u))
+    tol = get_tolerance(abstol, eltype(u))
 
-    time_limit = ifelse(maxtime === nothing, alg.time_limit, maxtime)
+    time_limit = ifelse(maxtime === nothing, 1000, maxtime)
 
     sol = speedmapping(u; m!, tol, Lp = Inf, maps_limit = maxiters, alg.orders,
         alg.check_obj, store_info, alg.σ_min, alg.stabilize, time_limit)
diff --git a/ext/NonlinearSolveSundialsExt.jl b/ext/NonlinearSolveSundialsExt.jl
new file mode 100644
index 000000000..edea3a49b
--- /dev/null
+++ b/ext/NonlinearSolveSundialsExt.jl
@@ -0,0 +1,16 @@
+module NonlinearSolveSundialsExt
+
+using NonlinearSolveBase: NonlinearSolveBase, nonlinearsolve_forwarddiff_solve,
+                          nonlinearsolve_dual_solution
+using NonlinearSolve: DualNonlinearProblem
+using SciMLBase: SciMLBase
+using Sundials: KINSOL
+
+function SciMLBase.__solve(prob::DualNonlinearProblem, alg::KINSOL, args...; kwargs...)
+    sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...)
+    dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+end
+
+end
diff --git a/lib/BracketingNonlinearSolve/LICENSE b/lib/BracketingNonlinearSolve/LICENSE
new file mode 100644
index 000000000..411ad533f
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Avik Pal <avikpal@mit.edu> and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/lib/BracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml
new file mode 100644
index 000000000..6fc241d7d
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/Project.toml
@@ -0,0 +1,42 @@
+name = "BracketingNonlinearSolve"
+uuid = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e"
+authors = ["Avik Pal <avikpal@mit.edu> and contributors"]
+version = "1.0.0"
+
+[deps]
+CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0"
+PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+
+[weakdeps]
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+
+[extensions]
+BracketingNonlinearSolveForwardDiffExt = "ForwardDiff"
+
+[compat]
+Aqua = "0.8.9"
+CommonSolve = "0.2.4"
+ConcreteStructs = "0.2.3"
+ExplicitImports = "1.10.1"
+ForwardDiff = "0.10.36"
+InteractiveUtils = "<0.0.1, 1"
+NonlinearSolveBase = "1"
+PrecompileTools = "1.2"
+SciMLBase = "2.50"
+Test = "1.10"
+TestItemRunner = "1"
+julia = "1.10"
+
+[extras]
+Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+
+[targets]
+test = ["Aqua", "ExplicitImports", "ForwardDiff", "InteractiveUtils", "Test", "TestItemRunner"]
diff --git a/lib/BracketingNonlinearSolve/README.md b/lib/BracketingNonlinearSolve/README.md
new file mode 100644
index 000000000..85839107e
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/README.md
@@ -0,0 +1,23 @@
+# BracketingNonlinearSolve.jl
+
+Fast implementations of interval root finding algorithms in Julia that satisfy the SciML
+common interface. BracketingNonlinearSolve.jl focuses on low-dependency implementations of
+very fast methods for very small and simple problems. For the full set of solvers, see
+[NonlinearSolve.jl](https://github.com/SciML/NonlinearSolve.jl), of which
+BracketingNonlinearSolve.jl is just one solver set.
+
+For information on using the package,
+[see the stable documentation](https://docs.sciml.ai/NonlinearSolve/stable/). Use the
+[in-development documentation](https://docs.sciml.ai/NonlinearSolve/dev/) for the version of
+the documentation which contains the unreleased features.
+
+## High Level Examples
+
+```julia
+using BracketingNonlinearSolve
+
+f(u, p) = u .* u .- 2.0
+u0 = (1.0, 2.0) # brackets
+probB = IntervalNonlinearProblem(f, u0)
+sol = solve(probB, ITP())
+```
diff --git a/lib/BracketingNonlinearSolve/ext/BracketingNonlinearSolveForwardDiffExt.jl b/lib/BracketingNonlinearSolve/ext/BracketingNonlinearSolveForwardDiffExt.jl
new file mode 100644
index 000000000..b41a88451
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/ext/BracketingNonlinearSolveForwardDiffExt.jl
@@ -0,0 +1,26 @@
+module BracketingNonlinearSolveForwardDiffExt
+
+using CommonSolve: CommonSolve
+using ForwardDiff: ForwardDiff, Dual
+using NonlinearSolveBase: nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution
+using SciMLBase: SciMLBase, IntervalNonlinearProblem
+
+using BracketingNonlinearSolve: Bisection, Brent, Alefeld, Falsi, ITP, Ridder
+
+for algT in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder)
+    @eval function CommonSolve.solve(
+            prob::IntervalNonlinearProblem{
+                uType, iip, <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
+            alg::$(algT),
+            args...;
+            kwargs...) where {uType, iip, T, V, P}
+        sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...)
+        dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p)
+        return SciMLBase.build_solution(
+            prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats,
+            sol.original, left = Dual{T, V, P}(sol.left, partials),
+            right = Dual{T, V, P}(sol.right, partials))
+    end
+end
+
+end
diff --git a/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl
new file mode 100644
index 000000000..62cf7a7b7
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl
@@ -0,0 +1,49 @@
+module BracketingNonlinearSolve
+
+using ConcreteStructs: @concrete
+
+using CommonSolve: CommonSolve, solve
+using NonlinearSolveBase: NonlinearSolveBase
+using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, IntervalNonlinearProblem, ReturnCode
+
+using PrecompileTools: @compile_workload, @setup_workload
+
+abstract type AbstractBracketingAlgorithm <: AbstractNonlinearAlgorithm end
+
+include("common.jl")
+
+include("alefeld.jl")
+include("bisection.jl")
+include("brent.jl")
+include("falsi.jl")
+include("itp.jl")
+include("ridder.jl")
+
+# Default Algorithm
+function CommonSolve.solve(prob::IntervalNonlinearProblem; kwargs...)
+    return CommonSolve.solve(prob, ITP(); kwargs...)
+end
+function CommonSolve.solve(prob::IntervalNonlinearProblem, nothing, args...; kwargs...)
+    return CommonSolve.solve(prob, ITP(), args...; kwargs...)
+end
+
+@setup_workload begin
+    for T in (Float32, Float64)
+        prob_brack = IntervalNonlinearProblem{false}(
+            (u, p) -> u^2 - p, T.((0.0, 2.0)), T(2))
+        algs = (Alefeld(), Bisection(), Brent(), Falsi(), ITP(), Ridder())
+
+        @compile_workload begin
+            for alg in algs
+                CommonSolve.solve(prob_brack, alg; abstol = 1e-6)
+            end
+        end
+    end
+end
+
+export IntervalNonlinearProblem
+export solve
+
+export Alefeld, Bisection, Brent, Falsi, ITP, Ridder
+
+end
diff --git a/lib/BracketingNonlinearSolve/src/alefeld.jl b/lib/BracketingNonlinearSolve/src/alefeld.jl
new file mode 100644
index 000000000..a669900dd
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/alefeld.jl
@@ -0,0 +1,135 @@
+"""
+    Alefeld()
+
+An implementation of algorithm 4.2 from [Alefeld](https://dl.acm.org/doi/10.1145/210089.210111).
+
+The paper brought up two new algorithms. Here choose to implement algorithm 4.2 rather than
+algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal procedure.
+"""
+struct Alefeld <: AbstractBracketingAlgorithm end
+
+function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...;
+        maxiters = 1000, abstol = nothing, kwargs...)
+    f = Base.Fix2(prob.f, prob.p)
+    a, b = prob.tspan
+    c = a - (b - a) / (f(b) - f(a)) * f(a)
+
+    fc = f(c)
+    if a == c || b == c
+        return SciMLBase.build_solution(
+            prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit, left = a, right = b)
+    end
+
+    if iszero(fc)
+        return SciMLBase.build_solution(
+            prob, alg, c, fc; retcode = ReturnCode.Success, left = a, right = b)
+    end
+
+    a, b, d = Impl.bracket(f, a, b, c)
+    e = zero(a)   # Set e as 0 before iteration to avoid a non-value f(e)
+
+    for i in 2:maxiters
+        # The first bracketing block
+        f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e)
+        if i == 2 || (f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄)
+            c = Impl.newton_quadratic(f, a, b, d, 2)
+        else
+            c = Impl.ipzero(f, a, b, d, e)
+            if (c - a) * (c - b) ≥ 0
+                c = Impl.newton_quadratic(f, a, b, d, 2)
+            end
+        end
+
+        ē, fc = d, f(c)
+        if a == c || b == c
+            return SciMLBase.build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                left = a, right = b)
+        end
+
+        if iszero(fc)
+            return SciMLBase.build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.Success, left = a, right = b)
+        end
+
+        ā, b̄, d̄ = Impl.bracket(f, a, b, c)
+
+        # The second bracketing block
+        f₁, f₂, f₃, f₄ = f(ā), f(b̄), f(d̄), f(ē)
+        if f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄
+            c = Impl.newton_quadratic(f, ā, b̄, d̄, 3)
+        else
+            c = Impl.ipzero(f, ā, b̄, d̄, ē)
+            if (c - ā) * (c - b̄) ≥ 0
+                c = Impl.newton_quadratic(f, ā, b̄, d̄, 3)
+            end
+        end
+        fc = f(c)
+
+        if ā == c || b̄ == c
+            return SciMLBase.build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                left = ā, right = b̄)
+        end
+
+        if iszero(fc)
+            return SciMLBase.build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄)
+        end
+
+        ā, b̄, d̄ = Impl.bracket(f, ā, b̄, c)
+
+        # The third bracketing block
+        u = ifelse(abs(f(ā)) < abs(f(b̄)), ā, b̄)
+        c = u - 2 * (b̄ - ā) / (f(b̄) - f(ā)) * f(u)
+        if (abs(c - u)) > 0.5 * (b̄ - ā)
+            c = 0.5 * (ā + b̄)
+        end
+        fc = f(c)
+
+        if ā == c || b̄ == c
+            return SciMLBase.build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                left = ā, right = b̄)
+        end
+
+        if iszero(fc)
+            return SciMLBase.build_solution(
+                prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄)
+        end
+
+        ā, b̄, d = Impl.bracket(f, ā, b̄, c)
+
+        # The last bracketing block
+        if b̄ - ā < 0.5 * (b - a)
+            a, b, e = ā, b̄, d̄
+        else
+            e = d
+            c = 0.5 * (ā + b̄)
+            fc = f(c)
+
+            if ā == c || b̄ == c
+                return SciMLBase.build_solution(
+                    prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit,
+                    left = ā, right = b̄)
+            end
+            if iszero(fc)
+                return SciMLBase.build_solution(
+                    prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄)
+            end
+            a, b, d = Impl.bracket(f, ā, b̄, c)
+        end
+    end
+
+    # Reassign the value a, b, and c
+    if b == c
+        b = d
+    elseif a == c
+        a = d
+    end
+    fc = f(c)
+
+    # Return solution when run out of max iteration
+    return SciMLBase.build_solution(
+        prob, alg, c, fc; retcode = ReturnCode.MaxIters, left = a, right = b)
+end
diff --git a/lib/BracketingNonlinearSolve/src/bisection.jl b/lib/BracketingNonlinearSolve/src/bisection.jl
new file mode 100644
index 000000000..e51416145
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/bisection.jl
@@ -0,0 +1,89 @@
+"""
+    Bisection(; exact_left = false, exact_right = false)
+
+A common bisection method.
+
+### Keyword Arguments
+
+  - `exact_left`: whether to enforce whether the left side of the interval must be exactly
+    zero for the returned result. Defaults to false.
+  - `exact_right`: whether to enforce whether the right side of the interval must be exactly
+    zero for the returned result. Defaults to false.
+
+!!! danger "Keyword Arguments"
+
+    Currently, the keyword arguments are not implemented.
+"""
+@kwdef struct Bisection <: AbstractBracketingAlgorithm
+    exact_left::Bool = false
+    exact_right::Bool = false
+end
+
+function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Bisection,
+        args...; maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...)
+    @assert !SciMLBase.isinplace(prob) "`Bisection` only supports out-of-place problems."
+
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.tspan
+    fl, fr = f(left), f(right)
+
+    abstol = NonlinearSolveBase.get_tolerance(
+        left, abstol, promote_type(eltype(left), eltype(right)))
+
+    if iszero(fl)
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
+    end
+
+    if iszero(fr)
+        return SciMLBase.build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
+    end
+
+    if sign(fl) == sign(fr)
+        verbose &&
+            @warn "The interval is not an enclosing interval, opposite signs at the \
+                   boundaries are required."
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
+    i = 1
+    while i ≤ maxiters
+        mid = (left + right) / 2
+
+        if mid == left || mid == right
+            return SciMLBase.build_solution(
+                prob, alg, left, fl; retcode = ReturnCode.FloatingPointLimit, left, right)
+        end
+
+        fm = f(mid)
+        if abs((right - left) / 2) < abstol
+            return SciMLBase.build_solution(
+                prob, alg, mid, fm; retcode = ReturnCode.Success, left, right)
+        end
+
+        if iszero(fm)
+            right = mid
+            break
+        end
+
+        if sign(fl) == sign(fm)
+            fl = fm
+            left = mid
+        else
+            fr = fm
+            right = mid
+        end
+
+        i += 1
+    end
+
+    sol, i, left, right, fl, fr = Impl.bisection(
+        left, right, fl, fr, f, abstol, maxiters - i, prob, alg)
+
+    sol !== nothing && return sol
+
+    return SciMLBase.build_solution(
+        prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
+end
diff --git a/lib/BracketingNonlinearSolve/src/brent.jl b/lib/BracketingNonlinearSolve/src/brent.jl
new file mode 100644
index 000000000..fb3740e98
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/brent.jl
@@ -0,0 +1,119 @@
+"""
+    Brent()
+
+Left non-allocating Brent method.
+"""
+struct Brent <: AbstractBracketingAlgorithm end
+
+function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Brent, args...;
+        maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...)
+    @assert !SciMLBase.isinplace(prob) "`Brent` only supports out-of-place problems."
+
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.tspan
+    fl, fr = f(left), f(right)
+    ϵ = eps(convert(typeof(fl), 1))
+
+    abstol = NonlinearSolveBase.get_tolerance(
+        left, abstol, promote_type(eltype(left), eltype(right)))
+
+    if iszero(fl)
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
+    end
+
+    if iszero(fr)
+        return SciMLBase.build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
+    end
+
+    if sign(fl) == sign(fr)
+        verbose &&
+            @warn "The interval is not an enclosing interval, opposite signs at the \
+                   boundaries are required."
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
+    if abs(fl) < abs(fr)
+        left, right = right, left
+        fl, fr = fr, fl
+    end
+
+    c = left
+    d = c
+    i = 1
+    cond = true
+
+    while i < maxiters
+        fc = f(c)
+
+        if fl != fc && fr != fc
+            # Inverse quadratic interpolation
+            s = left * fr * fc / ((fl - fr) * (fl - fc)) +
+                right * fl * fc / ((fr - fl) * (fr - fc)) +
+                c * fl * fr / ((fc - fl) * (fc - fr))
+        else
+            # Secant method
+            s = right - fr * (right - left) / (fr - fl)
+        end
+
+        if (s < min((3 * left + right) / 4, right) ||
+            s > max((3 * left + right) / 4, right)) ||
+           (cond && abs(s - right) ≥ abs(right - c) / 2) ||
+           (!cond && abs(s - right) ≥ abs(c - d) / 2) ||
+           (cond && abs(right - c) ≤ ϵ) ||
+           (!cond && abs(c - d) ≤ ϵ)
+            # Bisection method
+            s = (left + right) / 2
+            if s == left || s == right
+                return SciMLBase.build_solution(prob, alg, left, fl;
+                    retcode = ReturnCode.FloatingPointLimit, left, right)
+            end
+            cond = true
+        else
+            cond = false
+        end
+
+        fs = f(s)
+        if abs((right - left) / 2) < abstol
+            return SciMLBase.build_solution(
+                prob, alg, s, fs; retcode = ReturnCode.Success, left, right)
+        end
+
+        if iszero(fs)
+            if right < left
+                left = right
+                fl = fr
+            end
+            right = s
+            fr = fs
+            break
+        end
+
+        if fl * fs < 0
+            d, c, right = c, right, s
+            fr = fs
+        else
+            left = s
+            fl = fs
+        end
+
+        if abs(fl) < abs(fr)
+            d = c
+            c, right = right, left
+            left = c
+            fc, fr = fr, fl
+            fl = fc
+        end
+        i += 1
+    end
+
+    sol, i, left, right, fl, fr = Impl.bisection(
+        left, right, fl, fr, f, abstol, maxiters - i, prob, alg)
+
+    sol !== nothing && return sol
+
+    return SciMLBase.build_solution(
+        prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
+end
diff --git a/lib/BracketingNonlinearSolve/src/common.jl b/lib/BracketingNonlinearSolve/src/common.jl
new file mode 100644
index 000000000..65239ea63
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/common.jl
@@ -0,0 +1,91 @@
+module Impl
+
+using SciMLBase: SciMLBase, ReturnCode
+
+function bisection(left, right, fl, fr, f::F, abstol, maxiters, prob, alg) where {F}
+    i = 1
+    sol = nothing
+    while i ≤ maxiters
+        mid = (left + right) / 2
+
+        if mid == left || mid == right
+            sol = SciMLBase.build_solution(
+                prob, alg, left, fl; left, right, retcode = ReturnCode.FloatingPointLimit)
+            break
+        end
+
+        fm = f(mid)
+        if abs((right - left) / 2) < abstol
+            sol = SciMLBase.build_solution(
+                prob, alg, mid, fm; left, right, retcode = ReturnCode.Success)
+            break
+        end
+
+        if iszero(fm)
+            right = mid
+            fr = fm
+        else
+            left = mid
+            fl = fm
+        end
+
+        i += 1
+    end
+
+    return sol, i, left, right, fl, fr
+end
+
+prevfloat_tdir(x, x0, x1) = ifelse(x1 > x0, prevfloat(x), nextfloat(x))
+nextfloat_tdir(x, x0, x1) = ifelse(x1 > x0, nextfloat(x), prevfloat(x))
+max_tdir(a, b, x0, x1) = ifelse(x1 > x0, max(a, b), min(a, b))
+
+function bracket(f::F, a, b, c) where {F}
+    if iszero(f(c))
+        ā, b̄, d = a, b, c
+    else
+        if f(a) * f(c) < 0
+            ā, b̄, d = a, c, b
+        elseif f(b) * f(c) < 0
+            ā, b̄, d = c, b, a
+        end
+    end
+    return ā, b̄, d
+end
+
+function newton_quadratic(f::F, a, b, d, k) where {F}
+    A = ((f(d) - f(b)) / (d - b) - (f(b) - f(a)) / (b - a)) / (d - a)
+    B = (f(b) - f(a)) / (b - a)
+
+    if iszero(A)
+        return a - (1 / B) * f(a)
+    elseif A * f(a) > 0
+        rᵢ₋₁ = a
+    else
+        rᵢ₋₁ = b
+    end
+
+    for _ in 1:k
+        rᵢ = rᵢ₋₁ -
+             (f(a) + B * (rᵢ₋₁ - a) + A * (rᵢ₋₁ - a) * (rᵢ₋₁ - b)) /
+             (B + A * (2 * rᵢ₋₁ - a - b))
+        rᵢ₋₁ = rᵢ
+    end
+
+    return rᵢ₋₁
+end
+
+function ipzero(f::F, a, b, c, d) where {F}
+    Q₁₁ = (c - d) * f(c) / (f(d) - f(c))
+    Q₂₁ = (b - c) * f(b) / (f(c) - f(b))
+    Q₃₁ = (a - b) * f(a) / (f(b) - f(a))
+    D₂₁ = (b - c) * f(c) / (f(c) - f(b))
+    D₃₁ = (a - b) * f(b) / (f(b) - f(a))
+    Q₂₂ = (D₂₁ - Q₁₁) * f(b) / (f(d) - f(b))
+    Q₃₂ = (D₃₁ - Q₂₁) * f(a) / (f(c) - f(a))
+    D₃₂ = (D₃₁ - Q₂₁) * f(c) / (f(c) - f(a))
+    Q₃₃ = (D₃₂ - Q₂₂) * f(a) / (f(d) - f(a))
+
+    return a + Q₃₁ + Q₃₂ + Q₃₃
+end
+
+end
diff --git a/lib/BracketingNonlinearSolve/src/falsi.jl b/lib/BracketingNonlinearSolve/src/falsi.jl
new file mode 100644
index 000000000..f56155ef7
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/falsi.jl
@@ -0,0 +1,79 @@
+"""
+    Falsi()
+
+A non-allocating regula falsi method.
+"""
+struct Falsi <: AbstractBracketingAlgorithm end
+
+function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...;
+        maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...)
+    @assert !SciMLBase.isinplace(prob) "`False` only supports out-of-place problems."
+
+    f = Base.Fix2(prob.f, prob.p)
+    l, r = prob.tspan # don't reuse these variables
+    left, right = prob.tspan
+    fl, fr = f(left), f(right)
+
+    abstol = NonlinearSolveBase.get_tolerance(
+        left, abstol, promote_type(eltype(left), eltype(right)))
+
+    if iszero(fl)
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
+    end
+
+    if iszero(fr)
+        return SciMLBase.build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
+    end
+
+    if sign(fl) == sign(fr)
+        verbose &&
+            @warn "The interval is not an enclosing interval, opposite signs at the \
+                   boundaries are required."
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
+    i = 1
+    while i ≤ maxiters
+        if Impl.nextfloat_tdir(left, l, r) == right
+            return SciMLBase.build_solution(
+                prob, alg, left, fl; left, right, retcode = ReturnCode.FloatingPointLimit)
+        end
+
+        mid = (fr * left - fl * right) / (fr - fl)
+        for _ in 1:10
+            mid = Impl.max_tdir(left, Impl.prevfloat_tdir(mid, l, r), l, r)
+        end
+
+        (mid == left || mid == right) && break
+
+        fm = f(mid)
+        if abs((right - left) / 2) < abstol
+            return SciMLBase.build_solution(
+                prob, alg, mid, fm; left, right, retcode = ReturnCode.Success)
+        end
+
+        if abs(fm) < abstol
+            right = mid
+            break
+        end
+
+        if sign(fl) == sign(fm)
+            fl, left = fm, mid
+        else
+            fr, right = fm, mid
+        end
+
+        i += 1
+    end
+
+    sol, i, left, right, fl, fr = Impl.bisection(
+        left, right, fl, fr, f, abstol, maxiters - i, prob, alg)
+
+    sol !== nothing && return sol
+
+    return SciMLBase.build_solution(
+        prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
+end
diff --git a/lib/BracketingNonlinearSolve/src/itp.jl b/lib/BracketingNonlinearSolve/src/itp.jl
new file mode 100644
index 000000000..821047a5a
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/itp.jl
@@ -0,0 +1,149 @@
+"""
+    ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10)
+
+ITP (Interpolate Truncate & Project)
+
+Use the [ITP method](https://en.wikipedia.org/wiki/ITP_method) to find a root of a bracketed
+function, with a convergence rate between 1 and 1.62.
+
+This method was introduced in the paper "An Enhancement of the Bisection Method Average
+Performance Preserving Minmax Optimality" (https://doi.org/10.1145/3423597) by
+I. F. D. Oliveira and R. H. C. Takahashi.
+
+### Tuning Parameters
+
+The following keyword parameters are accepted.
+
+  - `n₀::Int = 10`, the 'slack'. Must not be negative. When n₀ = 0 the worst-case is
+    identical to that of bisection, but increasing n₀ provides greater opportunity for
+    superlinearity.
+  - `scaled_κ₁::Float64 = 0.2`. Must not be negative. The recommended value is `0.2`.
+    Lower values produce tighter asymptotic behaviour, while higher values improve the
+    steady-state behaviour when truncation is not helpful.
+  - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater
+    convergence rate, but also make the method more succeptable to worst-case performance.
+    In practice, κ₂=1, 2 seems to work well due to the computational simplicity, as κ₂ is
+    used as an exponent in the method.
+
+### Computation of κ₁
+
+In the current implementation, we compute κ₁ = scaled_κ₁·|Δx₀|^(1 - κ₂); this allows κ₁ to
+adapt to the length of the interval and keep the proposed steps proportional to Δx.
+
+### Worst Case Performance
+
+n½ + `n₀` iterations, where n½ is the number of iterations using bisection
+(n½ = ⌈log2(Δx)/2`tol`⌉).
+
+### Asymptotic Performance
+
+If `f` is twice differentiable and the root is simple, then with `n₀` > 0 the convergence
+rate is √`κ₂`.
+"""
+@concrete struct ITP <: AbstractBracketingAlgorithm
+    scaled_k1
+    k2
+    n0::Int
+end
+
+function ITP(; scaled_k1::Real = 0.2, k2::Real = 2, n0::Int = 10)
+    scaled_k1 < 0 && error("Hyper-parameter κ₁ should not be negative")
+    n0 < 0 && error("Hyper-parameter n₀ should not be negative")
+    if k2 < 1 || k2 > (1.5 + sqrt(5) / 2)
+        throw(ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where \
+                             ϕ ≈ 1.618... is the golden ratio"))
+    end
+    return ITP(scaled_k1, k2, n0)
+end
+
+function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::ITP, args...;
+        maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...)
+    @assert !SciMLBase.isinplace(prob) "`ITP` only supports out-of-place problems."
+
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.tspan
+    fl, fr = f(left), f(right)
+
+    abstol = NonlinearSolveBase.get_tolerance(
+        left, abstol, promote_type(eltype(left), eltype(right)))
+
+    if iszero(fl)
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
+    end
+
+    if iszero(fr)
+        return SciMLBase.build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
+    end
+
+    if sign(fl) == sign(fr)
+        verbose &&
+            @warn "The interval is not an enclosing interval, opposite signs at the \
+                   boundaries are required."
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
+    ϵ = abstol
+    k2 = alg.k2
+    k1 = alg.scaled_k1 * abs(right - left)^(1 - k2)
+    n0 = alg.n0
+    n_h = ceil(log2(abs(right - left) / (2 * ϵ)))
+    mid = (left + right) / 2
+    x_f = left + (right - left) * (fl / (fl - fr))
+    xt = left
+    xp = left
+    r = zero(left) # minmax radius
+    δ = zero(left) # truncation error
+    σ = 1.0
+    ϵ_s = ϵ * 2^(n_h + n0)
+
+    i = 1
+    while i ≤ maxiters
+        span = abs(right - left)
+        r = ϵ_s - (span / 2)
+        δ = k1 * span^k2
+
+        x_f = left + (right - left) * (fl / (fl - fr))  # Interpolation Step
+
+        diff = mid - x_f
+        σ = sign(diff)
+        xt = ifelse(δ ≤ diff, x_f + σ * δ, mid)  # Truncation Step
+
+        xp = ifelse(abs(xt - mid) ≤ r, xt, mid - σ * r)  # Projection Step
+
+        if abs((left - right) / 2) < ϵ
+            return SciMLBase.build_solution(
+                prob, alg, xt, f(xt); retcode = ReturnCode.Success, left, right)
+        end
+
+        # update
+        tmin, tmax = minmax(xt, xp)
+        xp ≥ tmax && (xp = prevfloat(tmax))
+        xp ≤ tmin && (xp = nextfloat(tmin))
+        yp = f(xp)
+        yps = yp * sign(fr)
+        T0 = zero(yps)
+        if yps > T0
+            right, fr = xp, yp
+        elseif yps < T0
+            left, fl = xp, yp
+        else
+            return SciMLBase.build_solution(
+                prob, alg, xp, yps; retcode = ReturnCode.Success, left, right)
+        end
+
+        i += 1
+        mid = (left + right) / 2
+        ϵ_s /= 2
+
+        if Impl.nextfloat_tdir(left, prob.tspan...) == right
+            return SciMLBase.build_solution(
+                prob, alg, right, fr; retcode = ReturnCode.FloatingPointLimit, left, right)
+        end
+    end
+
+    return SciMLBase.build_solution(
+        prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
+end
diff --git a/lib/BracketingNonlinearSolve/src/ridder.jl b/lib/BracketingNonlinearSolve/src/ridder.jl
new file mode 100644
index 000000000..d988c9dc5
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/src/ridder.jl
@@ -0,0 +1,91 @@
+"""
+    Ridder()
+
+A non-allocating ridder method.
+"""
+struct Ridder <: AbstractBracketingAlgorithm end
+
+function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...;
+        maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...)
+    @assert !SciMLBase.isinplace(prob) "`Ridder` only supports out-of-place problems."
+
+    f = Base.Fix2(prob.f, prob.p)
+    left, right = prob.tspan
+    fl, fr = f(left), f(right)
+
+    abstol = NonlinearSolveBase.get_tolerance(
+        left, abstol, promote_type(eltype(left), eltype(right)))
+
+    if iszero(fl)
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right)
+    end
+
+    if iszero(fr)
+        return SciMLBase.build_solution(
+            prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right)
+    end
+
+    if sign(fl) == sign(fr)
+        verbose &&
+            @warn "The interval is not an enclosing interval, opposite signs at the \
+                   boundaries are required."
+        return SciMLBase.build_solution(
+            prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right)
+    end
+
+    xo = oftype(left, Inf)
+    i = 1
+    while i ≤ maxiters
+        mid = (left + right) / 2
+
+        if mid == left || mid == right
+            return SciMLBase.build_solution(
+                prob, alg, left, fl; retcode = ReturnCode.FloatingPointLimit, left, right)
+        end
+
+        fm = f(mid)
+        s = sqrt(fm^2 - fl * fr)
+        if iszero(s)
+            return SciMLBase.build_solution(
+                prob, alg, left, fl; retcode = ReturnCode.Failure, left, right)
+        end
+
+        x = mid + (mid - left) * sign(fl - fm) * fm / s
+        fx = f(x)
+        xo = x
+        if abs((right - left) / 2) < abstol
+            return SciMLBase.build_solution(
+                prob, alg, mid, fm; retcode = ReturnCode.Success, left, right)
+        end
+
+        if iszero(fx)
+            right, fr = x, fx
+            break
+        end
+
+        if sign(fx) != sign(fm)
+            left = mid
+            fl = fm
+            right = x
+            fr = fx
+        elseif sign(fx) != sign(fl)
+            right = x
+            fr = fx
+        else
+            @assert sign(fx) != sign(fr)
+            left = x
+            fl = fx
+        end
+
+        i += 1
+    end
+
+    sol, i, left, right, fl, fr = Impl.bisection(
+        left, right, fl, fr, f, abstol, maxiters - i, prob, alg)
+
+    sol !== nothing && return sol
+
+    return SciMLBase.build_solution(
+        prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right)
+end
diff --git a/lib/BracketingNonlinearSolve/test/qa_tests.jl b/lib/BracketingNonlinearSolve/test/qa_tests.jl
new file mode 100644
index 000000000..c01c493f4
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/test/qa_tests.jl
@@ -0,0 +1,17 @@
+@testitem "Aqua" tags=[:core] begin
+    using Aqua, BracketingNonlinearSolve
+
+    Aqua.test_all(BracketingNonlinearSolve; piracies = false, ambiguities = false)
+    Aqua.test_piracies(BracketingNonlinearSolve; treat_as_own = [IntervalNonlinearProblem])
+    Aqua.test_ambiguities(BracketingNonlinearSolve; recursive = false)
+end
+
+@testitem "Explicit Imports" tags=[:core] begin
+    import ForwardDiff
+    using ExplicitImports, BracketingNonlinearSolve
+
+    @test check_no_implicit_imports(BracketingNonlinearSolve; skip = (Base, Core)) ===
+          nothing
+    @test check_no_stale_explicit_imports(BracketingNonlinearSolve) === nothing
+    @test check_all_qualified_accesses_via_owners(BracketingNonlinearSolve) === nothing
+end
diff --git a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl
new file mode 100644
index 000000000..6a490d6fa
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl
@@ -0,0 +1,94 @@
+@testsnippet RootfindingTestSnippet begin
+    quadratic_f(u, p) = u .* u .- p
+    quadratic_f!(du, u, p) = (du .= u .* u .- p)
+    quadratic_f2(u, p) = @. p[1] * u * u - p[2]
+end
+
+@testitem "Interval Nonlinear Problems" setup=[RootfindingTestSnippet] tags=[:core] begin
+    using ForwardDiff
+
+    @testset for alg in (Bisection(), Falsi(), Ridder(), Brent(), ITP(), Alefeld(), nothing)
+        tspan = (1.0, 20.0)
+
+        function g(p)
+            probN = IntervalNonlinearProblem{false}(quadratic_f, typeof(p).(tspan), p)
+            return solve(probN, alg; abstol = 1e-9).left
+        end
+
+        @testset for p in 1.1:0.1:100.0
+            @test g(p)≈sqrt(p) atol=1e-3 rtol=1e-3
+            @test ForwardDiff.derivative(g, p)≈1 / (2 * sqrt(p)) atol=1e-3 rtol=1e-3
+        end
+
+        t = (p) -> [sqrt(p[2] / p[1])]
+        p = [0.9, 50.0]
+
+        function g2(p)
+            probN = IntervalNonlinearProblem{false}(quadratic_f2, tspan, p)
+            sol = solve(probN, alg; abstol = 1e-9)
+            return [sol.u]
+        end
+
+        @test g2(p)≈[sqrt(p[2] / p[1])] atol=1e-3 rtol=1e-3
+        @test ForwardDiff.jacobian(g2, p)≈ForwardDiff.jacobian(t, p) atol=1e-3 rtol=1e-3
+
+        probB = IntervalNonlinearProblem{false}(quadratic_f, (1.0, 2.0), 2.0)
+        sol = solve(probB, alg; abstol = 1e-9)
+        @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
+
+        if !(alg isa Bisection || alg isa Falsi)
+            probB = IntervalNonlinearProblem{false}(quadratic_f, (sqrt(2.0), 10.0), 2.0)
+            sol = solve(probB, alg; abstol = 1e-9)
+            @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
+
+            probB = IntervalNonlinearProblem{false}(quadratic_f, (0.0, sqrt(2.0)), 2.0)
+            sol = solve(probB, alg; abstol = 1e-9)
+            @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3
+        end
+    end
+end
+
+@testitem "Tolerance Tests Interval Methods" setup=[RootfindingTestSnippet] tags=[:core] begin
+    prob = IntervalNonlinearProblem(quadratic_f, (1.0, 20.0), 2.0)
+    ϵ = eps(Float64) # least possible tol for all methods
+
+    @testset for alg in (Bisection(), Falsi(), ITP(), nothing)
+        @testset for abstol in [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7]
+            sol = solve(prob, alg; abstol)
+            result_tol = abs(sol.u - sqrt(2))
+            @test result_tol < abstol
+            # test that the solution is not calculated upto max precision
+            @test result_tol > ϵ
+        end
+    end
+
+    @testset for alg in (Ridder(), Brent())
+        # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it
+        # converges with max precision to the solution
+        @testset for abstol in [0.1]
+            sol = solve(prob, alg; abstol)
+            result_tol = abs(sol.u - sqrt(2))
+            @test result_tol < abstol
+            # test that the solution is not calculated upto max precision
+            @test result_tol > ϵ
+        end
+    end
+end
+
+@testitem "Flipped Signs and Reversed Tspan" setup=[RootfindingTestSnippet] tags=[:core] begin
+    @testset for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder(), nothing)
+        f1(u, p) = u * u - p
+        f2(u, p) = p - u * u
+
+        for p in 1:4
+            inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p)
+            inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p)
+            inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p)
+            inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p)
+            @test abs.(solve(inp1, alg).u) ≈ sqrt.(p)
+            @test abs.(solve(inp2, alg).u) ≈ sqrt.(p)
+            @test abs.(solve(inp3, alg).u) ≈ sqrt.(p)
+            @test abs.(solve(inp4, alg).u) ≈ sqrt.(p)
+        end
+    end
+end
diff --git a/lib/BracketingNonlinearSolve/test/runtests.jl b/lib/BracketingNonlinearSolve/test/runtests.jl
new file mode 100644
index 000000000..d2c6f80a4
--- /dev/null
+++ b/lib/BracketingNonlinearSolve/test/runtests.jl
@@ -0,0 +1,7 @@
+using TestItemRunner, InteractiveUtils, Test
+
+@info sprint(InteractiveUtils.versioninfo)
+
+@testset "BracketingNonlinearSolve.jl" begin
+    @run_package_tests
+end
diff --git a/lib/NonlinearSolveBase/LICENSE b/lib/NonlinearSolveBase/LICENSE
new file mode 100644
index 000000000..411ad533f
--- /dev/null
+++ b/lib/NonlinearSolveBase/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Avik Pal <avikpal@mit.edu> and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml
new file mode 100644
index 000000000..70c0f97d4
--- /dev/null
+++ b/lib/NonlinearSolveBase/Project.toml
@@ -0,0 +1,66 @@
+name = "NonlinearSolveBase"
+uuid = "be0214bd-f91f-a760-ac4e-3421ce2b2da0"
+authors = ["Avik Pal <avikpal@mit.edu> and contributors"]
+version = "1.0.0"
+
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
+CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
+Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
+FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
+FunctionProperties = "f62d2435-5019-4c03-9749-2d4c77af0cbc"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
+SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
+
+[weakdeps]
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[extensions]
+NonlinearSolveBaseDiffEqBaseExt = "DiffEqBase"
+NonlinearSolveBaseForwardDiffExt = "ForwardDiff"
+NonlinearSolveBaseSparseArraysExt = "SparseArrays"
+
+[compat]
+ADTypes = "1.9"
+Aqua = "0.8.7"
+ArrayInterface = "7.9"
+CommonSolve = "0.2.4"
+Compat = "4.15"
+ConcreteStructs = "0.2.3"
+DiffEqBase = "6.149"
+DifferentiationInterface = "0.6.16"
+EnzymeCore = "0.8"
+ExplicitImports = "1.10.1"
+FastClosures = "0.3"
+ForwardDiff = "0.10.36"
+FunctionProperties = "0.1.2"
+InteractiveUtils = "<0.0.1, 1"
+LinearAlgebra = "1.10"
+Markdown = "1.10"
+RecursiveArrayTools = "3"
+SciMLBase = "2.50"
+SparseArrays = "1.10"
+StaticArraysCore = "1.4"
+Test = "1.10"
+julia = "1.10"
+
+[extras]
+Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Aqua", "DiffEqBase", "ExplicitImports", "ForwardDiff", "InteractiveUtils", "SparseArrays", "Test"]
diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl
new file mode 100644
index 000000000..346a5ee55
--- /dev/null
+++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl
@@ -0,0 +1,16 @@
+module NonlinearSolveBaseDiffEqBaseExt
+
+using DiffEqBase: DiffEqBase
+using SciMLBase: remake
+
+using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem
+
+function DiffEqBase.get_concrete_problem(
+        prob::ImmutableNonlinearProblem, isadapt; kwargs...)
+    u0 = DiffEqBase.get_concrete_u0(prob, isadapt, nothing, kwargs)
+    u0 = DiffEqBase.promote_u0(u0, prob.p, nothing)
+    p = DiffEqBase.get_concrete_p(prob, kwargs)
+    return remake(prob; u0 = u0, p = p)
+end
+
+end
diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl
new file mode 100644
index 000000000..c4f1dc901
--- /dev/null
+++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl
@@ -0,0 +1,181 @@
+module NonlinearSolveBaseForwardDiffExt
+
+using ADTypes: ADTypes, AutoForwardDiff, AutoPolyesterForwardDiff
+using ArrayInterface: ArrayInterface
+using CommonSolve: solve
+using DifferentiationInterface: DifferentiationInterface
+using FastClosures: @closure
+using ForwardDiff: ForwardDiff, Dual
+using LinearAlgebra: mul!
+using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem,
+                 NonlinearProblem, NonlinearLeastSquaresProblem, remake
+
+using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, Utils
+
+const DI = DifferentiationInterface
+
+function NonlinearSolveBase.additional_incompatible_backend_check(
+        prob::AbstractNonlinearProblem, ::Union{AutoForwardDiff, AutoPolyesterForwardDiff})
+    return !ForwardDiff.can_dual(eltype(prob.u0))
+end
+
+Utils.value(::Type{Dual{T, V, N}}) where {T, V, N} = V
+Utils.value(x::Dual) = Utils.value(ForwardDiff.value(x))
+Utils.value(x::AbstractArray{<:Dual}) = Utils.value.(x)
+
+function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve(
+        prob::Union{IntervalNonlinearProblem, NonlinearProblem, ImmutableNonlinearProblem},
+        alg, args...; kwargs...)
+    p = Utils.value(prob.p)
+    if prob isa IntervalNonlinearProblem
+        tspan = Utils.value.(prob.tspan)
+        newprob = IntervalNonlinearProblem(prob.f, tspan, p; prob.kwargs...)
+    else
+        newprob = remake(prob; p, u0 = Utils.value(prob.u0))
+    end
+
+    sol = solve(newprob, alg, args...; kwargs...)
+
+    uu = sol.u
+    Jₚ = NonlinearSolveBase.nonlinearsolve_∂f_∂p(prob, prob.f, uu, p)
+    Jᵤ = NonlinearSolveBase.nonlinearsolve_∂f_∂u(prob, prob.f, uu, p)
+    z = -Jᵤ \ Jₚ
+    pp = prob.p
+    sumfun = ((z, p),) -> map(Base.Fix2(*, ForwardDiff.partials(p)), z)
+
+    if uu isa Number
+        partials = sum(sumfun, zip(z, pp))
+    elseif p isa Number
+        partials = sumfun((z, pp))
+    else
+        partials = sum(sumfun, zip(eachcol(z), pp))
+    end
+
+    return sol, partials
+end
+
+function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve(
+        prob::NonlinearLeastSquaresProblem, alg, args...; kwargs...)
+    p = Utils.value(prob.p)
+    newprob = remake(prob; p, u0 = Utils.value(prob.u0))
+    sol = solve(newprob, alg, args...; kwargs...)
+    uu = sol.u
+
+    # First check for custom `vjp` then custom `Jacobian` and if nothing is provided use
+    # nested autodiff as the last resort
+    if SciMLBase.has_vjp(prob.f)
+        if SciMLBase.isinplace(prob)
+            vjp_fn = @closure (du, u, p) -> begin
+                resid = Utils.safe_similar(du, length(sol.resid))
+                prob.f(resid, u, p)
+                prob.f.vjp(du, resid, u, p)
+                du .*= 2
+                return nothing
+            end
+        else
+            vjp_fn = @closure (u, p) -> begin
+                resid = prob.f(u, p)
+                return reshape(2 .* prob.f.vjp(resid, u, p), size(u))
+            end
+        end
+    elseif SciMLBase.has_jac(prob.f)
+        if SciMLBase.isinplace(prob)
+            vjp_fn = @closure (du, u, p) -> begin
+                J = Utils.safe_similar(du, length(sol.resid), length(u))
+                prob.f.jac(J, u, p)
+                resid = Utils.safe_similar(du, length(sol.resid))
+                prob.f(resid, u, p)
+                mul!(reshape(du, 1, :), vec(resid)', J, 2, false)
+                return nothing
+            end
+        else
+            vjp_fn = @closure (u, p) -> begin
+                return reshape(2 .* vec(prob.f(u, p))' * prob.f.jac(u, p), size(u))
+            end
+        end
+    else
+        # For small problems, nesting ForwardDiff is actually quite fast
+        autodiff = length(uu) + length(sol.resid) ≥ 50 ?
+                   NonlinearSolveBase.select_reverse_mode_autodiff(prob, nothing) :
+                   AutoForwardDiff()
+
+        if SciMLBase.isinplace(prob)
+            vjp_fn = @closure (du, u, p) -> begin
+                resid = Utils.safe_similar(du, length(sol.resid))
+                prob.f(resid, u, p)
+                # Using `Constant` lead to dual ordering issues
+                ff = @closure (du, u) -> prob.f(du, u, p)
+                resid2 = copy(resid)
+                DI.pullback!(ff, resid2, (du,), autodiff, u, (resid,))
+                @. du *= 2
+                return nothing
+            end
+        else
+            vjp_fn = @closure (u, p) -> begin
+                v = prob.f(u, p)
+                # Using `Constant` lead to dual ordering issues
+                ff = Base.Fix2(prob.f, p)
+                res = only(DI.pullback(ff, autodiff, u, (v,)))
+                ArrayInterface.can_setindex(res) || return 2 .* res
+                @. res *= 2
+                return res
+            end
+        end
+    end
+
+    Jₚ = NonlinearSolveBase.nonlinearsolve_∂f_∂p(prob, vjp_fn, uu, newprob.p)
+    Jᵤ = NonlinearSolveBase.nonlinearsolve_∂f_∂u(prob, vjp_fn, uu, newprob.p)
+    z = -Jᵤ \ Jₚ
+    pp = prob.p
+    sumfun = ((z, p),) -> map(Base.Fix2(*, ForwardDiff.partials(p)), z)
+
+    if uu isa Number
+        partials = sum(sumfun, zip(z, pp))
+    elseif p isa Number
+        partials = sumfun((z, pp))
+    else
+        partials = sum(sumfun, zip(eachcol(z), pp))
+    end
+
+    return sol, partials
+end
+
+function NonlinearSolveBase.nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F}
+    if SciMLBase.isinplace(prob)
+        f2 = @closure p -> begin
+            du = Utils.safe_similar(u, promote_type(eltype(u), eltype(p)))
+            f(du, u, p)
+            return du
+        end
+    else
+        f2 = Base.Fix1(f, u)
+    end
+    if p isa Number
+        return Utils.safe_reshape(ForwardDiff.derivative(f2, p), :, 1)
+    elseif u isa Number
+        return Utils.safe_reshape(ForwardDiff.gradient(f2, p), 1, :)
+    else
+        return ForwardDiff.jacobian(f2, p)
+    end
+end
+
+function NonlinearSolveBase.nonlinearsolve_∂f_∂u(prob, f::F, u, p) where {F}
+    if SciMLBase.isinplace(prob)
+        return ForwardDiff.jacobian(
+            @closure((du, u)->f(du, u, p)), Utils.safe_similar(u), u)
+    end
+    u isa Number && return ForwardDiff.derivative(Base.Fix2(f, p), u)
+    return ForwardDiff.jacobian(Base.Fix2(f, p), u)
+end
+
+function NonlinearSolveBase.nonlinearsolve_dual_solution(u::Number, partials,
+        ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P}
+    return Dual{T, V, P}(u, partials)
+end
+
+function NonlinearSolveBase.nonlinearsolve_dual_solution(u::AbstractArray, partials,
+        ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P}
+    return map(((uᵢ, pᵢ),) -> Dual{T, V, P}(uᵢ, pᵢ), zip(u, Utils.restructure(u, partials)))
+end
+
+end
diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl
new file mode 100644
index 000000000..be13ebb8f
--- /dev/null
+++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl
@@ -0,0 +1,10 @@
+module NonlinearSolveBaseSparseArraysExt
+
+using NonlinearSolveBase: NonlinearSolveBase
+using SparseArrays: AbstractSparseMatrixCSC, nonzeros
+
+function NonlinearSolveBase.NAN_CHECK(x::AbstractSparseMatrixCSC)
+    return any(NonlinearSolveBase.NAN_CHECK, nonzeros(x))
+end
+
+end
diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl
new file mode 100644
index 000000000..b07b4b168
--- /dev/null
+++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl
@@ -0,0 +1,44 @@
+module NonlinearSolveBase
+
+using ADTypes: ADTypes, AbstractADType
+using ArrayInterface: ArrayInterface
+using CommonSolve: CommonSolve
+using Compat: @compat
+using ConcreteStructs: @concrete
+using DifferentiationInterface: DifferentiationInterface
+using EnzymeCore: EnzymeCore
+using FastClosures: @closure
+using FunctionProperties: hasbranching
+using LinearAlgebra: norm
+using Markdown: @doc_str
+using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition
+using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator, AbstractNonlinearProblem,
+                 NonlinearProblem, NonlinearLeastSquaresProblem, AbstractNonlinearFunction,
+                 @add_kwonly, StandardNonlinearProblem, NullParameters, isinplace,
+                 warn_paramtype
+using StaticArraysCore: StaticArray
+
+const DI = DifferentiationInterface
+
+include("public.jl")
+include("utils.jl")
+
+include("immutable_problem.jl")
+include("common_defaults.jl")
+include("termination_conditions.jl")
+
+include("autodiff.jl")
+
+# Unexported Public API
+@compat(public, (L2_NORM, Linf_NORM, NAN_CHECK, UNITLESS_ABS2, get_tolerance))
+@compat(public, (nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution))
+@compat(public,
+    (select_forward_mode_autodiff, select_reverse_mode_autodiff,
+        select_jacobian_autodiff))
+
+export RelTerminationMode, AbsTerminationMode,
+       NormTerminationMode, RelNormTerminationMode, AbsNormTerminationMode,
+       RelNormSafeTerminationMode, AbsNormSafeTerminationMode,
+       RelNormSafeBestTerminationMode, AbsNormSafeBestTerminationMode
+
+end
diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl
new file mode 100644
index 000000000..395580924
--- /dev/null
+++ b/lib/NonlinearSolveBase/src/autodiff.jl
@@ -0,0 +1,130 @@
+# Here we determine the preferred AD backend. We have a predefined list of ADs and then
+# we select the first one that is available and would work with the problem.
+
+# Ordering is important here. We want to select the first one that is compatible with the
+# problem.
+# XXX: Remove this once Enzyme is properly supported on Julia 1.11+
+@static if VERSION ≥ v"1.11-"
+    const ReverseADs = (
+        ADTypes.AutoZygote(),
+        ADTypes.AutoTracker(),
+        ADTypes.AutoReverseDiff(; compile = true),
+        ADTypes.AutoReverseDiff(),
+        ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse),
+        ADTypes.AutoFiniteDiff()
+    )
+else
+    const ReverseADs = (
+        ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse),
+        ADTypes.AutoZygote(),
+        ADTypes.AutoTracker(),
+        ADTypes.AutoReverseDiff(; compile = true),
+        ADTypes.AutoReverseDiff(),
+        ADTypes.AutoFiniteDiff()
+    )
+end
+
+const ForwardADs = (
+    ADTypes.AutoPolyesterForwardDiff(),
+    ADTypes.AutoForwardDiff(),
+    ADTypes.AutoEnzyme(; mode = EnzymeCore.Forward),
+    ADTypes.AutoFiniteDiff()
+)
+
+function select_forward_mode_autodiff(
+        prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true)
+    if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ForwardMode) &&
+       !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) &&
+       !is_finite_differences_backend(ad)
+        @warn "The chosen AD backend $(ad) is not a forward mode AD. Use with caution."
+    end
+    if incompatible_backend_and_problem(prob, ad)
+        adₙ = select_forward_mode_autodiff(prob, nothing; warn_check_mode)
+        @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \
+               could be because the backend package for the chosen AD isn't loaded. After \
+               running autodiff selection detected `$(adₙ)` as a potential forward mode \
+               backend."
+        return adₙ
+    end
+    return ad
+end
+
+function select_forward_mode_autodiff(prob::AbstractNonlinearProblem, ::Nothing;
+        warn_check_mode::Bool = true)
+    idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ForwardADs)
+    idx !== nothing && return ForwardADs[idx]
+    throw(ArgumentError("No forward mode AD backend is compatible with the chosen problem. \
+                         This could be because no forward mode autodiff backend is loaded \
+                         or the loaded backends don't support the problem."))
+end
+
+function select_reverse_mode_autodiff(
+        prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true)
+    if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ReverseMode) &&
+       !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) &&
+       !is_finite_differences_backend(ad)
+        @warn "The chosen AD backend $(ad) is not a reverse mode AD. Use with caution."
+    end
+    if incompatible_backend_and_problem(prob, ad)
+        adₙ = select_reverse_mode_autodiff(prob, nothing; warn_check_mode)
+        @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \
+               could be because the backend package for the chosen AD isn't loaded. After \
+               running autodiff selection detected `$(adₙ)` as a potential reverse mode \
+               backend."
+        return adₙ
+    end
+    return ad
+end
+
+function select_reverse_mode_autodiff(prob::AbstractNonlinearProblem, ::Nothing;
+        warn_check_mode::Bool = true)
+    idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ReverseADs)
+    idx !== nothing && return ReverseADs[idx]
+    throw(ArgumentError("No reverse mode AD backend is compatible with the chosen problem. \
+                         This could be because no reverse mode autodiff backend is loaded \
+                         or the loaded backends don't support the problem."))
+end
+
+function select_jacobian_autodiff(prob::AbstractNonlinearProblem, ad::AbstractADType)
+    if incompatible_backend_and_problem(prob, ad)
+        adₙ = select_jacobian_autodiff(prob, nothing)
+        @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \
+               could be because the backend package for the chosen AD isn't loaded. After \
+               running autodiff selection detected `$(adₙ)` as a potential jacobian \
+               backend."
+        return adₙ
+    end
+    return ad
+end
+
+function select_jacobian_autodiff(prob::AbstractNonlinearProblem, ::Nothing)
+    idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ForwardADs)
+    idx !== nothing && !is_finite_differences_backend(ForwardADs[idx]) &&
+        return ForwardADs[idx]
+    idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ReverseADs)
+    idx !== nothing && return ReverseADs[idx]
+    throw(ArgumentError("No jacobian AD backend is compatible with the chosen problem. \
+                         This could be because no jacobian autodiff backend is loaded \
+                         or the loaded backends don't support the problem."))
+end
+
+function incompatible_backend_and_problem(
+        prob::AbstractNonlinearProblem, ad::AbstractADType)
+    !DI.check_available(ad) && return true
+    SciMLBase.isinplace(prob) && !DI.check_inplace(ad) && return true
+    return additional_incompatible_backend_check(prob, ad)
+end
+
+additional_incompatible_backend_check(::AbstractNonlinearProblem, ::AbstractADType) = false
+function additional_incompatible_backend_check(prob::AbstractNonlinearProblem,
+        ::ADTypes.AutoReverseDiff{true})
+    if SciMLBase.isinplace(prob)
+        fu = prob.f.resid_prototype === nothing ? zero(prob.u0) : prob.f.resid_prototype
+        return hasbranching(prob.f, fu, prob.u0, prob.p)
+    end
+    return hasbranching(prob.f, prob.u0, prob.p)
+end
+
+is_finite_differences_backend(ad::AbstractADType) = false
+is_finite_differences_backend(::ADTypes.AutoFiniteDiff) = true
+is_finite_differences_backend(::ADTypes.AutoFiniteDifferences) = true
diff --git a/lib/NonlinearSolveBase/src/common_defaults.jl b/lib/NonlinearSolveBase/src/common_defaults.jl
new file mode 100644
index 000000000..4518063a5
--- /dev/null
+++ b/lib/NonlinearSolveBase/src/common_defaults.jl
@@ -0,0 +1,47 @@
+UNITLESS_ABS2(x::Number) = abs2(x)
+function UNITLESS_ABS2(x::AbstractArray)
+    return mapreduce(UNITLESS_ABS2, Utils.abs2_and_sum, x; init = Utils.zero_init(x))
+end
+function UNITLESS_ABS2(x::Union{AbstractVectorOfArray, ArrayPartition})
+    return mapreduce(UNITLESS_ABS2, Utils.abs2_and_sum,
+        Utils.get_internal_array(x); init = Utils.zero_init(x))
+end
+
+NAN_CHECK(x::Number) = isnan(x)
+NAN_CHECK(x::Enum) = false
+NAN_CHECK(x::AbstractArray) = any(NAN_CHECK, x)
+function NAN_CHECK(x::Union{AbstractVectorOfArray, ArrayPartition})
+    return any(NAN_CHECK, Utils.get_internal_array(x))
+end
+
+L2_NORM(u::Union{AbstractFloat, Complex}) = @fastmath abs(u)
+L2_NORM(u::AbstractArray) = @fastmath sqrt(UNITLESS_ABS2(u))
+function L2_NORM(u::AbstractArray{<:Union{AbstractFloat, Complex}})
+    if Utils.fast_scalar_indexing(u)
+        x = zero(eltype(u))
+        @simd for i in eachindex(u)
+            @inbounds @fastmath x += abs2(u[i])
+        end
+        return @fastmath sqrt(real(x))
+    end
+    return @fastmath sqrt(UNITLESS_ABS2(u))
+end
+function L2_NORM(u::StaticArray{<:Union{AbstractFloat, Complex}})
+    return @fastmath sqrt(real(sum(abs2, u)))
+end
+L2_NORM(u) = norm(u, 2)
+
+Linf_NORM(u::Union{AbstractFloat, Complex}) = @fastmath abs(u)
+Linf_NORM(u) = maximum(abs, u)
+
+get_tolerance(η, ::Type{T}) where {T} = Utils.convert_real(T, η)
+function get_tolerance(::Nothing, ::Type{T}) where {T}
+    η = real(oneunit(T)) * (eps(real(one(T)))^(4 // 5))
+    return get_tolerance(η, T)
+end
+
+get_tolerance(_, η, ::Type{T}) where {T} = get_tolerance(η, T)
+function get_tolerance(::Union{StaticArray, Number}, ::Nothing, ::Type{T}) where {T}
+    # Rational numbers can throw an error if used inside GPU Kernels
+    return T(real(oneunit(T)) * (eps(real(one(T)))^(real(T)(0.8))))
+end
diff --git a/lib/NonlinearSolveBase/src/immutable_problem.jl b/lib/NonlinearSolveBase/src/immutable_problem.jl
new file mode 100644
index 000000000..03b44a9ec
--- /dev/null
+++ b/lib/NonlinearSolveBase/src/immutable_problem.jl
@@ -0,0 +1,57 @@
+struct ImmutableNonlinearProblem{uType, iip, P, F, K, PT} <:
+       AbstractNonlinearProblem{uType, iip}
+    f::F
+    u0::uType
+    p::P
+    problem_type::PT
+    kwargs::K
+
+    @add_kwonly function ImmutableNonlinearProblem{iip}(
+            f::AbstractNonlinearFunction{iip}, u0, p = NullParameters(),
+            problem_type = StandardNonlinearProblem(); kwargs...) where {iip}
+        if haskey(kwargs, :p)
+            error("`p` specified as a keyword argument `p = $(kwargs[:p])` to \
+                   `NonlinearProblem`. This is not supported.")
+        end
+        warn_paramtype(p)
+        return new{
+            typeof(u0), iip, typeof(p), typeof(f), typeof(kwargs), typeof(problem_type)}(
+            f, u0, p, problem_type, kwargs)
+    end
+
+    """
+    Define a steady state problem using the given function.
+    `isinplace` optionally sets whether the function is inplace or not.
+    This is determined automatically, but not inferred.
+    """
+    function ImmutableNonlinearProblem{iip}(
+            f, u0, p = NullParameters(); kwargs...) where {iip}
+        return ImmutableNonlinearProblem{iip}(NonlinearFunction{iip}(f), u0, p; kwargs...)
+    end
+end
+
+"""
+Define a nonlinear problem using an instance of
+[`AbstractNonlinearFunction`](@ref AbstractNonlinearFunction).
+"""
+function ImmutableNonlinearProblem(
+        f::AbstractNonlinearFunction, u0, p = NullParameters(); kwargs...)
+    return ImmutableNonlinearProblem{isinplace(f)}(f, u0, p; kwargs...)
+end
+
+function ImmutableNonlinearProblem(f, u0, p = NullParameters(); kwargs...)
+    return ImmutableNonlinearProblem(NonlinearFunction(f), u0, p; kwargs...)
+end
+
+"""
+Define a ImmutableNonlinearProblem problem from SteadyStateProblem
+"""
+function ImmutableNonlinearProblem(prob::AbstractNonlinearProblem)
+    return ImmutableNonlinearProblem{isinplace(prob)}(prob.f, prob.u0, prob.p)
+end
+
+function Base.convert(
+        ::Type{ImmutableNonlinearProblem}, prob::T) where {T <: NonlinearProblem}
+    return ImmutableNonlinearProblem{isinplace(prob)}(
+        prob.f, prob.u0, prob.p, prob.problem_type; prob.kwargs...)
+end
diff --git a/lib/NonlinearSolveBase/src/public.jl b/lib/NonlinearSolveBase/src/public.jl
new file mode 100644
index 000000000..b39aa26d2
--- /dev/null
+++ b/lib/NonlinearSolveBase/src/public.jl
@@ -0,0 +1,124 @@
+# forward declarations of public API
+function UNITLESS_ABS2 end
+function NAN_CHECK end
+function L2_NORM end
+function Linf_NORM end
+function get_tolerance end
+
+# Forward declarations of functions for forward mode AD
+function nonlinearsolve_forwarddiff_solve end
+function nonlinearsolve_dual_solution end
+function nonlinearsolve_∂f_∂p end
+function nonlinearsolve_∂f_∂u end
+
+# Nonlinear Solve Termination Conditions
+abstract type AbstractNonlinearTerminationMode end
+abstract type AbstractSafeNonlinearTerminationMode <: AbstractNonlinearTerminationMode end
+abstract type AbstractSafeBestNonlinearTerminationMode <:
+              AbstractSafeNonlinearTerminationMode end
+
+#! format: off
+const TERM_DOCS = Dict(
+    :Norm => doc"``\| \Delta u \| \leq reltol \times \| \Delta u + u \|`` or ``\| \Delta u \| \leq abstol``.",
+    :Rel => doc"``all \left(| \Delta u | \leq reltol \times | u | \right)``.",
+    :RelNorm => doc"``\| \Delta u \| \leq reltol \times \| \Delta u + u \|``.",
+    :Abs => doc"``all \left( | \Delta u | \leq abstol \right)``.",
+    :AbsNorm => doc"``\| \Delta u \| \leq abstol``."
+)
+
+const TERM_INTERNALNORM_DOCS = """
+where `internalnorm` is the norm to use for the termination condition. Special handling is
+done for `norm(_, 2)`, `norm`, `norm(_, Inf)`, and `maximum(abs, _)`."""
+#! format: on
+
+for name in (:Rel, :Abs)
+    struct_name = Symbol(name, :TerminationMode)
+    doctring = TERM_DOCS[name]
+
+    @eval begin
+        """
+            $($struct_name) <: AbstractNonlinearTerminationMode
+
+        Terminates if $($doctring).
+
+        ``\\Delta u`` denotes the increment computed by the nonlinear solver and ``u`` denotes the solution.
+        """
+        struct $(struct_name) <: AbstractNonlinearTerminationMode end
+    end
+end
+
+for name in (:Norm, :RelNorm, :AbsNorm)
+    struct_name = Symbol(name, :TerminationMode)
+    doctring = TERM_DOCS[name]
+
+    @eval begin
+        """
+            $($struct_name) <: AbstractNonlinearTerminationMode
+
+        Terminates if $($doctring).
+
+        ``\\Delta u`` denotes the increment computed by the inner nonlinear solver.
+
+        ## Constructor
+
+            $($struct_name)(internalnorm = nothing)
+
+        $($TERM_INTERNALNORM_DOCS).
+        """
+        struct $(struct_name){F} <: AbstractNonlinearTerminationMode
+            internalnorm::F
+
+            function $(struct_name)(internalnorm::F) where {F}
+                norm = Utils.standardize_norm(internalnorm)
+                return new{typeof(norm)}(norm)
+            end
+        end
+    end
+end
+
+for norm_type in (:RelNorm, :AbsNorm), safety in (:Safe, :SafeBest)
+    struct_name = Symbol(norm_type, safety, :TerminationMode)
+    supertype_name = Symbol(:Abstract, safety, :NonlinearTerminationMode)
+
+    doctring = safety == :Safe ?
+               "Essentially [`$(norm_type)TerminationMode`](@ref) + terminate if there \
+                has been no improvement for the last `patience_steps` + terminate if the \
+                solution blows up (diverges)." :
+               "Essentially [`$(norm_type)SafeTerminationMode`](@ref), but caches the best\
+                solution found so far."
+
+    @eval begin
+        """
+            $($struct_name) <: $($supertype_name)
+
+        $($doctring)
+
+        ## Constructor
+
+            $($struct_name)(internalnorm; protective_threshold = nothing,
+                patience_steps = 100, patience_objective_multiplier = 3,
+                min_max_factor = 1.3, max_stalled_steps = nothing)
+
+        $($TERM_INTERNALNORM_DOCS).
+        """
+        @concrete struct $(struct_name) <: $(supertype_name)
+            internalnorm
+            protective_threshold
+            patience_steps::Int
+            patience_objective_multiplier
+            min_max_factor
+            max_stalled_steps <: Union{Nothing, Int}
+
+            function $(struct_name)(internalnorm::F; protective_threshold = nothing,
+                    patience_steps = 100, patience_objective_multiplier = 3,
+                    min_max_factor = 1.3, max_stalled_steps = nothing) where {F}
+                norm = Utils.standardize_norm(internalnorm)
+                return new{typeof(norm), typeof(protective_threshold),
+                    typeof(patience_objective_multiplier),
+                    typeof(min_max_factor), typeof(max_stalled_steps)}(
+                    norm, protective_threshold, patience_steps,
+                    patience_objective_multiplier, min_max_factor, max_stalled_steps)
+            end
+        end
+    end
+end
diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl
new file mode 100644
index 000000000..9f20e46bc
--- /dev/null
+++ b/lib/NonlinearSolveBase/src/termination_conditions.jl
@@ -0,0 +1,283 @@
+const RelNormModes = Union{
+    RelNormTerminationMode, RelNormSafeTerminationMode, RelNormSafeBestTerminationMode}
+const AbsNormModes = Union{
+    AbsNormTerminationMode, AbsNormSafeTerminationMode, AbsNormSafeBestTerminationMode}
+
+# Core Implementation
+@concrete mutable struct NonlinearTerminationModeCache{uType, T}
+    u::uType
+    retcode::ReturnCode.T
+    abstol::T
+    reltol::T
+    best_objective_value::T
+    mode
+    initial_objective
+    objectives_trace
+    nsteps::Int
+    saved_values
+    u0_norm
+    step_norm_trace
+    max_stalled_steps
+    u_diff_cache::uType
+end
+
+function update_u!!(cache::NonlinearTerminationModeCache, u)
+    cache.u === nothing && return
+    if cache.u isa AbstractArray && ArrayInterface.can_setindex(cache.u)
+        copyto!(cache.u, u)
+    else
+        cache.u = u
+    end
+end
+
+function CommonSolve.init(
+        ::AbstractNonlinearProblem, mode::AbstractNonlinearTerminationMode, du, u,
+        saved_value_prototype...; abstol = nothing, reltol = nothing, kwargs...)
+    T = promote_type(eltype(du), eltype(u))
+    abstol = get_tolerance(u, abstol, T)
+    reltol = get_tolerance(u, reltol, T)
+    TT = typeof(abstol)
+
+    u_unaliased = mode isa AbstractSafeBestNonlinearTerminationMode ?
+                  (ArrayInterface.can_setindex(u) ? copy(u) : u) : nothing
+
+    if mode isa AbstractSafeNonlinearTerminationMode
+        if mode isa AbsNormSafeTerminationMode || mode isa AbsNormSafeBestTerminationMode
+            initial_objective = Linf_NORM(du)
+            u0_norm = nothing
+        else
+            initial_objective = Linf_NORM(du) /
+                                (Utils.nonallocating_maximum(+, du, u) + eps(TT))
+            u0_norm = mode.max_stalled_steps === nothing ? nothing : L2_NORM(u)
+        end
+        objectives_trace = Vector{TT}(undef, mode.patience_steps)
+        step_norm_trace = mode.max_stalled_steps === nothing ? nothing :
+                          Vector{TT}(undef, mode.max_stalled_steps)
+        if step_norm_trace !== nothing &&
+           ArrayInterface.can_setindex(u_unaliased) &&
+           !(u_unaliased isa Number)
+            u_diff_cache = similar(u_unaliased)
+        else
+            u_diff_cache = u_unaliased
+        end
+        best_value = initial_objective
+        max_stalled_steps = mode.max_stalled_steps
+    else
+        initial_objective = nothing
+        objectives_trace = nothing
+        u0_norm = nothing
+        step_norm_trace = nothing
+        best_value = Utils.convert_real(T, Inf)
+        max_stalled_steps = nothing
+        u_diff_cache = u_unaliased
+    end
+
+    length(saved_value_prototype) == 0 && (saved_value_prototype = nothing)
+
+    return NonlinearTerminationModeCache(
+        u_unaliased, ReturnCode.Default, abstol, reltol, best_value, mode,
+        initial_objective, objectives_trace, 0, saved_value_prototype,
+        u0_norm, step_norm_trace, max_stalled_steps, u_diff_cache)
+end
+
+function SciMLBase.reinit!(
+        cache::NonlinearTerminationModeCache, du, u, saved_value_prototype...;
+        abstol = cache.abstol, reltol = cache.reltol, kwargs...)
+    T = eltype(cache.abstol)
+    length(saved_value_prototype) != 0 && (cache.saved_values = saved_value_prototype)
+
+    mode = cache.mode
+    u_unaliased = mode isa AbstractSafeBestNonlinearTerminationMode ?
+                  (ArrayInterface.can_setindex(u) ? copy(u) : u) : nothing
+    cache.u = u_unaliased
+    cache.retcode = ReturnCode.Default
+
+    cache.abstol = get_tolerance(u, abstol, T)
+    cache.reltol = get_tolerance(u, reltol, T)
+    cache.nsteps = 0
+    TT = typeof(cache.abstol)
+
+    if mode isa AbstractSafeNonlinearTerminationMode
+        if mode isa AbsNormSafeTerminationMode || mode isa AbsNormSafeBestTerminationMode
+            cache.initial_objective = Linf_NORM(du)
+        else
+            cache.initial_objective = Linf_NORM(du) /
+                                      (Utils.nonallocating_maximum(+, du, u) + eps(TT))
+            cache.max_stalled_steps !== nothing && (cache.u0_norm = L2_NORM(u))
+        end
+        cache.best_objective_value = cache.initial_objective
+    else
+        cache.best_objective_value = Utils.convert_real(T, Inf)
+    end
+end
+
+## This dispatch is needed based on how Terminating Callback works!
+function (cache::NonlinearTerminationModeCache)(
+        integrator::AbstractODEIntegrator, abstol::Number, reltol::Number, min_t)
+    if min_t === nothing || integrator.t ≥ min_t
+        return cache(cache.mode, SciMLBase.get_du(integrator),
+            integrator.u, integrator.uprev, abstol, reltol)
+    end
+    return false
+end
+function (cache::NonlinearTerminationModeCache)(du, u, uprev, args...)
+    return cache(cache.mode, du, u, uprev, cache.abstol, cache.reltol, args...)
+end
+
+function (cache::NonlinearTerminationModeCache)(
+        mode::AbstractNonlinearTerminationMode, du, u, uprev, abstol, reltol, args...)
+    if check_convergence(mode, du, u, uprev, abstol, reltol)
+        cache.retcode = ReturnCode.Success
+        return true
+    end
+    return false
+end
+
+function (cache::NonlinearTerminationModeCache)(
+        mode::AbstractSafeNonlinearTerminationMode, du, u, uprev, abstol, reltol, args...)
+    if mode isa AbsNormSafeTerminationMode || mode isa AbsNormSafeBestTerminationMode
+        objective = Utils.apply_norm(mode.internalnorm, du)
+        criteria = abstol
+    else
+        objective = Utils.apply_norm(mode.internalnorm, du) /
+                    (Utils.apply_norm(mode.internalnorm, du, u) + eps(reltol))
+        criteria = reltol
+    end
+
+    # Protective Break
+    if !isfinite(objective)
+        cache.retcode = ReturnCode.Unstable
+        return true
+    end
+
+    # By default we turn this off since it have potential for false positives
+    if mode.protective_threshold !== nothing &&
+       (objective > cache.initial_objective * mode.protective_threshold * length(du))
+        cache.retcode = ReturnCode.Unstable
+        return true
+    end
+
+    # Check if it is the best solution
+    if mode isa AbstractSafeBestNonlinearTerminationMode &&
+       objective < cache.best_objective_value
+        cache.best_objective_value = objective
+        update_u!!(cache, u)
+        cache.saved_values !== nothing && length(args) ≥ 1 && (cache.saved_values = args)
+    end
+
+    # Main Termination Criteria
+    if objective ≤ criteria
+        cache.retcode = ReturnCode.Success
+        return true
+    end
+
+    # Terminate if we haven't improved for the last `patience_steps`
+    cache.nsteps += 1
+    cache.nsteps == 1 && (cache.initial_objective = objective)
+    cache.objectives_trace[mod1(cache.nsteps, length(cache.objectives_trace))] = objective
+
+    if objective ≤ mode.patience_objective_multiplier * criteria &&
+       cache.nsteps > mode.patience_steps
+        if cache.nsteps < length(cache.objectives_trace)
+            min_obj, max_obj = extrema(@view(cache.objectives_trace[1:(cache.nsteps)]))
+        else
+            min_obj, max_obj = extrema(cache.objectives_trace)
+        end
+        if min_obj < mode.min_max_factor * max_obj
+            cache.retcode = ReturnCode.Stalled
+            return true
+        end
+    end
+
+    # Test for stalling if that is enabled
+    if cache.step_norm_trace !== nothing
+        if ArrayInterface.can_setindex(cache.u_diff_cache) && !(u isa Number)
+            @. cache.u_diff_cache = u - uprev
+        else
+            cache.u_diff_cache = u .- uprev
+        end
+        du_norm = L2_NORM(cache.u_diff_cache)
+        cache.step_norm_trace[mod1(cache.nsteps, length(cache.step_norm_trace))] = du_norm
+        if cache.nsteps > mode.max_stalled_steps
+            max_step_norm = maximum(cache.step_norm_trace)
+            if mode isa AbsNormSafeTerminationMode ||
+               mode isa AbsNormSafeBestTerminationMode
+                stalled_step = max_step_norm ≤ abstol
+            else
+                stalled_step = max_step_norm ≤ reltol * (max_step_norm + cache.u0_norm)
+            end
+            if stalled_step
+                cache.retcode = ReturnCode.Stalled
+                return true
+            end
+        end
+    end
+
+    cache.retcode = ReturnCode.Failure
+    return false
+end
+
+# Check Convergence
+function check_convergence(::RelTerminationMode, duₙ, uₙ, __, ___, reltol)
+    if Utils.fast_scalar_indexing(duₙ)
+        return all(@closure(xy->begin
+                x, y = xy
+                return abs(y) ≤ reltol * abs(x + y)
+            end), zip(uₙ, duₙ))
+    else # using mapreduce here will almost certainly be faster on GPUs
+        return mapreduce(
+            @closure((xᵢ, yᵢ)->(abs(yᵢ) ≤ reltol * abs(xᵢ + yᵢ))), *, uₙ, duₙ; init = true)
+    end
+end
+function check_convergence(::AbsTerminationMode, duₙ, _, __, abstol, ___)
+    return all(@closure(x->abs(x) ≤ abstol), duₙ)
+end
+
+function check_convergence(norm::NormTerminationMode, duₙ, uₙ, _, abstol, reltol)
+    du_norm = Utils.apply_norm(norm.internalnorm, duₙ)
+    return (du_norm ≤ abstol) ||
+           (du_norm ≤ reltol * Utils.apply_norm(norm.internalnorm, duₙ, uₙ))
+end
+
+function check_convergence(mode::RelNormModes, duₙ, uₙ, _, __, reltol)
+    du_norm = Utils.apply_norm(mode.internalnorm, duₙ)
+    return du_norm ≤ reltol * Utils.apply_norm(mode.internalnorm, duₙ, uₙ)
+end
+
+function check_convergence(mode::AbsNormModes, duₙ, _, __, abstol, ___)
+    return Utils.apply_norm(mode.internalnorm, duₙ) ≤ abstol
+end
+
+# High-Level API with defaults.
+## This is mostly for internal usage in NonlinearSolve and SimpleNonlinearSolve
+function default_termination_mode(
+        ::Union{ImmutableNonlinearProblem, NonlinearProblem}, ::Val{:simple})
+    return AbsNormTerminationMode(Base.Fix1(maximum, abs))
+end
+function default_termination_mode(::NonlinearLeastSquaresProblem, ::Val{:simple})
+    return AbsNormTerminationMode(Base.Fix2(norm, 2))
+end
+
+function default_termination_mode(
+        ::Union{ImmutableNonlinearProblem, NonlinearProblem}, ::Val{:regular})
+    return AbsNormSafeBestTerminationMode(Base.Fix1(maximum, abs); max_stalled_steps = 32)
+end
+
+function default_termination_mode(::NonlinearLeastSquaresProblem, ::Val{:regular})
+    return AbsNormSafeBestTerminationMode(Base.Fix2(norm, 2); max_stalled_steps = 32)
+end
+
+function init_termination_cache(
+        prob::AbstractNonlinearProblem, abstol, reltol, du, u, ::Nothing, callee::Val)
+    return init_termination_cache(
+        prob, abstol, reltol, du, u, default_termination_mode(prob, callee), callee)
+end
+
+function init_termination_cache(prob::AbstractNonlinearProblem, abstol, reltol, du,
+        u, tc::AbstractNonlinearTerminationMode, ::Val)
+    T = promote_type(eltype(du), eltype(u))
+    abstol = get_tolerance(u, abstol, T)
+    reltol = get_tolerance(u, reltol, T)
+    cache = CommonSolve.init(prob, tc, du, u; abstol, reltol)
+    return abstol, reltol, cache
+end
diff --git a/lib/NonlinearSolveBase/src/utils.jl b/lib/NonlinearSolveBase/src/utils.jl
new file mode 100644
index 000000000..825f63767
--- /dev/null
+++ b/lib/NonlinearSolveBase/src/utils.jl
@@ -0,0 +1,93 @@
+module Utils
+
+using ArrayInterface: ArrayInterface
+using FastClosures: @closure
+using LinearAlgebra: norm
+using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition
+
+using ..NonlinearSolveBase: L2_NORM, Linf_NORM
+
+fast_scalar_indexing(xs...) = all(ArrayInterface.fast_scalar_indexing, xs)
+
+function nonallocating_isapprox(x::Number, y::Number; atol = false,
+        rtol = atol > 0 ? false : sqrt(eps(promote_type(typeof(x), typeof(y)))))
+    return isapprox(x, y; atol, rtol)
+end
+function nonallocating_isapprox(x::AbstractArray, y::AbstractArray; atol = false,
+        rtol = atol > 0 ? false : sqrt(eps(eltype(x))))
+    length(x) == length(y) || return false
+    d = nonallocating_maximum(-, x, y)
+    return d ≤ max(atol, rtol * max(maximum(abs, x), maximum(abs, y)))
+end
+
+function nonallocating_maximum(f::F, x, y) where {F}
+    if fast_scalar_indexing(x, y)
+        return maximum(@closure((xᵢyᵢ)->begin
+                xᵢ, yᵢ = xᵢyᵢ
+                return abs(f(xᵢ, yᵢ))
+            end), zip(x, y))
+    else
+        return mapreduce(@closure((xᵢ, yᵢ)->abs(f(xᵢ, yᵢ))), max, x, y)
+    end
+end
+
+function abs2_and_sum(x, y)
+    return reduce(Base.add_sum, x, init = zero(real(value(eltype(x))))) +
+           reduce(Base.add_sum, y, init = zero(real(value(eltype(y)))))
+end
+
+children(x::AbstractVectorOfArray) = x.u
+children(x::ArrayPartition) = x.x
+
+value(::Type{T}) where {T} = T
+value(x) = x
+
+zero_init(x) = zero(real(value(eltype(x))))
+one_init(x) = one(real(value(eltype(x))))
+
+standardize_norm(::typeof(Base.Fix1(maximum, abs))) = Linf_NORM
+standardize_norm(::typeof(Base.Fix2(norm, Inf))) = Linf_NORM
+standardize_norm(::typeof(Base.Fix2(norm, 2))) = L2_NORM
+standardize_norm(::typeof(norm)) = L2_NORM
+standardize_norm(f::F) where {F} = f
+
+norm_op(norm::N, op::OP, x, y) where {N, OP} = norm(op.(x, y))
+function norm_op(::typeof(L2_NORM), op::OP, x, y) where {OP}
+    if fast_scalar_indexing(x, y)
+        return sqrt(sum(@closure((xᵢyᵢ)->begin
+                xᵢ, yᵢ = xᵢyᵢ
+                return op(xᵢ, yᵢ)^2
+            end), zip(x, y)))
+    else
+        return sqrt(mapreduce(@closure((xᵢ, yᵢ)->op(xᵢ, yᵢ)^2), +, x, y))
+    end
+end
+function norm_op(::typeof(Linf_NORM), op::OP, x, y) where {OP}
+    return nonallocating_maximum(abs ∘ op, x, y)
+end
+
+apply_norm(f::F, x) where {F} = standardize_norm(f)(x)
+apply_norm(f::F, x, y) where {F} = norm_op(standardize_norm(f), +, x, y)
+
+convert_real(::Type{T}, ::Nothing) where {T} = nothing
+convert_real(::Type{T}, x) where {T} = real(T(x))
+
+restructure(::Number, x::Number) = x
+restructure(y, x) = ArrayInterface.restructure(y, x)
+
+function safe_similar(x, args...; kwargs...)
+    y = similar(x, args...; kwargs...)
+    return init_bigfloat_array!!(y)
+end
+
+init_bigfloat_array!!(x) = x
+
+function init_bigfloat_array!!(x::AbstractArray{<:BigFloat})
+    ArrayInterface.can_setindex(x) && fill!(x, BigFloat(0))
+    return x
+end
+
+safe_reshape(x::Number, args...) = x
+safe_reshape(x, args...) = reshape(x, args...)
+
+end
diff --git a/lib/NonlinearSolveBase/test/runtests.jl b/lib/NonlinearSolveBase/test/runtests.jl
new file mode 100644
index 000000000..07c0f14c6
--- /dev/null
+++ b/lib/NonlinearSolveBase/test/runtests.jl
@@ -0,0 +1,24 @@
+using InteractiveUtils, Test
+
+@info sprint(InteractiveUtils.versioninfo)
+
+# Changing any code here triggers all the other tests to be run. So we intentionally
+# keep the tests here minimal.
+@testset "NonlinearSolveBase.jl" begin
+    @testset "Aqua" begin
+        using Aqua, NonlinearSolveBase
+
+        Aqua.test_all(NonlinearSolveBase; piracies = false, ambiguities = false)
+        Aqua.test_piracies(NonlinearSolveBase)
+        Aqua.test_ambiguities(NonlinearSolveBase; recursive = false)
+    end
+
+    @testset "Explicit Imports" begin
+        import ForwardDiff, SparseArrays, DiffEqBase
+        using ExplicitImports, NonlinearSolveBase
+
+        @test check_no_implicit_imports(NonlinearSolveBase; skip = (Base, Core)) === nothing
+        @test check_no_stale_explicit_imports(NonlinearSolveBase) === nothing
+        @test check_all_qualified_accesses_via_owners(NonlinearSolveBase) === nothing
+    end
+end
diff --git a/lib/SciMLJacobianOperators/Project.toml b/lib/SciMLJacobianOperators/Project.toml
index 73ee5a55d..c889eb199 100644
--- a/lib/SciMLJacobianOperators/Project.toml
+++ b/lib/SciMLJacobianOperators/Project.toml
@@ -18,8 +18,8 @@ ADTypes = "1.8.1"
 Aqua = "0.8.7"
 ConcreteStructs = "0.2.3"
 ConstructionBase = "1.5"
-DifferentiationInterface = "0.6.1"
-Enzyme = "0.12, 0.13"
+DifferentiationInterface = "0.6.16"
+Enzyme = "0.13.11"
 ExplicitImports = "1.9.0"
 FastClosures = "0.3.2"
 FiniteDiff = "2.24.0"
diff --git a/lib/SimpleNonlinearSolve/LICENSE b/lib/SimpleNonlinearSolve/LICENSE
new file mode 100644
index 000000000..8eef16440
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Julia Computing, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file
diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml
new file mode 100644
index 000000000..cfda24544
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/Project.toml
@@ -0,0 +1,93 @@
+name = "SimpleNonlinearSolve"
+uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7"
+authors = ["SciML"]
+version = "2.0.0"
+
+[deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
+Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
+ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
+BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e"
+CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
+ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471"
+DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
+FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a"
+FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LineSearch = "87fe0de2-c867-4266-b59a-2f0a94fc965b"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb"
+NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0"
+PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
+
+[weakdeps]
+ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
+
+[extensions]
+SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore"
+SimpleNonlinearSolveDiffEqBaseExt = "DiffEqBase"
+SimpleNonlinearSolveReverseDiffExt = "ReverseDiff"
+SimpleNonlinearSolveTrackerExt = "Tracker"
+
+[compat]
+ADTypes = "1.2"
+Accessors = "0.1"
+Aqua = "0.8.7"
+ArrayInterface = "7.16"
+BracketingNonlinearSolve = "1"
+ChainRulesCore = "1.24"
+CommonSolve = "0.2.4"
+ConcreteStructs = "0.2.3"
+DiffEqBase = "6.149"
+DifferentiationInterface = "0.6.16"
+Enzyme = "0.13.11"
+ExplicitImports = "1.9"
+FastClosures = "0.3.2"
+FiniteDiff = "2.24.0"
+ForwardDiff = "0.10.36"
+InteractiveUtils = "<0.0.1, 1"
+LineSearch = "0.1.3"
+LinearAlgebra = "1.10"
+MaybeInplace = "0.1.4"
+NonlinearProblemLibrary = "0.1.2"
+NonlinearSolveBase = "1"
+Pkg = "1.10"
+PolyesterForwardDiff = "0.1"
+PrecompileTools = "1.2"
+Random = "1.10"
+Reexport = "1.2"
+ReverseDiff = "1.15"
+SciMLBase = "2.50"
+StaticArrays = "1.9"
+StaticArraysCore = "1.4.3"
+Test = "1.10"
+TestItemRunner = "1"
+Tracker = "0.2.35"
+Zygote = "0.6.70"
+julia = "1.10"
+
+[extras]
+Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
+Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+NonlinearProblemLibrary = "b7050fa9-e91f-4b37-bcee-a89a063da141"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+
+[targets]
+test = ["Aqua", "DiffEqBase", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"]
diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md
new file mode 100644
index 000000000..562a5a4ae
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/README.md
@@ -0,0 +1,23 @@
+# SimpleNonlinearSolve.jl
+
+Fast implementations of root finding algorithms in Julia that satisfy the SciML common interface.
+SimpleNonlinearSolve.jl focuses on low-dependency implementations of very fast methods for
+very small and simple problems. For the full set of solvers, see
+[NonlinearSolve.jl](https://github.com/SciML/NonlinearSolve.jl), of which
+SimpleNonlinearSolve.jl is just one solver set.
+
+For information on using the package,
+[see the stable documentation](https://docs.sciml.ai/NonlinearSolve/stable/). Use the
+[in-development documentation](https://docs.sciml.ai/NonlinearSolve/dev/) for the version of
+the documentation which contains the unreleased features.
+
+## High Level Examples
+
+```julia
+using SimpleNonlinearSolve, StaticArrays
+
+f(u, p) = u .* u .- 2
+u0 = @SVector[1.0, 1.0]
+probN = NonlinearProblem{false}(f, u0)
+solver = solve(probN, SimpleNewtonRaphson(), abstol = 1e-9)
+```
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
new file mode 100644
index 000000000..f56dee537
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl
@@ -0,0 +1,23 @@
+module SimpleNonlinearSolveChainRulesCoreExt
+
+using ChainRulesCore: ChainRulesCore, NoTangent
+using NonlinearSolveBase: ImmutableNonlinearProblem
+using SciMLBase: ChainRulesOriginator, NonlinearLeastSquaresProblem
+
+using SimpleNonlinearSolve: SimpleNonlinearSolve, simplenonlinearsolve_solve_up,
+                            solve_adjoint
+
+function ChainRulesCore.rrule(::typeof(simplenonlinearsolve_solve_up),
+        prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
+        sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...)
+    out, ∇internal = solve_adjoint(
+        prob, sensealg, u0, p, ChainRulesOriginator(), alg, args...; kwargs...)
+    function ∇simplenonlinearsolve_solve_up(Δ)
+        ∂f, ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Δ)
+        return (
+            ∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), NoTangent(), ∂args...)
+    end
+    return out, ∇simplenonlinearsolve_solve_up
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl
new file mode 100644
index 000000000..4954ffb26
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl
@@ -0,0 +1,13 @@
+module SimpleNonlinearSolveDiffEqBaseExt
+
+using DiffEqBase: DiffEqBase
+
+using SimpleNonlinearSolve: SimpleNonlinearSolve
+
+SimpleNonlinearSolve.is_extension_loaded(::Val{:DiffEqBase}) = true
+
+function SimpleNonlinearSolve.solve_adjoint_internal(args...; kwargs...)
+    return DiffEqBase._solve_adjoint(args...; kwargs...)
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
new file mode 100644
index 000000000..0a407986e
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl
@@ -0,0 +1,40 @@
+module SimpleNonlinearSolveReverseDiffExt
+
+using ArrayInterface: ArrayInterface
+using NonlinearSolveBase: ImmutableNonlinearProblem
+using ReverseDiff: ReverseDiff, TrackedArray, TrackedReal
+using SciMLBase: ReverseDiffOriginator, NonlinearLeastSquaresProblem, remake
+
+using SimpleNonlinearSolve: SimpleNonlinearSolve, solve_adjoint
+import SimpleNonlinearSolve: simplenonlinearsolve_solve_up
+
+for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem)
+    aTypes = (TrackedArray, AbstractArray{<:TrackedReal}, Any)
+    for (uT, pT) in collect(Iterators.product(aTypes, aTypes))[1:(end - 1)]
+        @eval function simplenonlinearsolve_solve_up(
+                prob::$(pType), sensealg, u0::$(uT), u0_changed,
+                p::$(pT), p_changed, alg, args...; kwargs...)
+            return ReverseDiff.track(SimpleNonlinearSolve.simplenonlinearsolve_solve_up,
+                prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
+                ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+        end
+    end
+
+    @eval ReverseDiff.@grad function simplenonlinearsolve_solve_up(
+            tprob::$(pType), sensealg, tu0, u0_changed,
+            tp, p_changed, alg, args...; kwargs...)
+        u0, p = ReverseDiff.value(tu0), ReverseDiff.value(tp)
+        prob = remake(tprob; u0, p)
+        out, ∇internal = solve_adjoint(
+            prob, sensealg, u0, p, ReverseDiffOriginator(), alg, args...; kwargs...)
+
+        function ∇simplenonlinearsolve_solve_up(Δ...)
+            ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Δ...)
+            return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+        end
+
+        return Array(out), ∇simplenonlinearsolve_solve_up
+    end
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
new file mode 100644
index 000000000..d29c2ac61
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl
@@ -0,0 +1,39 @@
+module SimpleNonlinearSolveTrackerExt
+
+using ArrayInterface: ArrayInterface
+using NonlinearSolveBase: ImmutableNonlinearProblem
+using SciMLBase: TrackerOriginator, NonlinearLeastSquaresProblem, remake
+using Tracker: Tracker, TrackedArray, TrackedReal
+
+using SimpleNonlinearSolve: SimpleNonlinearSolve, solve_adjoint
+
+for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem)
+    aTypes = (TrackedArray, AbstractArray{<:TrackedReal}, Any)
+    for (uT, pT) in collect(Iterators.product(aTypes, aTypes))[1:(end - 1)]
+        @eval function SimpleNonlinearSolve.simplenonlinearsolve_solve_up(
+                prob::$(pType), sensealg, u0::$(uT), u0_changed,
+                p::$(pT), p_changed, alg, args...; kwargs...)
+            return Tracker.track(SimpleNonlinearSolve.simplenonlinearsolve_solve_up,
+                prob, sensealg, ArrayInterface.aos_to_soa(u0), true,
+                ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...)
+        end
+    end
+
+    @eval Tracker.@grad function SimpleNonlinearSolve.simplenonlinearsolve_solve_up(
+            tprob::$(pType), sensealg, tu0, u0_changed,
+            tp, p_changed, alg, args...; kwargs...)
+        u0, p = Tracker.data(tu0), Tracker.data(tp)
+        prob = remake(tprob; u0, p)
+        out, ∇internal = solve_adjoint(
+            prob, sensealg, u0, p, TrackerOriginator(), alg, args...; kwargs...)
+
+        function ∇simplenonlinearsolve_solve_up(Δ)
+            ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Tracker.data(Δ))
+            return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...)
+        end
+
+        return out, ∇simplenonlinearsolve_solve_up
+    end
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
new file mode 100644
index 000000000..cee8f8dd3
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl
@@ -0,0 +1,152 @@
+module SimpleNonlinearSolve
+
+using Accessors: @reset
+using BracketingNonlinearSolve: BracketingNonlinearSolve
+using CommonSolve: CommonSolve, solve, init, solve!
+using ConcreteStructs: @concrete
+using FastClosures: @closure
+using LineSearch: LiFukushimaLineSearch
+using LinearAlgebra: LinearAlgebra, dot
+using MaybeInplace: @bb, setindex_trait, CannotSetindex, CanSetindex
+using PrecompileTools: @compile_workload, @setup_workload
+using Reexport: @reexport
+using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, NonlinearFunction, NonlinearProblem,
+                 NonlinearLeastSquaresProblem, IntervalNonlinearProblem, ReturnCode, remake
+using StaticArraysCore: StaticArray, SArray, SVector, MArray
+
+# AD Dependencies
+using ADTypes: ADTypes, AutoForwardDiff
+using DifferentiationInterface: DifferentiationInterface
+using FiniteDiff: FiniteDiff
+using ForwardDiff: ForwardDiff
+
+using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, L2_NORM,
+                          nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution
+
+const DI = DifferentiationInterface
+
+abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end
+
+const safe_similar = NonlinearSolveBase.Utils.safe_similar
+
+is_extension_loaded(::Val) = false
+
+include("utils.jl")
+
+include("broyden.jl")
+include("dfsane.jl")
+include("halley.jl")
+include("klement.jl")
+include("lbroyden.jl")
+include("raphson.jl")
+include("trust_region.jl")
+
+# By Pass the highlevel checks for NonlinearProblem for Simple Algorithms
+function CommonSolve.solve(prob::NonlinearProblem,
+        alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...)
+    prob = convert(ImmutableNonlinearProblem, prob)
+    return solve(prob, alg, args...; kwargs...)
+end
+
+function CommonSolve.solve(
+        prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip,
+            <:Union{
+                <:ForwardDiff.Dual{T, V, P}, <:AbstractArray{<:ForwardDiff.Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...;
+        kwargs...) where {T, V, P, iip}
+    if hasfield(typeof(alg), :autodiff) && alg.autodiff === nothing
+        @reset alg.autodiff = AutoForwardDiff()
+    end
+    prob = convert(ImmutableNonlinearProblem, prob)
+    sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...)
+    dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+end
+
+function CommonSolve.solve(
+        prob::NonlinearLeastSquaresProblem{<:Union{Number, <:AbstractArray}, iip,
+            <:Union{
+                <:ForwardDiff.Dual{T, V, P}, <:AbstractArray{<:ForwardDiff.Dual{T, V, P}}}},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...;
+        kwargs...) where {T, V, P, iip}
+    if hasfield(typeof(alg), :autodiff) && alg.autodiff === nothing
+        @reset alg.autodiff = AutoForwardDiff()
+    end
+    sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...)
+    dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p)
+    return SciMLBase.build_solution(
+        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+end
+
+function CommonSolve.solve(
+        prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
+        alg::AbstractSimpleNonlinearSolveAlgorithm,
+        args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...)
+    if sensealg === nothing && haskey(prob.kwargs, :sensealg)
+        sensealg = prob.kwargs[:sensealg]
+    end
+    new_u0 = u0 !== nothing ? u0 : prob.u0
+    new_p = p !== nothing ? p : prob.p
+    return simplenonlinearsolve_solve_up(prob, sensealg, new_u0, u0 === nothing, new_p,
+        p === nothing, alg, args...; prob.kwargs..., kwargs...)
+end
+
+function simplenonlinearsolve_solve_up(
+        prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0,
+        u0_changed, p, p_changed, alg, args...; kwargs...)
+    (u0_changed || p_changed) && (prob = remake(prob; u0, p))
+    return SciMLBase.__solve(prob, alg, args...; kwargs...)
+end
+
+# NOTE: This is defined like this so that we don't have to keep have 2 args for the
+# extensions
+function solve_adjoint(args...; kws...)
+    is_extension_loaded(Val(:DiffEqBase)) && return solve_adjoint_internal(args...; kws...)
+    error("Adjoint sensitivity analysis requires `DiffEqBase.jl` to be explicitly loaded.")
+end
+
+function solve_adjoint_internal end
+
+@setup_workload begin
+    for T in (Float64,)
+        prob_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2))
+        prob_iip = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p, ones(T, 3), T(2))
+        prob_oop = NonlinearProblem{false}((u, p) -> u .* u .- p, ones(T, 3), T(2))
+
+        # Only compile frequently used algorithms -- mostly from the NonlinearSolve default
+        #!format: off
+        algs = [
+            SimpleBroyden(),
+            # SimpleDFSane(),
+            SimpleKlement(),
+            # SimpleLimitedMemoryBroyden(),
+            # SimpleHalley(),
+            SimpleNewtonRaphson(; autodiff = AutoForwardDiff(; chunksize = 1)),
+            # SimpleTrustRegion()
+        ]
+        #!format: on
+
+        @compile_workload begin
+            @sync for alg in algs
+                for prob in (prob_scalar, prob_iip, prob_oop)
+                    Threads.@spawn CommonSolve.solve(prob, alg; abstol = 1e-2)
+                end
+            end
+        end
+    end
+end
+
+# Rexexports
+@reexport using ADTypes, SciMLBase, BracketingNonlinearSolve, NonlinearSolveBase
+
+export SimpleBroyden, SimpleKlement, SimpleLimitedMemoryBroyden
+export SimpleDFSane
+export SimpleGaussNewton, SimpleNewtonRaphson, SimpleTrustRegion
+export SimpleHalley
+
+export solve
+
+end
diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl
new file mode 100644
index 000000000..6537a4d2d
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/broyden.jl
@@ -0,0 +1,101 @@
+"""
+    SimpleBroyden(; linesearch = Val(false), alpha = nothing)
+
+A low-overhead implementation of Broyden. This method is non-allocating on scalar and static
+array problems.
+
+### Keyword Arguments
+
+  - `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch`
+    line search else no line search is used. For advanced customization of the line search,
+    use `Broyden` from `NonlinearSolve.jl`.
+  - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
+    will compute the scaling using `2 * norm(fu) / max(norm(u), true)`.
+"""
+@concrete struct SimpleBroyden <: AbstractSimpleNonlinearSolveAlgorithm
+    linesearch <: Union{Val{false}, Val{true}}
+    alpha
+end
+
+function SimpleBroyden(;
+        linesearch::Union{Bool, Val{true}, Val{false}} = Val(false), alpha = nothing)
+    linesearch = linesearch isa Bool ? Val(linesearch) : linesearch
+    return SimpleBroyden(linesearch, alpha)
+end
+
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleBroyden, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
+    x = Utils.maybe_unaliased(prob.u0, alias_u0)
+    fx = Utils.get_fx(prob, x)
+    fx = Utils.eval_f(prob, fx, x)
+    T = promote_type(eltype(fx), eltype(x))
+
+    iszero(fx) &&
+        return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+
+    @bb xo = copy(x)
+    @bb δx = similar(x)
+    @bb δf = copy(fx)
+    @bb fprev = copy(fx)
+
+    if alg.alpha === nothing
+        fx_norm = L2_NORM(fx)
+        x_norm = L2_NORM(x)
+        init_α = ifelse(fx_norm ≥ 1e-5, max(x_norm, T(true)) / (2 * fx_norm), T(true))
+    else
+        init_α = inv(alg.alpha)
+    end
+
+    J⁻¹ = Utils.identity_jacobian(fx, x, init_α)
+    @bb J⁻¹δf = copy(x)
+    @bb xᵀJ⁻¹ = copy(x)
+    @bb δJ⁻¹n = copy(x)
+    @bb δJ⁻¹ = copy(J⁻¹)
+
+    abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition, Val(:simple))
+
+    if alg.linesearch === Val(true)
+        ls_alg = LiFukushimaLineSearch(; nan_maxiters = nothing)
+        ls_cache = init(prob, ls_alg, fx, x)
+    else
+        ls_cache = nothing
+    end
+
+    for _ in 1:maxiters
+        @bb δx = J⁻¹ × vec(fprev)
+        @bb δx .*= -1
+
+        if ls_cache === nothing
+            α = true
+        else
+            ls_sol = solve!(ls_cache, xo, δx)
+            α = ls_sol.step_size # Ignores the return code for now
+        end
+
+        @bb @. x = xo + α * δx
+        fx = Utils.eval_f(prob, fx, x)
+        @bb @. δf = fx - fprev
+
+        # Termination Checks
+        solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob)
+        solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode)
+
+        @bb J⁻¹δf = J⁻¹ × vec(δf)
+        d = dot(δx, J⁻¹δf)
+        @bb xᵀJ⁻¹ = transpose(J⁻¹) × vec(δx)
+
+        @bb @. δJ⁻¹n = (δx - J⁻¹δf) / d
+
+        δJ⁻¹n_ = Utils.safe_vec(δJ⁻¹n)
+        xᵀJ⁻¹_ = Utils.safe_vec(xᵀJ⁻¹)
+        @bb δJ⁻¹ = δJ⁻¹n_ × transpose(xᵀJ⁻¹_)
+        @bb J⁻¹ .+= δJ⁻¹
+
+        @bb copyto!(xo, x)
+        @bb copyto!(fprev, fx)
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl
new file mode 100644
index 000000000..0d400b0ce
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/dfsane.jl
@@ -0,0 +1,170 @@
+"""
+    SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
+        M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+        nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2)
+
+A low-overhead implementation of the df-sane method for solving large-scale nonlinear
+systems of equations. For in depth information about all the parameters and the algorithm,
+see [la2006spectral](@citet).
+
+### Keyword Arguments
+
+  - `σ_min`: the minimum value of the spectral coefficient `σ_k` which is related to the
+    step size in the algorithm. Defaults to `1e-10`.
+  - `σ_max`: the maximum value of the spectral coefficient `σ_k` which is related to the
+    step size in the algorithm. Defaults to `1e10`.
+  - `σ_1`: the initial value of the spectral coefficient `σ_k` which is related to the step
+    size in the algorithm.. Defaults to `1.0`.
+  - `M`: The monotonicity of the algorithm is determined by a this positive integer.
+    A value of 1 for `M` would result in strict monotonicity in the decrease of the L2-norm
+    of the function `f`. However, higher values allow for more flexibility in this
+    reduction. Despite this, the algorithm still ensures global convergence through the use
+    of a non-monotone line-search algorithm that adheres to the Grippo-Lampariello-Lucidi
+    condition. Values in the range of 5 to 20 are usually sufficient, but some cases may call
+    for a higher value of `M`. The default setting is 10.
+  - `γ`: a parameter that influences if a proposed step will be accepted. Higher value of
+    `γ` will make the algorithm more restrictive in accepting steps. Defaults to `1e-4`.
+  - `τ_min`: if a step is rejected the new step size will get multiplied by factor, and this
+    parameter is the minimum value of that factor. Defaults to `0.1`.
+  - `τ_max`: if a step is rejected the new step size will get multiplied by factor, and this
+    parameter is the maximum value of that factor. Defaults to `0.5`.
+  - `nexp`: the exponent of the loss, i.e. ``f_k=||F(x_k)||^{nexp}``. The paper uses
+    `nexp ∈ {1,2}`. Defaults to `2`.
+  - `η_strategy`:  function to determine the parameter `η_k`, which enables growth
+    of ``||F||^2``. Called as `η_k = η_strategy(f_1, k, x, F)` with `f_1` initialized as
+    ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and
+    `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to
+    ``||F||^2 / k^2``.
+"""
+@concrete struct SimpleDFSane <: AbstractSimpleNonlinearSolveAlgorithm
+    σ_min
+    σ_max
+    σ_1
+    γ
+    τ_min
+    τ_max
+    nexp::Int
+    η_strategy
+    M <: Val
+end
+
+# XXX[breaking]: we should change the names to not have unicode
+function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0,
+        M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5,
+        nexp::Int = 2, η_strategy::F = (f_1, k, x, F) -> f_1 ./ k^2) where {F}
+    M = M isa Int ? Val(M) : M
+    return SimpleDFSane(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp, η_strategy, M)
+end
+
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleDFSane, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false,
+        termination_condition = nothing, kwargs...)
+    x = Utils.maybe_unaliased(prob.u0, alias_u0)
+    fx = Utils.get_fx(prob, x)
+    fx = Utils.eval_f(prob, fx, x)
+    T = promote_type(eltype(fx), eltype(x))
+
+    σ_min = T(alg.σ_min)
+    σ_max = T(alg.σ_max)
+    σ_k = T(alg.σ_1)
+
+    (; nexp, η_strategy, M) = alg
+    γ = T(alg.γ)
+    τ_min = T(alg.τ_min)
+    τ_max = T(alg.τ_max)
+
+    abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition, Val(:simple))
+
+    fx_norm = L2_NORM(fx)^nexp
+    α_1 = one(T)
+    f_1 = fx_norm
+
+    history_f_k = dfsane_history_vec(x, fx_norm, alg.M)
+
+    # Generate the cache
+    @bb x_cache = similar(x)
+    @bb d = copy(x)
+    @bb xo = copy(x)
+    @bb δx = copy(x)
+    @bb δf = copy(fx)
+
+    k = 0
+    while k < maxiters
+        # Spectral parameter range check
+        σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max)
+
+        # Line search direction
+        @bb @. d = -σ_k * fx
+
+        η = η_strategy(f_1, k + 1, x, fx)
+        f_bar = maximum(history_f_k)
+        α_p = α_1
+        α_m = α_1
+
+        @bb @. x_cache = x + α_p * d
+
+        fx = Utils.eval_f(prob, fx, x_cache)
+        fx_norm_new = L2_NORM(fx)^nexp
+
+        while k < maxiters
+            (fx_norm_new ≤ (f_bar + η - γ * α_p^2 * fx_norm)) && break
+
+            α_tp = α_p^2 * fx_norm / (fx_norm_new + (T(2) * α_p - T(1)) * fx_norm)
+            @bb @. x_cache = x - α_m * d
+
+            fx = Utils.eval_f(prob, fx, x_cache)
+            fx_norm_new = L2_NORM(fx)^nexp
+
+            (fx_norm_new ≤ (f_bar + η - γ * α_m^2 * fx_norm)) && break
+
+            α_tm = α_m^2 * fx_norm / (fx_norm_new + (T(2) * α_m - T(1)) * fx_norm)
+            α_p = clamp(α_tp, τ_min * α_p, τ_max * α_p)
+            α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m)
+            @bb @. x_cache = x + α_p * d
+
+            fx = Utils.eval_f(prob, fx, x_cache)
+            fx_norm_new = L2_NORM(fx)^nexp
+
+            k += 1
+        end
+
+        @bb copyto!(x, x_cache)
+
+        solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob)
+        solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode)
+
+        # Update spectral parameter
+        @bb @. δx = x - xo
+        @bb @. δf = fx - δf
+
+        σ_k = dot(δx, δx) / dot(δx, δf)
+
+        # Take step
+        @bb copyto!(xo, x)
+        @bb copyto!(δf, fx)
+        fx_norm = fx_norm_new
+
+        # Store function value
+        idx = mod1(k, SciMLBase._unwrap_val(alg.M))
+        if history_f_k isa SVector
+            history_f_k = Base.setindex(history_f_k, fx_norm_new, idx)
+        elseif history_f_k isa NTuple
+            @reset history_f_k[idx] = fx_norm_new
+        else
+            history_f_k[idx] = fx_norm_new
+        end
+        k += 1
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
+
+function dfsane_history_vec(x::StaticArray, fx_norm, ::Val{M}) where {M}
+    return ones(SVector{M, eltype(x)}) .* fx_norm
+end
+
+@generated function dfsane_history_vec(x, fx_norm, ::Val{M}) where {M}
+    M ≥ 11 && return :(fill(fx_norm, M)) # Julia can't specialize here
+    return :(ntuple(Returns(fx_norm), $(M)))
+end
diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl
new file mode 100644
index 000000000..30eb1a821
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/halley.jl
@@ -0,0 +1,85 @@
+"""
+    SimpleHalley(autodiff)
+    SimpleHalley(; autodiff = nothing)
+
+A low-overhead implementation of Halley's Method.
+
+!!! note
+
+    As part of the decreased overhead, this method omits some of the higher level error
+    catching of the other methods. Thus, to see better error messages, use one of the other
+    methods like `NewtonRaphson`.
+
+### Keyword Arguments
+
+  - `autodiff`: determines the backend used for the Jacobian. Defaults to  `nothing` (i.e.
+    automatic backend selection). Valid choices include jacobian backends from
+    `DifferentiationInterface.jl`.
+"""
+@kwdef @concrete struct SimpleHalley <: AbstractSimpleNonlinearSolveAlgorithm
+    autodiff = nothing
+end
+
+function SciMLBase.__solve(
+        prob::ImmutableNonlinearProblem, alg::SimpleHalley, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
+    x = Utils.maybe_unaliased(prob.u0, alias_u0)
+    fx = Utils.get_fx(prob, x)
+    fx = Utils.eval_f(prob, fx, x)
+    T = promote_type(eltype(fx), eltype(x))
+
+    iszero(fx) &&
+        return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+
+    abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition, Val(:simple))
+
+    # The way we write the 2nd order derivatives, we know Enzyme won't work there
+    autodiff = alg.autodiff === nothing ? AutoForwardDiff() : alg.autodiff
+
+    @bb xo = copy(x)
+
+    strait = setindex_trait(x)
+
+    A = strait isa CanSetindex ? safe_similar(x, length(x), length(x)) : x
+    Aaᵢ = strait isa CanSetindex ? safe_similar(x, length(x)) : x
+    cᵢ = strait isa CanSetindex ? safe_similar(x) : x
+
+    for _ in 1:maxiters
+        fx, J, H = Utils.compute_jacobian_and_hessian(autodiff, prob, fx, x)
+
+        strait isa CannotSetindex && (A = J)
+
+        # Factorize Once and Reuse
+        J_fact = if J isa Number
+            J
+        else
+            fact = LinearAlgebra.lu(J; check = false)
+            !LinearAlgebra.issuccess(fact) && return SciMLBase.build_solution(
+                prob, alg, x, fx; retcode = ReturnCode.Unstable)
+            fact
+        end
+
+        aᵢ = J_fact \ Utils.safe_vec(fx)
+        A_ = Utils.safe_vec(A)
+        @bb A_ = H × aᵢ
+        A = Utils.restructure(A, A_)
+
+        @bb Aaᵢ = A × aᵢ
+        @bb A .*= -1
+        bᵢ = J_fact \ Utils.safe_vec(Aaᵢ)
+
+        cᵢ_ = Utils.safe_vec(cᵢ)
+        @bb @. cᵢ_ = (aᵢ * aᵢ) / (-aᵢ + (T(0.5) * bᵢ))
+        cᵢ = Utils.restructure(cᵢ, cᵢ_)
+
+        solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob)
+        solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode)
+
+        @bb @. x += cᵢ
+        @bb copyto!(xo, x)
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl
new file mode 100644
index 000000000..31c4cca96
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/klement.jl
@@ -0,0 +1,49 @@
+"""
+    SimpleKlement()
+
+A low-overhead implementation of `Klement` [klement2014using](@citep). This
+method is non-allocating on scalar and static array problems.
+"""
+struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end
+
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleKlement, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
+    x = Utils.maybe_unaliased(prob.u0, alias_u0)
+    T = eltype(x)
+    fx = Utils.get_fx(prob, x)
+    fx = Utils.eval_f(prob, fx, x)
+
+    abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition, Val(:simple))
+
+    @bb δx = copy(x)
+    @bb fprev = copy(fx)
+    @bb xo = copy(x)
+    @bb d = copy(x)
+
+    J = one.(x)
+    @bb δx² = similar(x)
+
+    for _ in 1:maxiters
+        any(iszero, J) && (J = Utils.identity_jacobian!!(J))
+
+        @bb @. δx = fprev / J
+
+        @bb @. x = xo - δx
+        fx = Utils.eval_f(prob, fx, x)
+
+        # Termination Checks
+        solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob)
+        solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode)
+
+        @bb δx .*= -1
+        @bb @. δx² = δx^2 * J^2
+        @bb @. J += (fx - fprev - J * δx) / ifelse(iszero(δx²), T(1e-5), δx²) * δx * (J^2)
+
+        @bb copyto!(fprev, fx)
+        @bb copyto!(xo, x)
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl
new file mode 100644
index 000000000..d2bd6ef83
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl
@@ -0,0 +1,333 @@
+"""
+    SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
+        linesearch = Val(false), alpha = nothing)
+
+A limited memory implementation of Broyden. This method applies the L-BFGS scheme to
+Broyden's method.
+
+If the threshold is larger than the problem size, then this method will use `SimpleBroyden`.
+
+### Keyword Arguments:
+
+  - `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch`
+    line search else no line search is used. For advanced customization of the line search,
+    use `Broyden` from `NonlinearSolve.jl`.
+  - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we
+    will compute the scaling using `2 * norm(fu) / max(norm(u), true)`.
+
+!!! warning
+
+    Currently `alpha` is only used for StaticArray problems. This will be fixed in the
+    future.
+"""
+@concrete struct SimpleLimitedMemoryBroyden <: AbstractSimpleNonlinearSolveAlgorithm
+    linesearch <: Union{Val{false}, Val{true}}
+    threshold <: Val
+    alpha
+end
+
+function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27),
+        linesearch::Union{Bool, Val{true}, Val{false}} = Val(false), alpha = nothing)
+    linesearch = linesearch isa Bool ? Val(linesearch) : linesearch
+    threshold = threshold isa Int ? Val(threshold) : threshold
+    return SimpleLimitedMemoryBroyden(linesearch, threshold, alpha)
+end
+
+function SciMLBase.__solve(
+        prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+        args...; termination_condition = nothing, kwargs...)
+    if prob.u0 isa SArray
+        if termination_condition === nothing ||
+           termination_condition isa NonlinearSolveBase.AbsNormTerminationMode
+            return internal_static_solve(
+                prob, alg, args...; termination_condition, kwargs...)
+        end
+        @warn "Specifying `termination_condition = $(termination_condition)` for \
+               `SimpleLimitedMemoryBroyden` with `SArray` is not non-allocating. Use \
+               either `termination_condition = AbsNormTerminationMode(Base.Fix2(norm, Inf))` \
+               or `termination_condition = nothing`." maxlog=1
+    end
+    return internal_generic_solve(prob, alg, args...; termination_condition, kwargs...)
+end
+
+@views function internal_generic_solve(
+        prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden,
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
+    x = Utils.maybe_unaliased(prob.u0, alias_u0)
+    η = min(SciMLBase._unwrap_val(alg.threshold), maxiters)
+
+    # For scalar problems / if the threshold is larger than problem size just use Broyden
+    if x isa Number || length(x) ≤ η
+        return SciMLBase.__solve(prob, SimpleBroyden(; alg.linesearch), args...; abstol,
+            reltol, maxiters, termination_condition, kwargs...)
+    end
+
+    fx = Utils.get_fx(prob, x)
+
+    U, Vᵀ = init_low_rank_jacobian(x, fx, x isa StaticArray ? alg.threshold : Val(η))
+
+    abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition, Val(:simple))
+
+    @bb xo = copy(x)
+    @bb δx = copy(fx)
+    @bb δx .*= -1
+    @bb fo = copy(fx)
+    @bb δf = copy(fx)
+
+    @bb vᵀ_cache = copy(x)
+    Tcache = lbroyden_threshold_cache(x, x isa StaticArray ? alg.threshold : Val(η))
+    @bb mat_cache = copy(x)
+
+    if alg.linesearch === Val(true)
+        ls_alg = LiFukushimaLineSearch(; nan_maxiters = nothing)
+        ls_cache = init(prob, ls_alg, fx, x)
+    else
+        ls_cache = nothing
+    end
+
+    for i in 1:maxiters
+        if ls_cache === nothing
+            α = true
+        else
+            ls_sol = solve!(ls_cache, xo, δx)
+            α = ls_sol.step_size # Ignores the return code for now
+        end
+
+        @bb @. x = xo + α * δx
+        fx = Utils.eval_f(prob, fx, x)
+        @bb @. δf = fx - fo
+
+        # Termination Checks
+        solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob)
+        solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode)
+
+        Uₚ = selectdim(U, 2, 1:min(η, i - 1))
+        Vᵀₚ = selectdim(Vᵀ, 1, 1:min(η, i - 1))
+
+        vᵀ = rmatvec!!(vᵀ_cache, Tcache, Uₚ, Vᵀₚ, δx)
+        mvec = matvec!!(mat_cache, Tcache, Uₚ, Vᵀₚ, δf)
+        d = dot(vᵀ, δf)
+        @bb @. δx = (δx - mvec) / d
+
+        selectdim(U, 2, mod1(i, η)) .= Utils.safe_vec(δx)
+        selectdim(Vᵀ, 1, mod1(i, η)) .= Utils.safe_vec(vᵀ)
+
+        Uₚ = selectdim(U, 2, 1:min(η, i))
+        Vᵀₚ = selectdim(Vᵀ, 1, 1:min(η, i))
+        δx = matvec!!(δx, Tcache, Uₚ, Vᵀₚ, fx)
+        @bb @. δx *= -1
+
+        @bb copyto!(xo, x)
+        @bb copyto!(fo, fx)
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
+
+# Non-allocating StaticArrays version of SimpleLimitedMemoryBroyden is actually quite
+# finicky, so we'll implement it separately from the generic version
+# Ignore termination_condition. Don't pass things into internal functions
+function internal_static_solve(
+        prob::ImmutableNonlinearProblem{<:SArray}, alg::SimpleLimitedMemoryBroyden,
+        args...; abstol = nothing, maxiters = 1000, kwargs...)
+    x = prob.u0
+    fx = Utils.get_fx(prob, x)
+
+    U, Vᵀ = init_low_rank_jacobian(vec(x), vec(fx), alg.threshold)
+
+    abstol = NonlinearSolveBase.get_tolerance(x, abstol, eltype(x))
+
+    xo, δx, fo, δf = x, -fx, fx, fx
+
+    if alg.linesearch === Val(true)
+        ls_alg = LiFukushimaLineSearch(; nan_maxiters = nothing)
+        ls_cache = init(prob, ls_alg, fx, x)
+    else
+        ls_cache = nothing
+    end
+
+    T = promote_type(eltype(x), eltype(fx))
+    if alg.alpha === nothing
+        fx_norm = L2_NORM(fx)
+        x_norm = L2_NORM(x)
+        init_α = ifelse(fx_norm ≥ 1e-5, max(x_norm, T(true)) / (2 * fx_norm), T(true))
+    else
+        init_α = inv(alg.alpha)
+    end
+
+    converged, res = internal_unrolled_lbroyden_initial_iterations(
+        prob, xo, fo, δx, abstol, U, Vᵀ, alg.threshold, ls_cache, init_α)
+
+    converged && return SciMLBase.build_solution(
+        prob, alg, res.x, res.fx; retcode = ReturnCode.Success)
+
+    xo, fo, δx = res.x, res.fx, res.δx
+
+    for i in 1:(maxiters - SciMLBase._unwrap_val(alg.threshold))
+        if ls_cache === nothing
+            α = true
+        else
+            ls_sol = solve!(ls_cache, xo, δx)
+            α = ls_sol.step_size # Ignores the return code for now
+        end
+
+        x = xo + α * δx
+        fx = Utils.eval_f(prob, fx, x)
+        δf = fx - fo
+
+        maximum(abs, fx) ≤ abstol &&
+            return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+
+        vᵀ = Utils.restructure(x, rmatvec!!(U, Vᵀ, vec(δx), init_α))
+        mvec = Utils.restructure(x, matvec!!(U, Vᵀ, vec(δf), init_α))
+
+        d = dot(vᵀ, δf)
+        δx = @. (δx - mvec) / d
+
+        U = Base.setindex(U, vec(δx), mod1(i, SciMLBase._unwrap_val(alg.threshold)))
+        Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), mod1(i, SciMLBase._unwrap_val(alg.threshold)))
+
+        δx = -Utils.restructure(fx, matvec!!(U, Vᵀ, vec(fx), init_α))
+
+        xo, fo = x, fx
+    end
+
+    return SciMLBase.build_solution(prob, alg, xo, fo; retcode = ReturnCode.MaxIters)
+end
+
+@generated function internal_unrolled_lbroyden_initial_iterations(
+        prob, xo, fo, δx, abstol, U, Vᵀ, ::Val{threshold},
+        ls_cache, init_α) where {threshold}
+    calls = []
+    for i in 1:threshold
+        static_idx, static_idx_p1 = Val(i - 1), Val(i)
+        push!(calls, quote
+            if ls_cache === nothing
+                α = true
+            else
+                ls_sol = solve!(ls_cache, xo, δx)
+                α = ls_sol.step_size # Ignores the return code for now
+            end
+            x = xo .+ α .* δx
+            fx = prob.f(x, prob.p)
+            δf = fx - fo
+
+            maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx)
+
+            Uₚ = first_n_getindex(U, $(static_idx))
+            Vᵀₚ = first_n_getindex(Vᵀ, $(static_idx))
+
+            vᵀ = Utils.restructure(x, rmatvec!!(Uₚ, Vᵀₚ, vec(δx), init_α))
+            mvec = Utils.restructure(x, matvec!!(Uₚ, Vᵀₚ, vec(δf), init_α))
+
+            d = dot(vᵀ, δf)
+            δx = @. (δx - mvec) / d
+
+            U = Base.setindex(U, vec(δx), $(i))
+            Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), $(i))
+
+            Uₚ = first_n_getindex(U, $(static_idx_p1))
+            Vᵀₚ = first_n_getindex(Vᵀ, $(static_idx_p1))
+            δx = -Utils.restructure(fx, matvec!!(Uₚ, Vᵀₚ, vec(fx), init_α))
+
+            x0, fo = x, fx
+        end)
+    end
+    push!(calls, quote
+        # Termination Check
+        maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx)
+
+        return false, (; x, fx, δx)
+    end)
+    return Expr(:block, calls...)
+end
+
+function rmatvec!!(y, xᵀU, U, Vᵀ, x)
+    # xᵀ × (-I + UVᵀ)
+    η = size(U, 2)
+    if η == 0
+        @bb @. y = -x
+        return y
+    end
+    x_ = vec(x)
+    xᵀU_ = view(xᵀU, 1:η)
+    @bb xᵀU_ = transpose(U) × x_
+    @bb y = transpose(Vᵀ) × vec(xᵀU_)
+    @bb @. y -= x
+    return y
+end
+
+rmatvec!!(::Nothing, Vᵀ, x, init_α) = -x .* init_α
+rmatvec!!(U, Vᵀ, x, init_α) = fast_mapTdot(fast_mapdot(x, U), Vᵀ) .- x .* init_α
+
+function matvec!!(y, Vᵀx, U, Vᵀ, x)
+    # (-I + UVᵀ) × x
+    η = size(U, 2)
+    if η == 0
+        @bb @. y = -x
+        return y
+    end
+    x_ = vec(x)
+    Vᵀx_ = view(Vᵀx, 1:η)
+    @bb Vᵀx_ = Vᵀ × x_
+    @bb y = U × vec(Vᵀx_)
+    @bb @. y -= x
+    return y
+end
+
+@inline matvec!!(::Nothing, Vᵀ, x, init_α) = -x .* init_α
+@inline matvec!!(U, Vᵀ, x, init_α) = fast_mapTdot(fast_mapdot(x, Vᵀ), U) .- x .* init_α
+
+function fast_mapdot(x::SVector{S1}, Y::SVector{S2, <:SVector{S1}}) where {S1, S2}
+    return map(Base.Fix1(dot, x), Y)
+end
+@generated function fast_mapTdot(
+        x::SVector{S1}, Y::SVector{S1, <:SVector{S2}}) where {S1, S2}
+    calls = []
+    syms = [gensym("m$(i)") for i in 1:S1]
+    for i in 1:S1
+        push!(calls, :($(syms[i]) = x[$(i)] .* Y[$i]))
+    end
+    push!(calls, :(return .+($(syms...))))
+    return Expr(:block, calls...)
+end
+
+@generated function first_n_getindex(x::SVector{L, T}, ::Val{N}) where {L, T, N}
+    @assert N ≤ L
+    getcalls = ntuple(i -> :(x[$i]), N)
+    N == 0 && return :(return nothing)
+    return :(return SVector{$N, $T}(($(getcalls...))))
+end
+
+lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = safe_similar(x, threshold)
+function lbroyden_threshold_cache(x::StaticArray, ::Val{threshold}) where {threshold}
+    return zeros(MArray{Tuple{threshold}, eltype(x)})
+end
+lbroyden_threshold_cache(::SArray, ::Val{threshold}) where {threshold} = nothing
+
+function init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2},
+        ::Val{threshold}) where {S1, S2, T1, T2, threshold}
+    T = promote_type(T1, T2)
+    fuSize, uSize = Size(fu), Size(u)
+    Vᵀ = MArray{Tuple{threshold, prod(uSize)}, T}(undef)
+    U = MArray{Tuple{prod(fuSize), threshold}, T}(undef)
+    return U, Vᵀ
+end
+@generated function init_low_rank_jacobian(u::SVector{Lu, T1}, fu::SVector{Lfu, T2},
+        ::Val{threshold}) where {Lu, Lfu, T1, T2, threshold}
+    T = promote_type(T1, T2)
+    inner_inits_Vᵀ = [:(zeros(SVector{$Lu, $T})) for i in 1:threshold]
+    inner_inits_U = [:(zeros(SVector{$Lfu, $T})) for i in 1:threshold]
+    return quote
+        Vᵀ = SVector($(inner_inits_Vᵀ...))
+        U = SVector($(inner_inits_U...))
+        return U, Vᵀ
+    end
+end
+function init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold}
+    Vᵀ = safe_similar(u, threshold, length(u))
+    U = safe_similar(u, length(fu), threshold)
+    return U, Vᵀ
+end
diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl
new file mode 100644
index 000000000..ebbb5f9f9
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/raphson.jl
@@ -0,0 +1,63 @@
+"""
+    SimpleNewtonRaphson(autodiff)
+    SimpleNewtonRaphson(; autodiff = nothing)
+
+A low-overhead implementation of Newton-Raphson. This method is non-allocating on scalar
+and static array problems.
+
+!!! note
+
+    As part of the decreased overhead, this method omits some of the higher level error
+    catching of the other methods. Thus, to see better error messages, use one of the other
+    methods like `NewtonRaphson`.
+
+### Keyword Arguments
+
+  - `autodiff`: determines the backend used for the Jacobian. Defaults to  `nothing` (i.e.
+    automatic backend selection). Valid choices include jacobian backends from
+    `DifferentiationInterface.jl`.
+"""
+@kwdef @concrete struct SimpleNewtonRaphson <: AbstractSimpleNonlinearSolveAlgorithm
+    autodiff = nothing
+end
+
+const SimpleGaussNewton = SimpleNewtonRaphson
+
+function SciMLBase.__solve(
+        prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem},
+        alg::SimpleNewtonRaphson, args...;
+        abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
+    x = Utils.maybe_unaliased(prob.u0, alias_u0)
+    fx = Utils.get_fx(prob, x)
+    fx = Utils.eval_f(prob, fx, x)
+
+    iszero(fx) &&
+        return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success)
+
+    abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition, Val(:simple))
+
+    autodiff = SciMLBase.has_jac(prob.f) ? alg.autodiff :
+               NonlinearSolveBase.select_jacobian_autodiff(prob, alg.autodiff)
+
+    @bb xo = similar(x)
+    fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ?
+               safe_similar(fx) : fx
+    jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x)
+    J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache)
+
+    for _ in 1:maxiters
+        @bb copyto!(xo, x)
+        δx = Utils.restructure(x, J \ Utils.safe_vec(fx))
+        @bb x .-= δx
+
+        solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob)
+        solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode)
+
+        fx = Utils.eval_f(prob, fx, x)
+        J = Utils.compute_jacobian!!(J, prob, autodiff, fx_cache, x, jac_cache)
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl
new file mode 100644
index 000000000..32e7a6219
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/trust_region.jl
@@ -0,0 +1,211 @@
+"""
+    SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius = 0.0,
+        initial_trust_radius = 0.0, step_threshold = nothing,
+        shrink_threshold = nothing, expand_threshold = nothing,
+        shrink_factor = 0.25, expand_factor = 2.0, max_shrink_times::Int = 32,
+        nlsolve_update_rule = Val(false))
+
+A low-overhead implementation of a trust-region solver. This method is non-allocating on
+scalar and static array problems.
+
+### Keyword Arguments
+
+  - `autodiff`: determines the backend used for the Jacobian. Defaults to `nothing` (i.e.
+    automatic backend selection). Valid choices include jacobian backends from
+    `DifferentiationInterface.jl`.
+  - `max_trust_radius`: the maximum radius of the trust region. Defaults to
+    `max(norm(f(u0)), maximum(u0) - minimum(u0))`.
+  - `initial_trust_radius`: the initial trust region radius. Defaults to
+    `max_trust_radius / 11`.
+  - `step_threshold`: the threshold for taking a step. In every iteration, the threshold is
+    compared with a value `r`, which is the actual reduction in the objective function divided
+    by the predicted reduction. If `step_threshold > r` the model is not a good approximation,
+    and the step is rejected. Defaults to `0.1`. For more details, see
+    [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
+  - `shrink_threshold`: the threshold for shrinking the trust region radius. In every
+    iteration, the threshold is compared with a value `r` which is the actual reduction in the
+    objective function divided by the predicted reduction. If `shrink_threshold > r` the trust
+    region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see
+    [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4)
+  - `expand_threshold`: the threshold for expanding the trust region radius. If a step is
+    taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also
+    made to see if `expand_threshold < r`. If that is true, the trust region radius is
+    expanded by `expand_factor`. Defaults to `0.75`.
+  - `shrink_factor`: the factor to shrink the trust region radius with if
+    `shrink_threshold > r` (with `r` defined in `shrink_threshold`). Defaults to `0.25`.
+  - `expand_factor`: the factor to expand the trust region radius with if
+    `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`.
+  - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a
+    row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`.
+  - `nlsolve_update_rule`: If set to `Val(true)`, updates the trust region radius using the
+    update rule from NLSolve.jl. Defaults to `Val(false)`. If set to `Val(true)`, few of the
+    radius update parameters -- `step_threshold = 0.05`, `expand_threshold = 0.9`, and
+    `shrink_factor = 0.5` -- have different defaults.
+"""
+@kwdef @concrete struct SimpleTrustRegion <: AbstractSimpleNonlinearSolveAlgorithm
+    autodiff = nothing
+    max_trust_radius = 0.0
+    initial_trust_radius = 0.0
+    step_threshold = 0.0001
+    shrink_threshold = nothing
+    expand_threshold = nothing
+    shrink_factor = nothing
+    expand_factor = 2.0
+    max_shrink_times::Int = 32
+    nlsolve_update_rule = Val(false)
+end
+
+function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleTrustRegion,
+        args...; abstol = nothing, reltol = nothing, maxiters = 1000,
+        alias_u0 = false, termination_condition = nothing, kwargs...)
+    x = Utils.maybe_unaliased(prob.u0, alias_u0)
+    T = eltype(x)
+    Δₘₐₓ = T(alg.max_trust_radius)
+    Δ = T(alg.initial_trust_radius)
+    η₁ = T(alg.step_threshold)
+
+    if alg.shrink_threshold === nothing
+        η₂ = T(ifelse(SciMLBase._unwrap_val(alg.nlsolve_update_rule), 0.05, 0.25))
+    else
+        η₂ = T(alg.shrink_threshold)
+    end
+
+    if alg.expand_threshold === nothing
+        η₃ = T(ifelse(SciMLBase._unwrap_val(alg.nlsolve_update_rule), 0.9, 0.75))
+    else
+        η₃ = T(alg.expand_threshold)
+    end
+
+    if alg.shrink_factor === nothing
+        t₁ = T(ifelse(SciMLBase._unwrap_val(alg.nlsolve_update_rule), 0.5, 0.25))
+    else
+        t₁ = T(alg.shrink_factor)
+    end
+
+    t₂ = T(alg.expand_factor)
+    max_shrink_times = alg.max_shrink_times
+
+    autodiff = SciMLBase.has_jac(prob.f) ? alg.autodiff :
+               NonlinearSolveBase.select_jacobian_autodiff(prob, alg.autodiff)
+
+    fx = Utils.get_fx(prob, x)
+    fx = Utils.eval_f(prob, fx, x)
+    norm_fx = L2_NORM(fx)
+
+    @bb xo = copy(x)
+    fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ?
+               safe_similar(fx) : fx
+    jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x)
+    J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache)
+
+    abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+        prob, abstol, reltol, fx, x, termination_condition, Val(:simple))
+
+    # Set default trust region radius if not specified by user.
+    iszero(Δₘₐₓ) && (Δₘₐₓ = max(L2_NORM(fx), maximum(x) - minimum(x)))
+    if iszero(Δ)
+        if SciMLBase._unwrap_val(alg.nlsolve_update_rule)
+            norm_x = L2_NORM(x)
+            Δ = T(ifelse(norm_x > 0, norm_x, 1))
+        else
+            Δ = T(Δₘₐₓ / 11)
+        end
+    end
+
+    fₖ = 0.5 * norm_fx^2
+    H = transpose(J) * J
+    g = Utils.restructure(x, J' * Utils.safe_vec(fx))
+    shrink_counter = 0
+
+    @bb δsd = copy(x)
+    @bb δN_δsd = copy(x)
+    @bb δN = copy(x)
+    @bb Hδ = copy(x)
+    dogleg_cache = (; δsd, δN_δsd, δN)
+
+    for _ in 1:maxiters
+        # Solve the trust region subproblem.
+        δ = dogleg_method!!(dogleg_cache, J, fx, g, Δ)
+        @bb @. x = xo + δ
+
+        fx = Utils.eval_f(prob, fx, x)
+
+        fₖ₊₁ = L2_NORM(fx)^2 / T(2)
+
+        # Compute the ratio of the actual to predicted reduction.
+        @bb Hδ = H × vec(δ)
+        r = (fₖ₊₁ - fₖ) / (dot(δ, g) + (dot(δ, Hδ) / T(2)))
+
+        # Update the trust region radius.
+        if r ≥ η₂
+            shrink_counter = 0
+        else
+            Δ = t₁ * Δ
+            shrink_counter += 1
+            shrink_counter > max_shrink_times && return SciMLBase.build_solution(
+                prob, alg, x, fx; retcode = ReturnCode.ShrinkThresholdExceeded)
+        end
+
+        if r ≥ η₁
+            # Termination Checks
+            solved, retcode, fx_sol, x_sol = Utils.check_termination(
+                tc_cache, fx, x, xo, prob)
+            solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode)
+
+            # Take the step.
+            @bb copyto!(xo, x)
+
+            J = Utils.compute_jacobian!!(J, prob, autodiff, fx_cache, x, jac_cache)
+            fx = Utils.eval_f(prob, fx, x)
+
+            # Update the trust region radius.
+            if !SciMLBase._unwrap_val(alg.nlsolve_update_rule) && r > η₃
+                Δ = min(t₂ * Δ, Δₘₐₓ)
+            end
+            fₖ = fₖ₊₁
+
+            @bb H = transpose(J) × J
+            @bb g = transpose(J) × vec(fx)
+        end
+
+        if SciMLBase._unwrap_val(alg.nlsolve_update_rule)
+            if r > η₃
+                Δ = t₂ * L2_NORM(δ)
+            elseif r > 0.5
+                Δ = max(Δ, t₂ * L2_NORM(δ))
+            end
+        end
+    end
+
+    return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters)
+end
+
+function dogleg_method!!(cache, J, f, g, Δ)
+    (; δsd, δN_δsd, δN) = cache
+
+    # Compute the Newton step
+    @bb δN .= Utils.restructure(δN, J \ Utils.safe_vec(f))
+    @bb δN .*= -1
+    # Test if the full step is within the trust region
+    (L2_NORM(δN) ≤ Δ) && return δN
+
+    # Calculate Cauchy point, optimum along the steepest descent direction
+    @bb δsd .= g
+    @bb @. δsd *= -1
+    norm_δsd = L2_NORM(δsd)
+
+    if (norm_δsd ≥ Δ)
+        @bb @. δsd *= Δ / norm_δsd
+        return δsd
+    end
+
+    # Find the intersection point on the boundary
+    @bb @. δN_δsd = δN - δsd
+    dot_δN_δsd = dot(δN_δsd, δN_δsd)
+    dot_δsd_δN_δsd = dot(δsd, δN_δsd)
+    dot_δsd = dot(δsd, δsd)
+    fact = dot_δsd_δN_δsd^2 - dot_δN_δsd * (dot_δsd - Δ^2)
+    tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd
+    @bb @. δsd += tau * δN_δsd
+    return δsd
+end
diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl
new file mode 100644
index 000000000..bd7368bd7
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/src/utils.jl
@@ -0,0 +1,231 @@
+module Utils
+
+using ArrayInterface: ArrayInterface
+using ConcreteStructs: @concrete
+using DifferentiationInterface: DifferentiationInterface, Constant
+using FastClosures: @closure
+using LinearAlgebra: LinearAlgebra, I, diagind
+using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem,
+                          AbstractNonlinearTerminationMode,
+                          AbstractSafeNonlinearTerminationMode,
+                          AbstractSafeBestNonlinearTerminationMode
+using SciMLBase: SciMLBase, AbstractNonlinearProblem, NonlinearLeastSquaresProblem,
+                 NonlinearProblem, NonlinearFunction, ReturnCode
+using StaticArraysCore: StaticArray, SArray, SMatrix, SVector
+
+const DI = DifferentiationInterface
+
+const safe_similar = NonlinearSolveBase.Utils.safe_similar
+
+pickchunksize(n::Int) = min(n, 12)
+
+can_dual(::Type{<:Real}) = true
+can_dual(::Type) = false
+
+maybe_unaliased(x::Union{Number, SArray}, ::Bool) = x
+function maybe_unaliased(x::T, alias::Bool) where {T <: AbstractArray}
+    (alias || !ArrayInterface.can_setindex(T)) && return x
+    return copy(x)
+end
+
+# NOTE: This doesn't initialize the `f(x)` but just returns a buffer of the same size
+function get_fx(prob::NonlinearLeastSquaresProblem, x)
+    if SciMLBase.isinplace(prob) && prob.f.resid_prototype === nothing
+        error("Inplace NonlinearLeastSquaresProblem requires a `resid_prototype` to be \
+               specified.")
+    end
+    return get_fx(prob.f, x, prob.p)
+end
+function get_fx(prob::Union{ImmutableNonlinearProblem, NonlinearProblem}, x)
+    return get_fx(prob.f, x, prob.p)
+end
+function get_fx(f::NonlinearFunction, x, p)
+    if SciMLBase.isinplace(f)
+        f.resid_prototype === nothing || return eltype(x).(f.resid_prototype)
+        return safe_similar(x)
+    end
+    return f(x, p)
+end
+
+function eval_f(prob, fx, x)
+    SciMLBase.isinplace(prob) || return prob.f(x, prob.p)
+    prob.f(fx, x, prob.p)
+    return fx
+end
+
+function fixed_parameter_function(prob::AbstractNonlinearProblem)
+    SciMLBase.isinplace(prob) && return @closure (du, u) -> prob.f(du, u, prob.p)
+    return Base.Fix2(prob.f, prob.p)
+end
+
+function identity_jacobian(u::Number, fu::Number, α = true)
+    return convert(promote_type(eltype(u), eltype(fu)), α)
+end
+function identity_jacobian(u, fu, α = true)
+    J = safe_similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u))
+    fill!(J, zero(eltype(J)))
+    J[diagind(J)] .= eltype(J)(α)
+    return J
+end
+function identity_jacobian(u::StaticArray, fu, α = true)
+    return SMatrix{length(fu), length(u), promote_type(eltype(fu), eltype(u))}(I * α)
+end
+
+identity_jacobian!!(J::Number) = one(J)
+function identity_jacobian!!(J::AbstractVector)
+    ArrayInterface.can_setindex(J) || return one.(J)
+    fill!(J, true)
+    return J
+end
+function identity_jacobian!!(J::AbstractMatrix)
+    ArrayInterface.can_setindex(J) || return convert(typeof(J), I)
+    J[diagind(J)] .= true
+    return J
+end
+identity_jacobian!!(::SMatrix{S1, S2, T}) where {S1, S2, T} = SMatrix{S1, S2, T}(I)
+identity_jacobian!!(::SVector{S1, T}) where {S1, T} = ones(SVector{S1, T})
+
+# Termination Conditions
+function check_termination(cache, fx, x, xo, prob)
+    return check_termination(cache, fx, x, xo, prob, cache.mode)
+end
+
+function check_termination(cache, fx, x, xo, _, ::AbstractNonlinearTerminationMode)
+    return cache(fx, x, xo), ReturnCode.Success, fx, x
+end
+function check_termination(cache, fx, x, xo, _, ::AbstractSafeNonlinearTerminationMode)
+    return cache(fx, x, xo), cache.retcode, fx, x
+end
+function check_termination(
+        cache, fx, x, xo, prob, ::AbstractSafeBestNonlinearTerminationMode)
+    if cache(fx, x, xo)
+        x = cache.u
+        if SciMLBase.isinplace(prob)
+            prob.f(fx, x, prob.p)
+        else
+            fx = prob.f(x, prob.p)
+        end
+        return true, cache.retcode, fx, x
+    end
+    return false, ReturnCode.Default, fx, x
+end
+
+restructure(y, x) = ArrayInterface.restructure(y, x)
+restructure(::Number, x::Number) = x
+
+safe_vec(x::AbstractArray) = vec(x)
+safe_vec(x::Number) = x
+
+abstract type AbstractJacobianMode end
+
+struct AnalyticJacobian <: AbstractJacobianMode end
+@concrete struct DIExtras <: AbstractJacobianMode
+    prep
+end
+struct DINoPreparation <: AbstractJacobianMode end
+
+# While we could run prep in other cases, we don't since we need it completely
+# non-allocating for running inside GPU kernels
+function prepare_jacobian(prob, autodiff, _, x::Number)
+    if SciMLBase.has_jac(prob.f) || SciMLBase.has_vjp(prob.f) || SciMLBase.has_jvp(prob.f)
+        return AnalyticJacobian()
+    end
+    return DINoPreparation()
+end
+function prepare_jacobian(prob, autodiff, fx, x)
+    SciMLBase.has_jac(prob.f) && return AnalyticJacobian()
+    if SciMLBase.isinplace(prob.f)
+        return DIExtras(DI.prepare_jacobian(prob.f, fx, autodiff, x, Constant(prob.p)))
+    else
+        x isa SArray && return DINoPreparation()
+        return DIExtras(DI.prepare_jacobian(prob.f, autodiff, x, Constant(prob.p)))
+    end
+end
+
+function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::AnalyticJacobian)
+    if SciMLBase.has_jac(prob.f)
+        return prob.f.jac(x, prob.p)
+    elseif SciMLBase.has_vjp(prob.f)
+        return prob.f.vjp(one(x), x, prob.p)
+    elseif SciMLBase.has_jvp(prob.f)
+        return prob.f.jvp(one(x), x, prob.p)
+    end
+end
+function compute_jacobian!!(_, prob, autodiff, fx, x::Number, extras::DIExtras)
+    return DI.derivative(prob.f, extras.prep, autodiff, x, Constant(prob.p))
+end
+function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::DINoPreparation)
+    return DI.derivative(prob.f, autodiff, x, Constant(prob.p))
+end
+
+function compute_jacobian!!(J, prob, autodiff, fx, x, ::AnalyticJacobian)
+    if J === nothing
+        if SciMLBase.isinplace(prob.f)
+            J = safe_similar(fx, length(fx), length(x))
+            prob.f.jac(J, x, prob.p)
+            return J
+        else
+            return prob.f.jac(x, prob.p)
+        end
+    end
+    if SciMLBase.isinplace(prob.f)
+        prob.f.jac(J, x, prob.p)
+        return J
+    else
+        return prob.f.jac(x, prob.p)
+    end
+end
+
+function compute_jacobian!!(J, prob, autodiff, fx, x, extras::DIExtras)
+    if J === nothing
+        if SciMLBase.isinplace(prob.f)
+            return DI.jacobian(prob.f, fx, extras.prep, autodiff, x, Constant(prob.p))
+        else
+            return DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p))
+        end
+    end
+    if SciMLBase.isinplace(prob.f)
+        DI.jacobian!(prob.f, fx, J, extras.prep, autodiff, x, Constant(prob.p))
+    else
+        if ArrayInterface.can_setindex(J)
+            DI.jacobian!(prob.f, J, extras.prep, autodiff, x, Constant(prob.p))
+        else
+            J = DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p))
+        end
+    end
+    return J
+end
+function compute_jacobian!!(J, prob, autodiff, fx, x, ::DINoPreparation)
+    @assert !SciMLBase.isinplace(prob.f) "This shouldn't happen. Open an issue."
+    J === nothing && return DI.jacobian(prob.f, autodiff, x, Constant(prob.p))
+    if ArrayInterface.can_setindex(J)
+        DI.jacobian!(prob.f, J, autodiff, x, Constant(prob.p))
+    else
+        J = DI.jacobian(prob.f, autodiff, x, Constant(prob.p))
+    end
+    return J
+end
+
+function compute_jacobian_and_hessian(autodiff, prob, _, x::Number)
+    H = DI.second_derivative(prob.f, autodiff, x, Constant(prob.p))
+    fx, J = DI.value_and_derivative(prob.f, autodiff, x, Constant(prob.p))
+    return fx, J, H
+end
+function compute_jacobian_and_hessian(autodiff, prob, fx, x)
+    if SciMLBase.isinplace(prob)
+        jac_fn = @closure (u, p) -> begin
+            du = safe_similar(fx, promote_type(eltype(fx), eltype(u)))
+            return DI.jacobian(prob.f, du, autodiff, u, Constant(p))
+        end
+        J, H = DI.value_and_jacobian(jac_fn, autodiff, x, Constant(prob.p))
+        fx = Utils.eval_f(prob, fx, x)
+        return fx, J, H
+    else
+        jac_fn = @closure (u, p) -> DI.jacobian(prob.f, autodiff, u, Constant(p))
+        J, H = DI.value_and_jacobian(jac_fn, autodiff, x, Constant(prob.p))
+        fx = Utils.eval_f(prob, fx, x)
+        return fx, J, H
+    end
+end
+
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
new file mode 100644
index 000000000..6dd85b94b
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl
@@ -0,0 +1,106 @@
+@testsnippet RobustnessTestSnippet begin
+    using NonlinearProblemLibrary, NonlinearSolveBase, LinearAlgebra, ADTypes
+
+    problems = NonlinearProblemLibrary.problems
+    dicts = NonlinearProblemLibrary.dicts
+
+    function test_on_library(
+            problems, dicts, alg_ops, broken_tests, ϵ = 1e-4; skip_tests = nothing)
+        for (idx, (problem, dict)) in enumerate(zip(problems, dicts))
+            x = dict["start"]
+            res = similar(x)
+            nlprob = NonlinearProblem(problem, copy(x))
+            @testset "$idx: $(dict["title"])" begin
+                for alg in alg_ops
+                    try
+                        sol = solve(nlprob, alg;
+                            termination_condition = AbsNormTerminationMode(norm))
+                        problem(res, sol.u, nothing)
+
+                        skip = skip_tests !== nothing && idx in skip_tests[alg]
+                        if skip
+                            @test_skip norm(res) ≤ ϵ
+                            continue
+                        end
+                        broken = idx in broken_tests[alg] ? true : false
+                        @test norm(res)≤ϵ broken=broken
+                    catch e
+                        @error e
+                        broken = idx in broken_tests[alg] ? true : false
+                        if broken
+                            @test false broken=true
+                        else
+                            @test 1 == 2
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+@testitem "23 Test Problems: SimpleNewtonRaphson" setup=[RobustnessTestSnippet] tags=[:core] begin
+    alg_ops = (SimpleNewtonRaphson(; autodiff = AutoForwardDiff()),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = []
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "23 Test Problems: SimpleHalley" setup=[RobustnessTestSnippet] tags=[:core] begin
+    alg_ops = (SimpleHalley(; autodiff = AutoForwardDiff()),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    if Sys.isapple()
+        broken_tests[alg_ops[1]] = [1, 5, 11, 15, 16, 18]
+    else
+        broken_tests[alg_ops[1]] = [1, 5, 15, 16, 18]
+    end
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "23 Test Problems: SimpleTrustRegion" setup=[RobustnessTestSnippet] tags=[:core] begin
+    alg_ops = (
+        SimpleTrustRegion(; autodiff = AutoForwardDiff()),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true), autodiff = AutoForwardDiff())
+    )
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [3, 15, 16, 21]
+    broken_tests[alg_ops[2]] = [15, 16]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "23 Test Problems: SimpleDFSane" setup=[RobustnessTestSnippet] tags=[:core] begin
+    alg_ops = (SimpleDFSane(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    if Sys.isapple()
+        broken_tests[alg_ops[1]] = [1, 2, 3, 5, 6, 21]
+    else
+        broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21]
+    end
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "23 Test Problems: SimpleBroyden" setup=[RobustnessTestSnippet] tags=[:core] begin
+    alg_ops = (SimpleBroyden(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 5, 11]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
+
+@testitem "23 Test Problems: SimpleKlement" setup=[RobustnessTestSnippet] tags=[:core] begin
+    alg_ops = (SimpleKlement(),)
+
+    broken_tests = Dict(alg => Int[] for alg in alg_ops)
+    broken_tests[alg_ops[1]] = [1, 2, 4, 5, 11, 12, 22]
+
+    test_on_library(problems, dicts, alg_ops, broken_tests)
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
new file mode 100644
index 000000000..449801ad7
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl
@@ -0,0 +1,22 @@
+@testitem "Simple Adjoint Test" tags=[:adjoint] begin
+    using ForwardDiff, ReverseDiff, SciMLSensitivity, Tracker, Zygote, DiffEqBase,
+          SimpleNonlinearSolve
+
+    ff(u, p) = u .^ 2 .- p
+
+    function solve_nlprob(p)
+        prob = NonlinearProblem{false}(ff, [1.0, 2.0], p)
+        sol = solve(prob, SimpleNewtonRaphson())
+        res = sol isa AbstractArray ? sol : sol.u
+        return sum(abs2, res)
+    end
+
+    p = [3.0, 2.0]
+
+    ∂p_zygote = only(Zygote.gradient(solve_nlprob, p))
+    ∂p_forwarddiff = ForwardDiff.gradient(solve_nlprob, p)
+    ∂p_tracker = Tracker.data(only(Tracker.gradient(solve_nlprob, p)))
+    ∂p_reversediff = ReverseDiff.gradient(solve_nlprob, p)
+    @test ∂p_zygote ≈ ∂p_tracker ≈ ∂p_reversediff
+    @test ∂p_zygote ≈ ∂p_forwarddiff ≈ ∂p_tracker ≈ ∂p_reversediff
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl
new file mode 100644
index 000000000..1f8472cf3
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl
@@ -0,0 +1,40 @@
+@itesitem "Allocation Tests" tags=[:alloc_check] begin
+    using SimpleNonlinearSolve, StaticArrays, AllocCheck
+
+    quadratic_f(u, p) = u .* u .- p
+    quadratic_f!(du, u, p) = (du .= u .* u .- p)
+
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleNewtonRaphson(),
+        SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        SimpleBroyden(),
+        SimpleLimitedMemoryBroyden(),
+        SimpleKlement(),
+        SimpleHalley(),
+        SimpleBroyden(; linesearch = Val(true)),
+        SimpleLimitedMemoryBroyden(; linesearch = Val(true))
+    )
+        @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9)
+
+        nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0)
+        nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0)
+
+        try
+            nlsolve(nlprob_scalar, alg)
+            @test true
+        catch e
+            @error e
+            @test false
+        end
+
+        # ForwardDiff allocates for hessian since we don't propagate the chunksize
+        try
+            nlsolve(nlprob_sa, alg)
+            @test true
+        catch e
+            @error e
+            @test false broken=(alg isa SimpleHalley)
+        end
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl
new file mode 100644
index 000000000..6227348a2
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl
@@ -0,0 +1,30 @@
+@testitem "BigFloat Support" tags=[:core] begin
+    using SimpleNonlinearSolve, LinearAlgebra, ADTypes, SciMLBase
+
+    fn_iip = NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p)
+    fn_oop = NonlinearFunction{false}((u, p) -> u .* u .- p)
+
+    u0 = BigFloat[1.0, 1.0, 1.0]
+    prob_iip_bf = NonlinearProblem{true}(fn_iip, u0, BigFloat(2))
+    prob_oop_bf = NonlinearProblem{false}(fn_oop, u0, BigFloat(2))
+
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleNewtonRaphson(; autodiff = AutoForwardDiff()),
+        SimpleBroyden(),
+        SimpleKlement(),
+        SimpleDFSane(),
+        SimpleTrustRegion(; autodiff = AutoForwardDiff()),
+        SimpleLimitedMemoryBroyden(),
+        SimpleHalley(; autodiff = AutoForwardDiff())
+    )
+        sol = solve(prob_oop_bf, alg)
+        @test maximum(abs, sol.resid) < 1e-6
+        @test SciMLBase.successful_retcode(sol.retcode)
+
+        alg isa SimpleHalley && continue
+
+        sol = solve(prob_iip_bf, alg)
+        @test maximum(abs, sol.resid) < 1e-6
+        @test SciMLBase.successful_retcode(sol.retcode)
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl
new file mode 100644
index 000000000..a857e7a17
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl
@@ -0,0 +1,198 @@
+@testitem "ForwardDiff.jl Integration: NonlinearProblem" tags=[:core] begin
+    using ArrayInterface
+    using ForwardDiff, FiniteDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra,
+          Zygote, ReverseDiff, SciMLBase
+    using DifferentiationInterface
+
+    const DI = DifferentiationInterface
+
+    test_f!(du, u, p) = (@. du = u^2 - p)
+    test_f(u, p) = u .^ 2 .- p
+
+    jacobian_f(::Number, p) = 1 / (2 * √p)
+    jacobian_f(::Number, p::Number) = 1 / (2 * √p)
+    jacobian_f(u, p::Number) = one.(u) .* (1 / (2 * √p))
+    jacobian_f(u, p::AbstractArray) = diagm(vec(@. 1 / (2 * √p)))
+
+    @testset "#(nameof(typeof(alg)))" for alg in (
+        SimpleNewtonRaphson(),
+        SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true)),
+        SimpleHalley(),
+        SimpleBroyden(),
+        SimpleKlement(),
+        SimpleDFSane()
+    )
+        us = (
+            2.0,
+            @SVector([1.0, 1.0]),
+            [1.0, 1.0],
+            ones(2, 2),
+            @SArray(ones(2, 2))
+        )
+
+        @testset "Scalar AD" begin
+            for p in 1.0:0.1:100.0, u0 in us
+                sol = solve(NonlinearProblem{false}(test_f, u0, p), alg)
+                if SciMLBase.successful_retcode(sol)
+                    gs = abs.(ForwardDiff.derivative(p) do pᵢ
+                        solve(NonlinearProblem{false}(test_f, u0, pᵢ), alg).u
+                    end)
+                    gs_true = abs.(jacobian_f(u0, p))
+
+                    if !(isapprox(gs, gs_true, atol = 1e-5))
+                        @show sol.retcode, sol.u
+                        @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_gradient=gs true_gradient=gs_true
+                    else
+                        @test abs.(gs)≈abs.(gs_true) atol=1e-5
+                    end
+                end
+            end
+        end
+
+        @testset "Jacobian" begin
+            @testset "$(typeof(u0))" for u0 in us[2:end],
+                p in ([2.0, 1.0], [2.0 1.0; 3.0 4.0])
+
+                if u0 isa AbstractArray && p isa AbstractArray
+                    size(u0) != size(p) && continue
+                end
+
+                @testset for (iip, fn) in ((false, test_f), (true, test_f!))
+                    iip && (u0 isa Number || !ArrayInterface.can_setindex(u0)) && continue
+
+                    sol = solve(NonlinearProblem{iip}(fn, u0, p), alg)
+                    if SciMLBase.successful_retcode(sol)
+                        gs = abs.(ForwardDiff.jacobian(p) do pᵢ
+                            solve(NonlinearProblem{iip}(fn, u0, pᵢ), alg).u
+                        end)
+                        gs_true = abs.(jacobian_f(u0, p))
+
+                        if !(isapprox(gs, gs_true, atol = 1e-5))
+                            @show sol.retcode, sol.u
+                            @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_jacobian=gs true_jacobian=gs_true
+                        else
+                            @test abs.(gs)≈abs.(gs_true) atol=1e-5
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+@testitem "ForwardDiff.jl Integration NonlinearLeastSquaresProblem" tags=[:core] begin
+    using ForwardDiff, FiniteDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra,
+          Zygote, ReverseDiff
+    using DifferentiationInterface
+
+    const DI = DifferentiationInterface
+
+    true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
+
+    θ_true = [1.0, 0.1, 2.0, 0.5]
+    x = [-1.0, -0.5, 0.0, 0.5, 1.0]
+    y_target = true_function(x, θ_true)
+
+    loss_function(θ, p) = true_function(p, θ) .- y_target
+
+    loss_function_jac(θ, p) = ForwardDiff.jacobian(Base.Fix2(loss_function, p), θ)
+
+    loss_function_vjp(v, θ, p) = reshape(vec(v)' * loss_function_jac(θ, p), size(θ))
+
+    function loss_function!(resid, θ, p)
+        ŷ = true_function(p, θ)
+        @. resid = ŷ - y_target
+        return
+    end
+
+    function loss_function_jac!(J, θ, p)
+        J .= ForwardDiff.jacobian(θ -> loss_function(θ, p), θ)
+        return
+    end
+
+    function loss_function_vjp!(vJ, v, θ, p)
+        vec(vJ) .= reshape(vec(v)' * loss_function_jac(θ, p), size(θ))
+        return
+    end
+
+    θ_init = θ_true .+ 0.1
+
+    for alg in (
+        SimpleGaussNewton(),
+        SimpleGaussNewton(; autodiff = AutoForwardDiff()),
+        SimpleGaussNewton(; autodiff = AutoFiniteDiff()),
+        SimpleGaussNewton(; autodiff = AutoReverseDiff())
+    )
+        function obj_1(p)
+            prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, p)
+            sol = solve(prob_oop, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_2(p)
+            ff = NonlinearFunction{false}(
+                loss_function; resid_prototype = zeros(length(y_target)))
+            prob_oop = NonlinearLeastSquaresProblem{false}(ff, θ_init, p)
+            sol = solve(prob_oop, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_3(p)
+            ff = NonlinearFunction{false}(loss_function; vjp = loss_function_vjp)
+            prob_oop = NonlinearLeastSquaresProblem{false}(ff, θ_init, p)
+            sol = solve(prob_oop, alg)
+            return sum(abs2, sol.u)
+        end
+
+        finitediff = DI.gradient(obj_1, AutoFiniteDiff(), x)
+
+        fdiff1 = DI.gradient(obj_1, AutoForwardDiff(), x)
+        fdiff2 = DI.gradient(obj_2, AutoForwardDiff(), x)
+        fdiff3 = DI.gradient(obj_3, AutoForwardDiff(), x)
+
+        @test finitediff≈fdiff1 atol=1e-5
+        @test finitediff≈fdiff2 atol=1e-5
+        @test finitediff≈fdiff3 atol=1e-5
+        @test fdiff1 ≈ fdiff2 ≈ fdiff3
+
+        function obj_4(p)
+            prob_iip = NonlinearLeastSquaresProblem(
+                NonlinearFunction{true}(
+                    loss_function!; resid_prototype = zeros(length(y_target))),
+                θ_init,
+                p)
+            sol = solve(prob_iip, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_5(p)
+            ff = NonlinearFunction{true}(
+                loss_function!; resid_prototype = zeros(length(y_target)),
+                jac = loss_function_jac!)
+            prob_iip = NonlinearLeastSquaresProblem(ff, θ_init, p)
+            sol = solve(prob_iip, alg)
+            return sum(abs2, sol.u)
+        end
+
+        function obj_6(p)
+            ff = NonlinearFunction{true}(
+                loss_function!; resid_prototype = zeros(length(y_target)),
+                vjp = loss_function_vjp!)
+            prob_iip = NonlinearLeastSquaresProblem(ff, θ_init, p)
+            sol = solve(prob_iip, alg)
+            return sum(abs2, sol.u)
+        end
+
+        finitediff = DI.gradient(obj_4, AutoFiniteDiff(), x)
+
+        fdiff4 = DI.gradient(obj_4, AutoForwardDiff(), x)
+        fdiff5 = DI.gradient(obj_5, AutoForwardDiff(), x)
+        fdiff6 = DI.gradient(obj_6, AutoForwardDiff(), x)
+
+        @test finitediff≈fdiff4 atol=1e-5
+        @test finitediff≈fdiff5 atol=1e-5
+        @test finitediff≈fdiff6 atol=1e-5
+        @test fdiff4 ≈ fdiff5 ≈ fdiff6
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
new file mode 100644
index 000000000..17d9a6f42
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl
@@ -0,0 +1,19 @@
+@testitem "Matrix Resizing" tags=[:core] begin
+    ff(u, p) = u .* u .- p
+    u0 = ones(2, 3)
+    p = 2.0
+    vecprob = NonlinearProblem(ff, vec(u0), p)
+    prob = NonlinearProblem(ff, u0, p)
+
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleKlement(),
+        SimpleBroyden(),
+        SimpleNewtonRaphson(),
+        SimpleDFSane(),
+        SimpleLimitedMemoryBroyden(; threshold = Val(2)),
+        SimpleTrustRegion(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true))
+    )
+        @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/qa_tests.jl b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl
new file mode 100644
index 000000000..cef74ac38
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl
@@ -0,0 +1,18 @@
+@testitem "Aqua" tags=[:core] begin
+    using Aqua, SimpleNonlinearSolve
+
+    Aqua.test_all(SimpleNonlinearSolve; piracies = false, ambiguities = false)
+    Aqua.test_piracies(SimpleNonlinearSolve;
+        treat_as_own = [
+            NonlinearProblem, NonlinearLeastSquaresProblem, IntervalNonlinearProblem])
+    Aqua.test_ambiguities(SimpleNonlinearSolve; recursive = false)
+end
+
+@testitem "Explicit Imports" tags=[:core] begin
+    import ReverseDiff, Tracker, StaticArrays, Zygote
+    using ExplicitImports, SimpleNonlinearSolve
+
+    @test check_no_implicit_imports(SimpleNonlinearSolve; skip = (Base, Core)) === nothing
+    @test check_no_stale_explicit_imports(SimpleNonlinearSolve) === nothing
+    @test check_all_qualified_accesses_via_owners(SimpleNonlinearSolve) === nothing
+end
diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
new file mode 100644
index 000000000..b6a39a9e9
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl
@@ -0,0 +1,155 @@
+@testsnippet RootfindTestSnippet begin
+    using StaticArrays, Random, LinearAlgebra, ForwardDiff, NonlinearSolveBase, SciMLBase
+    using ADTypes, PolyesterForwardDiff, Enzyme, ReverseDiff
+
+    quadratic_f(u, p) = u .* u .- p
+    quadratic_f!(du, u, p) = (du .= u .* u .- p)
+
+    function newton_fails(u, p)
+        return 0.010000000000000002 .+
+               10.000000000000002 ./ (1 .+
+                (0.21640425613334457 .+
+                 216.40425613334457 ./ (1 .+
+                  (0.21640425613334457 .+
+                   216.40425613334457 ./ (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^
+                2.0) .- 0.0011552453009332421u .- p
+    end
+
+    const TERMINATION_CONDITIONS = [
+        NormTerminationMode(Base.Fix1(maximum, abs)),
+        RelTerminationMode(),
+        RelNormTerminationMode(Base.Fix1(maximum, abs)),
+        RelNormSafeTerminationMode(Base.Fix1(maximum, abs)),
+        RelNormSafeBestTerminationMode(Base.Fix1(maximum, abs)),
+        AbsTerminationMode(),
+        AbsNormTerminationMode(Base.Fix1(maximum, abs)),
+        AbsNormSafeTerminationMode(Base.Fix1(maximum, abs)),
+        AbsNormSafeBestTerminationMode(Base.Fix1(maximum, abs))
+    ]
+
+    function run_nlsolve_oop(f::F, u0, p = 2.0; solver) where {F}
+        return solve(NonlinearProblem{false}(f, u0, p), solver; abstol = 1e-9)
+    end
+    function run_nlsolve_iip(f!::F, u0, p = 2.0; solver) where {F}
+        return solve(NonlinearProblem{true}(f!, u0, p), solver; abstol = 1e-9)
+    end
+end
+
+@testitem "First Order Methods" setup=[RootfindTestSnippet] tags=[:core] begin
+    for alg in (
+        SimpleNewtonRaphson,
+        SimpleTrustRegion,
+        (; kwargs...) -> SimpleTrustRegion(; kwargs..., nlsolve_update_rule = Val(true))
+    )
+        @testset for autodiff in (
+            AutoForwardDiff(),
+            AutoFiniteDiff(),
+            AutoReverseDiff(),
+            AutoEnzyme(),
+            nothing
+        )
+            @testset "[OOP] u0: $(typeof(u0))" for u0 in (
+                [1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+                sol = run_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff))
+                @test SciMLBase.successful_retcode(sol)
+                @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9
+            end
+
+            @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
+                sol = run_nlsolve_iip(quadratic_f!, u0; solver = alg(; autodiff))
+                @test SciMLBase.successful_retcode(sol)
+                @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9
+            end
+
+            @testset "Termination Condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+                u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
+
+                probN = NonlinearProblem(quadratic_f, u0, 2.0)
+                @test all(solve(
+                    probN, alg(; autodiff = AutoForwardDiff()); termination_condition).u .≈
+                          sqrt(2.0))
+            end
+        end
+    end
+end
+
+@testitem "Second Order Methods" setup=[RootfindTestSnippet] tags=[:core] begin
+    @testset for alg in (
+        SimpleHalley,
+    )
+        @testset for autodiff in (
+            AutoForwardDiff(),
+            AutoFiniteDiff(),
+            AutoReverseDiff(),
+            nothing
+        )
+            @testset "[OOP] u0: $(typeof(u0))" for u0 in (
+                [1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+                sol = run_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff))
+                @test SciMLBase.successful_retcode(sol)
+                @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9
+            end
+        end
+
+        @testset "Termination Condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+            u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
+
+            probN = NonlinearProblem(quadratic_f, u0, 2.0)
+            @test all(solve(
+                probN, alg(; autodiff = AutoForwardDiff()); termination_condition).u .≈
+                      sqrt(2.0))
+        end
+    end
+end
+
+@testitem "Derivative Free Methods" setup=[RootfindTestSnippet] tags=[:core] begin
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleBroyden(),
+        SimpleKlement(),
+        SimpleDFSane(),
+        SimpleLimitedMemoryBroyden(),
+        SimpleBroyden(; linesearch = Val(true)),
+        SimpleLimitedMemoryBroyden(; linesearch = Val(true))
+    )
+        @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
+            sol = run_nlsolve_oop(quadratic_f, u0; solver = alg)
+            @test SciMLBase.successful_retcode(sol)
+            @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9
+        end
+
+        @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],)
+            sol = run_nlsolve_iip(quadratic_f!, u0; solver = alg)
+            @test SciMLBase.successful_retcode(sol)
+            @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9
+        end
+
+        @testset "Termination Condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS,
+            u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0])
+
+            probN = NonlinearProblem(quadratic_f, u0, 2.0)
+            @test all(solve(probN, alg; termination_condition).u .≈ sqrt(2.0))
+        end
+    end
+end
+
+@testitem "Newton Fails" setup=[RootfindTestSnippet] tags=[:core] begin
+    u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0]
+    p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
+
+    @testset "$(nameof(typeof(alg)))" for alg in (
+        SimpleDFSane(),
+        SimpleTrustRegion(),
+        SimpleHalley(),
+        SimpleTrustRegion(; nlsolve_update_rule = Val(true))
+    )
+        sol = run_nlsolve_oop(newton_fails, u0, p; solver = alg)
+        @test SciMLBase.successful_retcode(sol)
+        @test maximum(abs, newton_fails(sol.u, p)) < 1e-9
+    end
+end
+
+@testitem "Kwargs Propagation" setup=[RootfindTestSnippet] tags=[:core] begin
+    prob = NonlinearProblem(quadratic_f, ones(4), 2.0; maxiters = 2)
+    sol = solve(prob, SimpleNewtonRaphson())
+    @test sol.retcode === ReturnCode.MaxIters
+end
diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
new file mode 100644
index 000000000..09fa52304
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl
@@ -0,0 +1,92 @@
+@testitem "Solving on CUDA" tags=[:cuda] begin
+    using StaticArrays, CUDA, SimpleNonlinearSolve, ADTypes
+
+    if CUDA.functional()
+        CUDA.allowscalar(false)
+
+        f(u, p) = u .* u .- 2
+        f!(du, u, p) = (du .= u .* u .- 2)
+
+        @testset "$(nameof(typeof(alg)))" for alg in (
+            SimpleNewtonRaphson(; autodiff = AutoForwardDiff()),
+            SimpleDFSane(),
+            SimpleTrustRegion(; autodiff = AutoForwardDiff()),
+            SimpleTrustRegion(;
+                nlsolve_update_rule = Val(true), autodiff = AutoForwardDiff()),
+            SimpleBroyden(),
+            SimpleLimitedMemoryBroyden(),
+            SimpleKlement(),
+            SimpleHalley(; autodiff = AutoForwardDiff()),
+            SimpleBroyden(; linesearch = Val(true)),
+            SimpleLimitedMemoryBroyden(; linesearch = Val(true))
+        )
+            # Static Arrays
+            u0 = @SVector[1.0f0, 1.0f0]
+            probN = NonlinearProblem{false}(f, u0)
+            sol = solve(probN, alg; abstol = 1.0f-6)
+            @test SciMLBase.successful_retcode(sol)
+            @test maximum(abs, sol.resid) ≤ 1.0f-6
+
+            # Regular Arrays
+            u0 = [1.0, 1.0]
+            probN = NonlinearProblem{false}(f, u0)
+            sol = solve(probN, alg; abstol = 1.0f-6)
+            @test SciMLBase.successful_retcode(sol)
+            @test maximum(abs, sol.resid) ≤ 1.0f-6
+
+            # Regular Arrays Inplace
+            if !(alg isa SimpleHalley)
+                u0 = [1.0, 1.0]
+                probN = NonlinearProblem{true}(f!, u0)
+                sol = solve(probN, alg; abstol = 1.0f-6)
+                @test SciMLBase.successful_retcode(sol)
+                @test maximum(abs, sol.resid) ≤ 1.0f-6
+            end
+        end
+    end
+end
+
+@testitem "CUDA Kernel Launch Test" tags=[:cuda] begin
+    using StaticArrays, CUDA, SimpleNonlinearSolve, ADTypes
+    using NonlinearSolveBase: ImmutableNonlinearProblem
+
+    if CUDA.functional()
+        CUDA.allowscalar(false)
+
+        f(u, p) = u .* u .- p
+
+        function kernel_function(prob, alg)
+            solve(prob, alg)
+            return nothing
+        end
+
+        @testset for u0 in (1.0f0, @SVector[1.0f0, 1.0f0])
+            prob = convert(ImmutableNonlinearProblem, NonlinearProblem{false}(f, u0, 2.0f0))
+
+            @testset "$(nameof(typeof(alg)))" for alg in (
+                SimpleNewtonRaphson(; autodiff = AutoForwardDiff()),
+                SimpleDFSane(),
+                SimpleTrustRegion(; autodiff = AutoForwardDiff()),
+                SimpleTrustRegion(;
+                    nlsolve_update_rule = Val(true), autodiff = AutoForwardDiff()),
+                SimpleBroyden(),
+                SimpleLimitedMemoryBroyden(),
+                SimpleKlement(),
+                SimpleHalley(; autodiff = AutoForwardDiff()),
+                SimpleBroyden(; linesearch = Val(true)),
+                SimpleLimitedMemoryBroyden(; linesearch = Val(true))
+            )
+                @test begin
+                    try
+                        @cuda kernel_function(prob, alg)
+                        @info "Successfully launched kernel for $(alg)."
+                        true
+                    catch err
+                        @error "Kernel Launch failed for $(alg)."
+                        false
+                    end
+                end broken=(alg isa SimpleHalley && u0 isa StaticArray)
+            end
+        end
+    end
+end
diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl
new file mode 100644
index 000000000..a76760dc8
--- /dev/null
+++ b/lib/SimpleNonlinearSolve/test/runtests.jl
@@ -0,0 +1,17 @@
+using TestItemRunner, InteractiveUtils, Pkg, Test
+
+@info sprint(InteractiveUtils.versioninfo)
+
+const GROUP = lowercase(get(ENV, "GROUP", "All"))
+
+(GROUP == "all" || GROUP == "cuda") && Pkg.add(["CUDA"])
+(GROUP == "all" || GROUP == "adjoint") && Pkg.add(["SciMLSensitivity"])
+(GROUP == "all" || GROUP == "alloc_check") && Pkg.add(["AllocCheck"])
+
+@testset "SimpleNonlinearSolve.jl" begin
+    if GROUP == "all"
+        @run_package_tests
+    else
+        @run_package_tests filter = ti -> (Symbol(GROUP) in ti.tags)
+    end
+end
diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl
index 5e2dc5555..60e2f0663 100644
--- a/src/NonlinearSolve.jl
+++ b/src/NonlinearSolve.jl
@@ -1,41 +1,39 @@
 module NonlinearSolve
 
-if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@max_methods"))
-    @eval Base.Experimental.@max_methods 1
-end
-
 using Reexport: @reexport
 using PrecompileTools: @compile_workload, @setup_workload
 
 using ArrayInterface: ArrayInterface, can_setindex, restructure, fast_scalar_indexing,
                       ismutable
+using CommonSolve: solve, init, solve!
 using ConcreteStructs: @concrete
-using DiffEqBase: DiffEqBase, AbstractNonlinearTerminationMode,
-                  AbstractSafeBestNonlinearTerminationMode, AbsNormTerminationMode,
-                  AbsSafeBestTerminationMode, AbsSafeTerminationMode, AbsTerminationMode,
-                  NormTerminationMode, RelNormTerminationMode, RelSafeBestTerminationMode,
-                  RelSafeTerminationMode, RelTerminationMode,
-                  SimpleNonlinearSolveTerminationMode, SteadyStateDiffEqTerminationMode
-using FastBroadcast: @..
+using DiffEqBase: DiffEqBase # Needed for `init` / `solve` dispatches
 using FastClosures: @closure
 using LazyArrays: LazyArrays, ApplyArray, cache
 using LinearAlgebra: LinearAlgebra, ColumnNorm, Diagonal, I, LowerTriangular, Symmetric,
                      UpperTriangular, axpy!, cond, diag, diagind, dot, issuccess, istril,
                      istriu, lu, mul!, norm, pinv, tril!, triu!
-using LineSearch: LineSearch, AbstractLineSearchAlgorithm, AbstractLineSearchCache,
-                  NoLineSearch, RobustNonMonotoneLineSearch
-using LineSearches: LineSearches
-using LinearSolve: LinearSolve, LUFactorization, QRFactorization,
-                   needs_concrete_A, AbstractFactorization,
+using LineSearch: LineSearch, AbstractLineSearchCache, LineSearchesJL, NoLineSearch,
+                  RobustNonMonotoneLineSearch, BackTracking, LiFukushimaLineSearch
+using LinearSolve: LinearSolve, QRFactorization, needs_concrete_A, AbstractFactorization,
                    DefaultAlgorithmChoice, DefaultLinearSolver
 using MaybeInplace: @bb
+using NonlinearSolveBase: NonlinearSolveBase, nonlinearsolve_forwarddiff_solve,
+                          nonlinearsolve_dual_solution, nonlinearsolve_∂f_∂p,
+                          nonlinearsolve_∂f_∂u, L2_NORM, AbsNormTerminationMode,
+                          AbstractNonlinearTerminationMode,
+                          AbstractSafeBestNonlinearTerminationMode,
+                          select_forward_mode_autodiff, select_reverse_mode_autodiff,
+                          select_jacobian_autodiff
 using Printf: @printf
 using Preferences: Preferences, @load_preference, @set_preferences!
 using RecursiveArrayTools: recursivecopy!
-using SciMLBase: AbstractNonlinearAlgorithm, AbstractNonlinearProblem, _unwrap_val,
-                 isinplace, NLStats
+using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, AbstractNonlinearProblem,
+                 _unwrap_val, isinplace, NLStats, NonlinearFunction,
+                 NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode, get_du, step!,
+                 set_u!, LinearProblem, IdentityOperator
 using SciMLOperators: AbstractSciMLOperator
-using Setfield: @set!
+using SimpleNonlinearSolve: SimpleNonlinearSolve
 using StaticArraysCore: StaticArray, SVector, SArray, MArray, Size, SMatrix
 using SymbolicIndexingInterface: SymbolicIndexingInterface, ParameterIndexingProxy,
                                  symbolic_container, parameter_values, state_values, getu,
@@ -45,8 +43,6 @@ using SymbolicIndexingInterface: SymbolicIndexingInterface, ParameterIndexingPro
 using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff,
                AutoPolyesterForwardDiff, AutoZygote, AutoEnzyme, AutoSparse,
                NoSparsityDetector, KnownJacobianSparsityDetector
-using ADTypes: AutoSparseFiniteDiff, AutoSparseForwardDiff, AutoSparsePolyesterForwardDiff,
-               AutoSparseZygote # FIXME: deprecated, remove in future
 using DifferentiationInterface: DifferentiationInterface, Constant
 using FiniteDiff: FiniteDiff
 using ForwardDiff: ForwardDiff, Dual
@@ -55,12 +51,9 @@ using SciMLJacobianOperators: AbstractJacobianOperator, JacobianOperator, VecJac
 
 ## Sparse AD Support
 using SparseArrays: AbstractSparseMatrix, SparseMatrixCSC
-using SparseConnectivityTracer: TracerSparsityDetector # This can be dropped in the next release
 using SparseMatrixColorings: ConstantColoringAlgorithm, GreedyColoringAlgorithm,
                              LargestFirst
 
-@reexport using SciMLBase, SimpleNonlinearSolve
-
 const DI = DifferentiationInterface
 
 const True = Val(true)
@@ -78,7 +71,6 @@ include("descent/damped_newton.jl")
 include("descent/geodesic_acceleration.jl")
 
 include("internal/jacobian.jl")
-include("internal/forward_diff.jl")
 include("internal/linear_solve.jl")
 include("internal/termination.jl")
 include("internal/tracing.jl")
@@ -107,6 +99,8 @@ include("algorithms/extension_algs.jl")
 include("utils.jl")
 include("default.jl")
 
+include("internal/forward_diff.jl") # we need to define after the algorithms
+
 @setup_workload begin
     nlfuncs = ((NonlinearFunction{false}((u, p) -> u .* u .- p), 0.1),
         (NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p), [0.1]))
@@ -115,11 +109,18 @@ include("default.jl")
         push!(probs_nls, NonlinearProblem(fn, u0, 2.0))
     end
 
-    nls_algs = (NewtonRaphson(), TrustRegion(), LevenbergMarquardt(),
-        PseudoTransient(), Broyden(), Klement(), DFSane(), nothing)
+    nls_algs = (
+        NewtonRaphson(),
+        TrustRegion(),
+        LevenbergMarquardt(),
+        Broyden(),
+        Klement(),
+        nothing
+    )
 
     probs_nlls = NonlinearLeastSquaresProblem[]
-    nlfuncs = ((NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]),
+    nlfuncs = (
+        (NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]),
         (NonlinearFunction{false}((u, p) -> vcat(u .* u .- p, u .* u .- p)), [0.1, 0.1]),
         (
             NonlinearFunction{true}(
@@ -128,15 +129,19 @@ include("default.jl")
         (
             NonlinearFunction{true}((du, u, p) -> du .= vcat(u .* u .- p, u .* u .- p),
                 resid_prototype = zeros(4)),
-            [0.1, 0.1]))
+            [0.1, 0.1]
+        )
+    )
     for (fn, u0) in nlfuncs
         push!(probs_nlls, NonlinearLeastSquaresProblem(fn, u0, 2.0))
     end
 
-    nlls_algs = (LevenbergMarquardt(), GaussNewton(), TrustRegion(),
-        LevenbergMarquardt(; linsolve = LUFactorization()),
-        GaussNewton(; linsolve = LUFactorization()),
-        TrustRegion(; linsolve = LUFactorization()), nothing)
+    nlls_algs = (
+        LevenbergMarquardt(),
+        GaussNewton(),
+        TrustRegion(),
+        nothing
+    )
 
     @compile_workload begin
         @sync begin
@@ -156,6 +161,9 @@ include("default.jl")
     end
 end
 
+# Rexexports
+@reexport using SciMLBase, SimpleNonlinearSolve, NonlinearSolveBase
+
 # Core Algorithms
 export NewtonRaphson, PseudoTransient, Klement, Broyden, LimitedMemoryBroyden, DFSane
 export GaussNewton, LevenbergMarquardt, TrustRegion
@@ -174,26 +182,16 @@ export NewtonDescent, SteepestDescent, Dogleg, DampedNewtonDescent, GeodesicAcce
 
 # Globalization
 ## Line Search Algorithms
-export LineSearchesJL, LiFukushimaLineSearch # FIXME: deprecated. use LineSearch.jl directly
-export Static, HagerZhang, MoreThuente, StrongWolfe, BackTracking  # FIXME: deprecated
-export LineSearch, NoLineSearch, RobustNonMonotoneLineSearch
+export LineSearch, BackTracking, NoLineSearch, RobustNonMonotoneLineSearch,
+       LiFukushimaLineSearch, LineSearchesJL
 ## Trust Region Algorithms
 export RadiusUpdateSchemes
 
-# Export the termination conditions from DiffEqBase
-export SteadyStateDiffEqTerminationMode, SimpleNonlinearSolveTerminationMode,
-       NormTerminationMode, RelTerminationMode, RelNormTerminationMode, AbsTerminationMode,
-       AbsNormTerminationMode, RelSafeTerminationMode, AbsSafeTerminationMode,
-       RelSafeBestTerminationMode, AbsSafeBestTerminationMode
-
 # Tracing Functionality
 export TraceAll, TraceMinimal, TraceWithJacobianConditionNumber
 
 # Reexport ADTypes
 export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff, AutoZygote, AutoEnzyme,
        AutoSparse
-# FIXME: deprecated, remove in future
-export AutoSparseFiniteDiff, AutoSparseForwardDiff, AutoSparsePolyesterForwardDiff,
-       AutoSparseZygote
 
-end # module
+end
diff --git a/src/abstract_types.jl b/src/abstract_types.jl
index 0bb2e13d2..255c5e541 100644
--- a/src/abstract_types.jl
+++ b/src/abstract_types.jl
@@ -263,8 +263,7 @@ Abstract Type for Damping Functions in DampedNewton.
 ```julia
 __internal_init(
     prob::AbstractNonlinearProblem, f::AbstractDampingFunction, initial_damping,
-    J, fu, u, args...; internal_norm = DEFAULT_NORM, kwargs...) -->
-AbstractDampingFunctionCache
+    J, fu, u, args...; internal_norm = L2_NORM, kwargs...) --> AbstractDampingFunctionCache
 ```
 
 Returns a [`AbstractDampingFunctionCache`](@ref).
@@ -348,7 +347,7 @@ Abstract Type for all Jacobian Initialization Algorithms used in NonlinearSolve.
 ```julia
 __internal_init(
     prob::AbstractNonlinearProblem, alg::AbstractJacobianInitialization, solver,
-    f::F, fu, u, p; linsolve = missing, internalnorm::IN = DEFAULT_NORM, kwargs...)
+    f::F, fu, u, p; linsolve = missing, internalnorm::IN = L2_NORM, kwargs...)
 ```
 
 Returns a [`NonlinearSolve.InitializedApproximateJacobianCache`](@ref).
@@ -382,8 +381,8 @@ Abstract Type for all Approximate Jacobian Update Rules used in NonlinearSolve.j
 
 ```julia
 __internal_init(
-    prob::AbstractNonlinearProblem, alg::AbstractApproximateJacobianUpdateRule, J,
-    fu, u, du, args...; internalnorm::F = DEFAULT_NORM, kwargs...) where {F} -->
+    prob::AbstractNonlinearProblem, alg::AbstractApproximateJacobianUpdateRule, J, fu, u,
+    du, args...; internalnorm::F = L2_NORM, kwargs...) where {F} -->
 AbstractApproximateJacobianUpdateRuleCache{INV}
 ```
 """
@@ -440,9 +439,8 @@ Abstract Type for all Trust Region Methods used in NonlinearSolve.jl.
 
 ```julia
 __internal_init(
-    prob::AbstractNonlinearProblem, alg::AbstractTrustRegionMethod, f::F, fu, u,
-    p, args...; internalnorm::IF = DEFAULT_NORM, kwargs...) where {F, IF} -->
-AbstractTrustRegionMethodCache
+    prob::AbstractNonlinearProblem, alg::AbstractTrustRegionMethod, f::F, fu, u, p, args...;
+    internalnorm::IF = L2_NORM, kwargs...) where {F, IF} --> AbstractTrustRegionMethodCache
 ```
 """
 abstract type AbstractTrustRegionMethod end
diff --git a/src/algorithms/broyden.jl b/src/algorithms/broyden.jl
index 77d2a06c4..39962bc6a 100644
--- a/src/algorithms/broyden.jl
+++ b/src/algorithms/broyden.jl
@@ -1,5 +1,5 @@
 """
-    Broyden(; max_resets::Int = 100, linesearch = NoLineSearch(), reset_tolerance = nothing,
+    Broyden(; max_resets::Int = 100, linesearch = nothing, reset_tolerance = nothing,
         init_jacobian::Val = Val(:identity), autodiff = nothing, alpha = nothing)
 
 An implementation of `Broyden`'s Method [broyden1965class](@cite) with resetting and line
@@ -29,36 +29,37 @@ search.
         problem
 """
 function Broyden(;
-        max_resets = 100, linesearch = NoLineSearch(), reset_tolerance = nothing,
-        init_jacobian::Val{IJ} = Val(:identity), autodiff = nothing,
-        alpha = nothing, update_rule::Val{UR} = Val(:good_broyden)) where {IJ, UR}
-    if IJ === :identity
-        if UR === :diagonal
-            initialization = IdentityInitialization(alpha, DiagonalStructure())
-        else
-            initialization = IdentityInitialization(alpha, FullStructure())
-        end
-    elseif IJ === :true_jacobian
-        initialization = TrueJacobianInitialization(FullStructure(), autodiff)
-    else
-        throw(ArgumentError("`init_jacobian` must be one of `:identity` or \
-                             `:true_jacobian`"))
-    end
+        max_resets = 100, linesearch = nothing, reset_tolerance = nothing,
+        init_jacobian = Val(:identity), autodiff = nothing, alpha = nothing,
+        update_rule = Val(:good_broyden))
+    initialization = broyden_init(init_jacobian, update_rule, autodiff, alpha)
+    update_rule = broyden_update_rule(update_rule)
+    return ApproximateJacobianSolveAlgorithm{
+        init_jacobian isa Val{:true_jacobian}, :Broyden}(;
+        linesearch, descent = NewtonDescent(), update_rule, max_resets, initialization,
+        reinit_rule = NoChangeInStateReset(; reset_tolerance))
+end
 
-    update_rule = if UR === :good_broyden
-        GoodBroydenUpdateRule()
-    elseif UR === :bad_broyden
-        BadBroydenUpdateRule()
-    elseif UR === :diagonal
-        GoodBroydenUpdateRule()
-    else
-        throw(ArgumentError("`update_rule` must be one of `:good_broyden`, `:bad_broyden`, \
-                             or `:diagonal`"))
-    end
+function broyden_init(::Val{:identity}, ::Val{:diagonal}, autodiff, alpha)
+    return IdentityInitialization(alpha, DiagonalStructure())
+end
+function broyden_init(::Val{:identity}, ::Val, autodiff, alpha)
+    IdentityInitialization(alpha, FullStructure())
+end
+function broyden_init(::Val{:true_jacobian}, ::Val, autodiff, alpha)
+    return TrueJacobianInitialization(FullStructure(), autodiff)
+end
+function broyden_init(::Val{IJ}, ::Val{UR}, autodiff, alpha) where {IJ, UR}
+    error("Unknown combination of `init_jacobian = Val($(Meta.quot(IJ)))` and \
+           `update_rule = Val($(Meta.quot(UR)))`. Please choose a valid combination.")
+end
 
-    return ApproximateJacobianSolveAlgorithm{IJ === :true_jacobian, :Broyden}(;
-        linesearch, descent = NewtonDescent(), update_rule, max_resets,
-        initialization, reinit_rule = NoChangeInStateReset(; reset_tolerance))
+broyden_update_rule(::Val{:good_broyden}) = GoodBroydenUpdateRule()
+broyden_update_rule(::Val{:bad_broyden}) = BadBroydenUpdateRule()
+broyden_update_rule(::Val{:diagonal}) = GoodBroydenUpdateRule()
+function broyden_update_rule(::Val{UR}) where {UR}
+    error("Unknown update rule `update_rule = Val($(Meta.quot(UR)))`. Please choose a \
+           valid update rule.")
 end
 
 # Checks for no significant change for `nsteps`
@@ -171,7 +172,7 @@ end
 
 function __internal_init(prob::AbstractNonlinearProblem,
         alg::Union{GoodBroydenUpdateRule, BadBroydenUpdateRule}, J, fu, u,
-        du, args...; internalnorm::F = DEFAULT_NORM, kwargs...) where {F}
+        du, args...; internalnorm::F = L2_NORM, kwargs...) where {F}
     @bb J⁻¹dfu = similar(u)
     @bb dfu = copy(fu)
     if alg isa GoodBroydenUpdateRule || J isa Diagonal
diff --git a/src/algorithms/dfsane.jl b/src/algorithms/dfsane.jl
index b42544055..1ece5f5da 100644
--- a/src/algorithms/dfsane.jl
+++ b/src/algorithms/dfsane.jl
@@ -1,3 +1,4 @@
+# XXX: remove kwargs with unicode
 """
     DFSane(; σ_min = 1 // 10^10, σ_max = 1e10, σ_1 = 1, M::Int = 10, γ = 1 // 10^4,
         τ_min = 1 // 10, τ_max = 1 // 2, n_exp::Int = 2, max_inner_iterations::Int = 100,
diff --git a/src/algorithms/extension_algs.jl b/src/algorithms/extension_algs.jl
index 46ebc8dae..3cf36ca9f 100644
--- a/src/algorithms/extension_algs.jl
+++ b/src/algorithms/extension_algs.jl
@@ -142,39 +142,15 @@ NonlinearLeastSquaresProblem.
     This algorithm is only available if `MINPACK.jl` is installed.
 """
 @concrete struct CMINPACK <: AbstractNonlinearSolveExtensionAlgorithm
-    show_trace::Bool
-    tracing::Bool
     method::Symbol
     autodiff
 end
 
-function CMINPACK(; show_trace = missing, tracing = missing,
-        method::Symbol = :auto, autodiff = missing)
+function CMINPACK(; method::Symbol = :auto, autodiff = missing)
     if Base.get_extension(@__MODULE__, :NonlinearSolveMINPACKExt) === nothing
         error("CMINPACK requires MINPACK.jl to be loaded")
     end
-
-    if show_trace !== missing
-        Base.depwarn(
-            "`show_trace` for CMINPACK has been deprecated and will be removed \
-             in v4. Use the `show_trace` keyword argument via the logging API \
-             https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \
-             instead.", :CMINPACK)
-    else
-        show_trace = false
-    end
-
-    if tracing !== missing
-        Base.depwarn(
-            "`tracing` for CMINPACK has been deprecated and will be removed \
-             in v4. Use the `store_trace` keyword argument via the logging API \
-             https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \
-             instead.", :CMINPACK)
-    else
-        tracing = false
-    end
-
-    return CMINPACK(show_trace, tracing, method, autodiff)
+    return CMINPACK(method, autodiff)
 end
 
 """
@@ -203,6 +179,12 @@ end
     acceleration of the fixed-point iteration xₙ₊₁ = xₙ + beta*f(xₙ), where by default
     beta = 1.
 
+!!! warning
+
+    Line Search Algorithms from [`LineSearch.jl`](https://github.com/SciML/LineSearch.jl)
+    aren't supported by `NLsolveJL`. Instead, use the line search algorithms from
+    [`LineSearches.jl`](https://github.com/JuliaNLSolvers/LineSearches.jl).
+
 ### Submethod Choice
 
 Choices for methods in `NLsolveJL`:
@@ -222,63 +204,26 @@ For more information on these arguments, consult the
 @concrete struct NLsolveJL <: AbstractNonlinearSolveExtensionAlgorithm
     method::Symbol
     autodiff
-    store_trace::Bool
-    extended_trace::Bool
     linesearch
     linsolve
     factor
     autoscale::Bool
     m::Int
     beta
-    show_trace::Bool
 end
 
-function NLsolveJL(; method = :trust_region, autodiff = :central, store_trace = missing,
-        extended_trace = missing, linesearch = LineSearches.Static(),
+function NLsolveJL(; method = :trust_region, autodiff = :central, linesearch = missing,
         linsolve = (x, A, b) -> copyto!(x, A \ b), factor = 1.0,
-        autoscale = true, m = 10, beta = one(Float64), show_trace = missing)
+        autoscale = true, m = 10, beta = one(Float64))
     if Base.get_extension(@__MODULE__, :NonlinearSolveNLsolveExt) === nothing
         error("NLsolveJL requires NLsolve.jl to be loaded")
     end
 
-    if show_trace !== missing
-        Base.depwarn("`show_trace` for NLsolveJL has been deprecated and will be removed \
-                      in v4. Use the `show_trace` keyword argument via the logging API \
-                      https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \
-                      instead.",
-            :NLsolveJL)
-    else
-        show_trace = false
-    end
-
-    if store_trace !== missing
-        Base.depwarn(
-            "`store_trace` for NLsolveJL has been deprecated and will be removed \
-             in v4. Use the `store_trace` keyword argument via the logging API \
-             https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \
-             instead.",
-            :NLsolveJL)
-    else
-        store_trace = false
-    end
-
-    if extended_trace !== missing
-        Base.depwarn(
-            "`extended_trace` for NLsolveJL has been deprecated and will be \
-             removed in v4. Use the `trace_level = TraceAll()` keyword argument \
-             via the logging API \
-             https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ instead.",
-            :NLsolveJL)
-    else
-        extended_trace = false
-    end
-
     if autodiff isa Symbol && autodiff !== :central && autodiff !== :forward
         error("`autodiff` must be `:central` or `:forward`.")
     end
 
-    return NLsolveJL(method, autodiff, store_trace, extended_trace, linesearch,
-        linsolve, factor, autoscale, m, beta, show_trace)
+    return NLsolveJL(method, autodiff, linesearch, linsolve, factor, autoscale, m, beta)
 end
 
 """
@@ -343,25 +288,15 @@ Fixed Point Problems. We allow using this algorithm to solve root finding proble
     stabilize::Bool
     check_obj::Bool
     orders::Vector{Int}
-    time_limit
 end
 
 function SpeedMappingJL(; σ_min = 0.0, stabilize::Bool = false, check_obj::Bool = false,
-        orders::Vector{Int} = [3, 3, 2], time_limit = missing)
+        orders::Vector{Int} = [3, 3, 2])
     if Base.get_extension(@__MODULE__, :NonlinearSolveSpeedMappingExt) === nothing
         error("SpeedMappingJL requires SpeedMapping.jl to be loaded")
     end
 
-    if time_limit !== missing
-        Base.depwarn("`time_limit` keyword argument to `SpeedMappingJL` has been \
-                      deprecated and will be removed in v4. Pass `maxtime = <value>` to \
-                      `SciMLBase.solve`.",
-            :SpeedMappingJL)
-    else
-        time_limit = 1000
-    end
-
-    return SpeedMappingJL(σ_min, stabilize, check_obj, orders, time_limit)
+    return SpeedMappingJL(σ_min, stabilize, check_obj, orders)
 end
 
 """
diff --git a/src/algorithms/gauss_newton.jl b/src/algorithms/gauss_newton.jl
index 54b7193fc..0f4ec5cd9 100644
--- a/src/algorithms/gauss_newton.jl
+++ b/src/algorithms/gauss_newton.jl
@@ -1,14 +1,15 @@
 """
-    GaussNewton(; concrete_jac = nothing, linsolve = nothing, linesearch = NoLineSearch(),
-        precs = DEFAULT_PRECS, adkwargs...)
+    GaussNewton(; concrete_jac = nothing, linsolve = nothing, precs = DEFAULT_PRECS,
+        linesearch = nothing, vjp_autodiff = nothing, autodiff = nothing,
+        jvp_autodiff = nothing)
 
 An advanced GaussNewton implementation with support for efficient handling of sparse
 matrices via colored automatic differentiation and preconditioned linear solvers. Designed
 for large-scale and numerically-difficult nonlinear least squares problems.
 """
 function GaussNewton(; concrete_jac = nothing, linsolve = nothing, precs = DEFAULT_PRECS,
-        linesearch = NoLineSearch(), vjp_autodiff = nothing, autodiff = nothing)
-    descent = NewtonDescent(; linsolve, precs)
-    return GeneralizedFirstOrderAlgorithm(; concrete_jac, name = :GaussNewton, descent,
-        jacobian_ad = autodiff, reverse_ad = vjp_autodiff, linesearch)
+        linesearch = nothing, vjp_autodiff = nothing, autodiff = nothing,
+        jvp_autodiff = nothing)
+    return GeneralizedFirstOrderAlgorithm{concrete_jac, :GaussNewton}(; linesearch,
+        descent = NewtonDescent(; linsolve, precs), autodiff, vjp_autodiff, jvp_autodiff)
 end
diff --git a/src/algorithms/klement.jl b/src/algorithms/klement.jl
index 5e911d8c0..be94efc9c 100644
--- a/src/algorithms/klement.jl
+++ b/src/algorithms/klement.jl
@@ -1,5 +1,5 @@
 """
-    Klement(; max_resets = 100, linsolve = NoLineSearch(), linesearch = nothing,
+    Klement(; max_resets = 100, linsolve = nothing, linesearch = nothing,
         precs = DEFAULT_PRECS, alpha = nothing, init_jacobian::Val = Val(:identity),
         autodiff = nothing)
 
@@ -25,34 +25,31 @@ over this.
         differentiable problems.
 """
 function Klement(; max_resets::Int = 100, linsolve = nothing, alpha = nothing,
-        linesearch = NoLineSearch(), precs = DEFAULT_PRECS,
-        autodiff = nothing, init_jacobian::Val{IJ} = Val(:identity)) where {IJ}
-    if !(linesearch isa AbstractLineSearchAlgorithm)
-        Base.depwarn(
-            "Passing in a `LineSearches.jl` algorithm directly is deprecated. \
-             Please use `LineSearchesJL` instead.", :Klement)
-        linesearch = LineSearchesJL(; method = linesearch)
-    end
-
-    if IJ === :identity
-        initialization = IdentityInitialization(alpha, DiagonalStructure())
-    elseif IJ === :true_jacobian
-        initialization = TrueJacobianInitialization(FullStructure(), autodiff)
-    elseif IJ === :true_jacobian_diagonal
-        initialization = TrueJacobianInitialization(DiagonalStructure(), autodiff)
-    else
-        throw(ArgumentError("`init_jacobian` must be one of `:identity`, `:true_jacobian`, \
-                             or `:true_jacobian_diagonal`"))
-    end
-
-    CJ = IJ === :true_jacobian || IJ === :true_jacobian_diagonal
-
+        linesearch = nothing, precs = DEFAULT_PRECS,
+        autodiff = nothing, init_jacobian::Val = Val(:identity))
+    initialization = klement_init(init_jacobian, autodiff, alpha)
+    CJ = init_jacobian isa Val{:true_jacobian} ||
+         init_jacobian isa Val{:true_jacobian_diagonal}
     return ApproximateJacobianSolveAlgorithm{CJ, :Klement}(;
         linesearch, descent = NewtonDescent(; linsolve, precs),
         update_rule = KlementUpdateRule(),
         reinit_rule = IllConditionedJacobianReset(), max_resets, initialization)
 end
 
+function klement_init(::Val{:identity}, autodiff, alpha)
+    return IdentityInitialization(alpha, DiagonalStructure())
+end
+function klement_init(::Val{:true_jacobian}, autodiff, alpha)
+    return TrueJacobianInitialization(FullStructure(), autodiff)
+end
+function klement_init(::Val{:true_jacobian_diagonal}, autodiff, alpha)
+    return TrueJacobianInitialization(DiagonalStructure(), autodiff)
+end
+function klement_init(::Val{IJ}, autodiff, alpha) where {IJ}
+    error("Unknown `init_jacobian = Val($(Meta.quot(IJ)))`. Please choose a valid \
+           `init_jacobian`.")
+end
+
 # Essentially checks ill conditioned Jacobian
 """
     IllConditionedJacobianReset()
diff --git a/src/algorithms/lbroyden.jl b/src/algorithms/lbroyden.jl
index bead1a057..89df6ece5 100644
--- a/src/algorithms/lbroyden.jl
+++ b/src/algorithms/lbroyden.jl
@@ -1,5 +1,5 @@
 """
-    LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = NoLineSearch(),
+    LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = nothing,
         threshold::Val = Val(10), reset_tolerance = nothing, alpha = nothing)
 
 An implementation of `LimitedMemoryBroyden` [ziani2008autoadaptative](@cite) with resetting
@@ -15,16 +15,13 @@ and line search.
   - `alpha`: The initial Jacobian inverse is set to be `(αI)⁻¹`. Defaults to `nothing`
     which implies `α = max(norm(u), 1) / (2 * norm(fu))`.
 """
-function LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = NoLineSearch(),
+function LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = nothing,
         threshold::Union{Val, Int} = Val(10), reset_tolerance = nothing, alpha = nothing)
     threshold isa Int && (threshold = Val(threshold))
+    initialization = BroydenLowRankInitialization{_unwrap_val(threshold)}(alpha, threshold)
     return ApproximateJacobianSolveAlgorithm{false, :LimitedMemoryBroyden}(; linesearch,
-        descent = NewtonDescent(),
-        update_rule = GoodBroydenUpdateRule(),
-        max_resets,
-        initialization = BroydenLowRankInitialization{_unwrap_val(threshold)}(
-            alpha, threshold),
-        reinit_rule = NoChangeInStateReset(; reset_tolerance))
+        descent = NewtonDescent(), update_rule = GoodBroydenUpdateRule(),
+        max_resets, initialization, reinit_rule = NoChangeInStateReset(; reset_tolerance))
 end
 
 """
@@ -44,7 +41,7 @@ jacobian_initialized_preinverted(::BroydenLowRankInitialization) = true
 function __internal_init(
         prob::AbstractNonlinearProblem, alg::BroydenLowRankInitialization{T},
         solver, f::F, fu, u, p; maxiters = 1000,
-        internalnorm::IN = DEFAULT_NORM, kwargs...) where {T, F, IN}
+        internalnorm::IN = L2_NORM, kwargs...) where {T, F, IN}
     if u isa Number # Use the standard broyden
         return __internal_init(prob, IdentityInitialization(true, FullStructure()),
             solver, f, fu, u, p; maxiters, kwargs...)
diff --git a/src/algorithms/levenberg_marquardt.jl b/src/algorithms/levenberg_marquardt.jl
index 66554ee2d..01896bd14 100644
--- a/src/algorithms/levenberg_marquardt.jl
+++ b/src/algorithms/levenberg_marquardt.jl
@@ -3,7 +3,8 @@
         precs = DEFAULT_PRECS, damping_initial::Real = 1.0, α_geodesic::Real = 0.75,
         damping_increase_factor::Real = 2.0, damping_decrease_factor::Real = 3.0,
         finite_diff_step_geodesic = 0.1, b_uphill::Real = 1.0, autodiff = nothing,
-        min_damping_D::Real = 1e-8, disable_geodesic = Val(false))
+        min_damping_D::Real = 1e-8, disable_geodesic = Val(false), vjp_autodiff = nothing,
+        jvp_autodiff = nothing)
 
 An advanced Levenberg-Marquardt implementation with the improvements suggested in
 [transtrum2012improvements](@citet). Designed for large-scale and numerically-difficult
@@ -31,31 +32,20 @@ For the remaining arguments, see [`GeodesicAcceleration`](@ref) and
 [`NonlinearSolve.LevenbergMarquardtTrustRegion`](@ref) documentations.
 """
 function LevenbergMarquardt(;
-        concrete_jac = missing, linsolve = nothing, precs = DEFAULT_PRECS,
-        damping_initial::Real = 1.0, α_geodesic::Real = 0.75,
-        damping_increase_factor::Real = 2.0, damping_decrease_factor::Real = 3.0,
-        finite_diff_step_geodesic = 0.1, b_uphill::Real = 1.0,
-        autodiff = nothing, min_damping_D::Real = 1e-8, disable_geodesic = False)
-    if concrete_jac !== missing
-        Base.depwarn("The `concrete_jac` keyword argument is deprecated and will be \
-                      removed in v0.4. This kwarg doesn't make sense (and is currently \
-                      ignored) for LM since it needs to materialize the Jacobian to \
-                      compute the Damping Term",
-            :LevenbergMarquardt)
-    end
-
-    descent = DampedNewtonDescent(; linsolve,
-        precs,
-        initial_damping = damping_initial,
+        linsolve = nothing, precs = DEFAULT_PRECS, damping_initial::Real = 1.0,
+        α_geodesic::Real = 0.75, damping_increase_factor::Real = 2.0,
+        damping_decrease_factor::Real = 3.0, finite_diff_step_geodesic = 0.1,
+        b_uphill::Real = 1.0, min_damping_D::Real = 1e-8, disable_geodesic = False,
+        autodiff = nothing, vjp_autodiff = nothing, jvp_autodiff = nothing)
+    descent = DampedNewtonDescent(; linsolve, precs, initial_damping = damping_initial,
         damping_fn = LevenbergMarquardtDampingFunction(
             damping_increase_factor, damping_decrease_factor, min_damping_D))
     if disable_geodesic === False
         descent = GeodesicAcceleration(descent, finite_diff_step_geodesic, α_geodesic)
     end
     trustregion = LevenbergMarquardtTrustRegion(b_uphill)
-    return GeneralizedFirstOrderAlgorithm(;
-        concrete_jac = true, name = :LevenbergMarquardt,
-        trustregion, descent, jacobian_ad = autodiff)
+    return GeneralizedFirstOrderAlgorithm{true, :LevenbergMarquardt}(;
+        trustregion, descent, autodiff, vjp_autodiff, jvp_autodiff)
 end
 
 @concrete struct LevenbergMarquardtDampingFunction <: AbstractDampingFunction
@@ -106,7 +96,7 @@ end
 function __internal_init(
         prob::AbstractNonlinearProblem, f::LevenbergMarquardtDampingFunction,
         initial_damping, J, fu, u, ::Val{NF};
-        internalnorm::F = DEFAULT_NORM, kwargs...) where {F, NF}
+        internalnorm::F = L2_NORM, kwargs...) where {F, NF}
     T = promote_type(eltype(u), eltype(fu))
     DᵀD = __init_diagonal(u, T(f.min_damping))
     if NF
@@ -164,18 +154,16 @@ end
 @inline function __update_LM_diagonal!!(y::Diagonal, x::AbstractMatrix)
     if __can_setindex(y.diag)
         if fast_scalar_indexing(y.diag)
-            @inbounds for i in axes(x, 1)
-                y.diag[i] = max(y.diag[i], x[i, i])
+            @simd for i in axes(x, 1)
+                @inbounds y.diag[i] = max(y.diag[i], x[i, i])
             end
             return y
         else
-            idxs = diagind(x)
-            @.. broadcast=false y.diag=max(y.diag, @view(x[idxs]))
+            y .= max.(y.diag, @view(x[diagind(x)]))
             return y
         end
     else
-        idxs = diagind(x)
-        return Diagonal(@.. broadcast=false max(y.diag, @view(x[idxs])))
+        return Diagonal(max.(y.diag, @view(x[diagind(x)])))
     end
 end
 
diff --git a/src/algorithms/pseudo_transient.jl b/src/algorithms/pseudo_transient.jl
index 0da85dd94..1e6b94763 100644
--- a/src/algorithms/pseudo_transient.jl
+++ b/src/algorithms/pseudo_transient.jl
@@ -1,6 +1,7 @@
 """
     PseudoTransient(; concrete_jac = nothing, linsolve = nothing,
-        linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing)
+        linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing,
+        jvp_autodiff = nothing, vjp_autodiff = nothing)
 
 An implementation of PseudoTransient Method [coffey2003pseudotransient](@cite) that is used
 to solve steady state problems in an accelerated manner. It uses an adaptive time-stepping
@@ -14,13 +15,14 @@ This implementation specifically uses "switched evolution relaxation"
   - `alpha_initial` : the initial pseudo time step. It defaults to `1e-3`. If it is small,
     you are going to need more iterations to converge but it can be more stable.
 """
-function PseudoTransient(; concrete_jac = nothing, linsolve = nothing,
-        linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing,
-        alpha_initial = 1e-3)
+function PseudoTransient(;
+        concrete_jac = nothing, linsolve = nothing, linesearch = nothing,
+        precs = DEFAULT_PRECS, alpha_initial = 1e-3, autodiff = nothing,
+        jvp_autodiff = nothing, vjp_autodiff = nothing)
     descent = DampedNewtonDescent(; linsolve, precs, initial_damping = alpha_initial,
         damping_fn = SwitchedEvolutionRelaxation())
-    return GeneralizedFirstOrderAlgorithm(;
-        concrete_jac, name = :PseudoTransient, linesearch, descent, jacobian_ad = autodiff)
+    return GeneralizedFirstOrderAlgorithm{concrete_jac, :PseudoTransient}(;
+        linesearch, descent, autodiff, vjp_autodiff, jvp_autodiff)
 end
 
 """
@@ -42,18 +44,18 @@ Cache for the [`SwitchedEvolutionRelaxation`](@ref) method.
     internalnorm
 end
 
-function requires_normal_form_jacobian(cache::Union{
+function requires_normal_form_jacobian(::Union{
         SwitchedEvolutionRelaxation, SwitchedEvolutionRelaxationCache})
     return false
 end
-function requires_normal_form_rhs(cache::Union{
+function requires_normal_form_rhs(::Union{
         SwitchedEvolutionRelaxation, SwitchedEvolutionRelaxationCache})
     return false
 end
 
 function __internal_init(
         prob::AbstractNonlinearProblem, f::SwitchedEvolutionRelaxation, initial_damping,
-        J, fu, u, args...; internalnorm::F = DEFAULT_NORM, kwargs...) where {F}
+        J, fu, u, args...; internalnorm::F = L2_NORM, kwargs...) where {F}
     T = promote_type(eltype(u), eltype(fu))
     return SwitchedEvolutionRelaxationCache(
         internalnorm(fu), T(1 / initial_damping), internalnorm)
diff --git a/src/algorithms/raphson.jl b/src/algorithms/raphson.jl
index c6847f54c..e5dee4c91 100644
--- a/src/algorithms/raphson.jl
+++ b/src/algorithms/raphson.jl
@@ -1,14 +1,15 @@
 """
-    NewtonRaphson(; concrete_jac = nothing, linsolve = nothing, linesearch = NoLineSearch(),
-        precs = DEFAULT_PRECS, autodiff = nothing)
+    NewtonRaphson(; concrete_jac = nothing, linsolve = nothing, linesearch = missing,
+        precs = DEFAULT_PRECS, autodiff = nothing, vjp_autodiff = nothing,
+        jvp_autodiff = nothing)
 
 An advanced NewtonRaphson implementation with support for efficient handling of sparse
 matrices via colored automatic differentiation and preconditioned linear solvers. Designed
 for large-scale and numerically-difficult nonlinear systems.
 """
-function NewtonRaphson(; concrete_jac = nothing, linsolve = nothing,
-        linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing)
-    descent = NewtonDescent(; linsolve, precs)
-    return GeneralizedFirstOrderAlgorithm(;
-        concrete_jac, name = :NewtonRaphson, linesearch, descent, jacobian_ad = autodiff)
+function NewtonRaphson(; concrete_jac = nothing, linsolve = nothing, linesearch = nothing,
+        precs = DEFAULT_PRECS, autodiff = nothing, vjp_autodiff = nothing,
+        jvp_autodiff = nothing)
+    return GeneralizedFirstOrderAlgorithm{concrete_jac, :NewtonRaphson}(; linesearch,
+        descent = NewtonDescent(; linsolve, precs), autodiff, vjp_autodiff, jvp_autodiff)
 end
diff --git a/src/algorithms/trust_region.jl b/src/algorithms/trust_region.jl
index d68e2d9ec..dab6843e8 100644
--- a/src/algorithms/trust_region.jl
+++ b/src/algorithms/trust_region.jl
@@ -4,7 +4,8 @@
         initial_trust_radius::Real = 0 // 1, step_threshold::Real = 1 // 10000,
         shrink_threshold::Real = 1 // 4, expand_threshold::Real = 3 // 4,
         shrink_factor::Real = 1 // 4, expand_factor::Real = 2 // 1,
-        max_shrink_times::Int = 32, vjp_autodiff = nothing, autodiff = nothing)
+        max_shrink_times::Int = 32,
+        vjp_autodiff = nothing, autodiff = nothing, jvp_autodiff = nothing)
 
 An advanced TrustRegion implementation with support for efficient handling of sparse
 matrices via colored automatic differentiation and preconditioned linear solvers. Designed
@@ -24,22 +25,12 @@ function TrustRegion(; concrete_jac = nothing, linsolve = nothing, precs = DEFAU
         initial_trust_radius::Real = 0 // 1, step_threshold::Real = 1 // 10000,
         shrink_threshold::Real = 1 // 4, expand_threshold::Real = 3 // 4,
         shrink_factor::Real = 1 // 4, expand_factor::Real = 2 // 1,
-        max_shrink_times::Int = 32, autodiff = nothing, vjp_autodiff = nothing)
+        max_shrink_times::Int = 32,
+        autodiff = nothing, vjp_autodiff = nothing, jvp_autodiff = nothing)
     descent = Dogleg(; linsolve, precs)
-    if autodiff !== nothing && ADTypes.mode(autodiff) isa ADTypes.ForwardMode
-        forward_ad = autodiff
-    else
-        forward_ad = nothing
-    end
-    if isnothing(vjp_autodiff) &&
-       autodiff isa Union{ADTypes.AutoFiniteDiff, ADTypes.AutoFiniteDifferences}
-        # TODO: why not just ForwardMode?
-        vjp_autodiff = autodiff
-    end
     trustregion = GenericTrustRegionScheme(;
         method = radius_update_scheme, step_threshold, shrink_threshold, expand_threshold,
-        shrink_factor, expand_factor, reverse_ad = vjp_autodiff, forward_ad)
-    return GeneralizedFirstOrderAlgorithm(;
-        concrete_jac, name = :TrustRegion, trustregion, descent,
-        jacobian_ad = autodiff, reverse_ad = vjp_autodiff, max_shrink_times)
+        shrink_factor, expand_factor, initial_trust_radius, max_trust_radius)
+    return GeneralizedFirstOrderAlgorithm{concrete_jac, :TrustRegion}(;
+        trustregion, descent, autodiff, vjp_autodiff, jvp_autodiff, max_shrink_times)
 end
diff --git a/src/core/approximate_jacobian.jl b/src/core/approximate_jacobian.jl
index 6484c0408..b8f479a6d 100644
--- a/src/core/approximate_jacobian.jl
+++ b/src/core/approximate_jacobian.jl
@@ -59,12 +59,6 @@ function ApproximateJacobianSolveAlgorithm{concrete_jac, name}(;
         linesearch = missing, trustregion = missing, descent, update_rule,
         reinit_rule, initialization, max_resets::Int = typemax(Int),
         max_shrink_times::Int = typemax(Int)) where {concrete_jac, name}
-    if linesearch !== missing && !(linesearch isa AbstractLineSearchAlgorithm)
-        Base.depwarn("Passing in a `LineSearches.jl` algorithm directly is deprecated. \
-                      Please use `LineSearchesJL` instead.",
-            :GeneralizedFirstOrderAlgorithm)
-        linesearch = LineSearchesJL(; method = linesearch)
-    end
     return ApproximateJacobianSolveAlgorithm{concrete_jac, name}(
         linesearch, trustregion, descent, update_rule,
         reinit_rule, max_resets, max_shrink_times, initialization)
@@ -154,7 +148,7 @@ function SciMLBase.__init(
         args...; stats = empty_nlstats(), alias_u0 = false, maxtime = nothing,
         maxiters = 1000, abstol = nothing, reltol = nothing,
         linsolve_kwargs = (;), termination_condition = nothing,
-        internalnorm::F = DEFAULT_NORM, kwargs...) where {uType, iip, F}
+        internalnorm::F = L2_NORM, kwargs...) where {uType, iip, F}
     timer = get_timer_output()
     @static_timeit timer "cache construction" begin
         (; f, u0, p) = prob
@@ -168,8 +162,8 @@ function SciMLBase.__init(
         initialization_cache = __internal_init(prob, alg.initialization, alg, f, fu, u, p;
             stats, linsolve, maxiters, internalnorm)
 
-        abstol, reltol, termination_cache = init_termination_cache(
-            prob, abstol, reltol, fu, u, termination_condition)
+        abstol, reltol, termination_cache = NonlinearSolveBase.init_termination_cache(
+            prob, abstol, reltol, fu, u, termination_condition, Val(:regular))
         linsolve_kwargs = merge((; abstol, reltol), linsolve_kwargs)
 
         J = initialization_cache(nothing)
@@ -180,7 +174,10 @@ function SciMLBase.__init(
 
         reinit_rule_cache = __internal_init(alg.reinit_rule, J, fu, u, du)
 
-        if alg.trustregion !== missing && alg.linesearch !== missing
+        has_linesearch = alg.linesearch !== missing && alg.linesearch !== nothing
+        has_trustregion = alg.trustregion !== missing && alg.trustregion !== nothing
+
+        if has_trustregion && has_linesearch
             error("TrustRegion and LineSearch methods are algorithmically incompatible.")
         end
 
@@ -188,7 +185,7 @@ function SciMLBase.__init(
         linesearch_cache = nothing
         trustregion_cache = nothing
 
-        if alg.trustregion !== missing
+        if has_trustregion
             supports_trust_region(alg.descent) || error("Trust Region not supported by \
                                                         $(alg.descent).")
             trustregion_cache = __internal_init(
@@ -196,7 +193,7 @@ function SciMLBase.__init(
             GB = :TrustRegion
         end
 
-        if alg.linesearch !== missing
+        if has_linesearch
             supports_line_search(alg.descent) || error("Line Search not supported by \
                                                         $(alg.descent).")
             linesearch_cache = init(
diff --git a/src/core/generalized_first_order.jl b/src/core/generalized_first_order.jl
index a485c7c65..d5ee3eeab 100644
--- a/src/core/generalized_first_order.jl
+++ b/src/core/generalized_first_order.jl
@@ -1,7 +1,7 @@
 """
     GeneralizedFirstOrderAlgorithm{concrete_jac, name}(; descent, linesearch = missing,
-        trustregion = missing, jacobian_ad = nothing, forward_ad = nothing,
-        reverse_ad = nothing, max_shrink_times::Int = typemax(Int))
+        trustregion = missing, autodiff = nothing, vjp_autodiff = nothing,
+        jvp_autodiff = nothing, max_shrink_times::Int = typemax(Int))
     GeneralizedFirstOrderAlgorithm(; concrete_jac = nothing, name::Symbol = :unknown,
         kwargs...)
 
@@ -11,7 +11,9 @@ common example of this is Newton-Raphson Method.
 First Order here refers to the order of differentiation, and should not be confused with the
 order of convergence.
 
-`trustregion` and `linesearch` cannot be specified together.
+!!! danger
+
+    `trustregion` and `linesearch` cannot be specified together.
 
 ### Keyword Arguments
 
@@ -28,9 +30,10 @@ order of convergence.
     trustregion
     descent
     max_shrink_times::Int
-    jacobian_ad
-    forward_ad
-    reverse_ad
+
+    autodiff
+    vjp_autodiff
+    jvp_autodiff
 end
 
 function __show_algorithm(io::IO, alg::GeneralizedFirstOrderAlgorithm, name, indent)
@@ -38,9 +41,9 @@ function __show_algorithm(io::IO, alg::GeneralizedFirstOrderAlgorithm, name, ind
     __is_present(alg.linesearch) && push!(modifiers, "linesearch = $(alg.linesearch)")
     __is_present(alg.trustregion) && push!(modifiers, "trustregion = $(alg.trustregion)")
     push!(modifiers, "descent = $(alg.descent)")
-    __is_present(alg.jacobian_ad) && push!(modifiers, "jacobian_ad = $(alg.jacobian_ad)")
-    __is_present(alg.forward_ad) && push!(modifiers, "forward_ad = $(alg.forward_ad)")
-    __is_present(alg.reverse_ad) && push!(modifiers, "reverse_ad = $(alg.reverse_ad)")
+    __is_present(alg.autodiff) && push!(modifiers, "autodiff = $(alg.autodiff)")
+    __is_present(alg.vjp_autodiff) && push!(modifiers, "vjp_autodiff = $(alg.vjp_autodiff)")
+    __is_present(alg.jvp_autodiff) && push!(modifiers, "jvp_autodiff = $(alg.jvp_autodiff)")
     spacing = " "^indent * "   "
     spacing_last = " "^indent
     print(io, "$(name)(\n$(spacing)$(join(modifiers, ",\n$(spacing)"))\n$(spacing_last))")
@@ -53,29 +56,11 @@ end
 
 function GeneralizedFirstOrderAlgorithm{concrete_jac, name}(;
         descent, linesearch = missing, trustregion = missing,
-        jacobian_ad = nothing, forward_ad = nothing, reverse_ad = nothing,
+        autodiff = nothing, jvp_autodiff = nothing, vjp_autodiff = nothing,
         max_shrink_times::Int = typemax(Int)) where {concrete_jac, name}
-    forward_ad = ifelse(forward_ad !== nothing,
-        forward_ad,
-        ifelse(
-            jacobian_ad !== nothing && ADTypes.mode(jacobian_ad) isa ADTypes.ForwardMode,
-            jacobian_ad, nothing))
-    reverse_ad = ifelse(reverse_ad !== nothing,
-        reverse_ad,
-        ifelse(
-            jacobian_ad !== nothing && ADTypes.mode(jacobian_ad) isa ADTypes.ReverseMode,
-            jacobian_ad, nothing))
-
-    if linesearch !== missing && !(linesearch isa AbstractLineSearchAlgorithm)
-        Base.depwarn("Passing in a `LineSearches.jl` algorithm directly is deprecated. \
-                      Please use `LineSearchesJL` instead.",
-            :GeneralizedFirstOrderAlgorithm)
-        linesearch = LineSearchesJL(; method = linesearch)
-    end
-
     return GeneralizedFirstOrderAlgorithm{concrete_jac, name}(
         linesearch, trustregion, descent, max_shrink_times,
-        jacobian_ad, forward_ad, reverse_ad)
+        autodiff, vjp_autodiff, jvp_autodiff)
 end
 
 concrete_jac(::GeneralizedFirstOrderAlgorithm{CJ}) where {CJ} = CJ
@@ -157,8 +142,24 @@ function SciMLBase.__init(
         prob::AbstractNonlinearProblem{uType, iip}, alg::GeneralizedFirstOrderAlgorithm,
         args...; stats = empty_nlstats(), alias_u0 = false, maxiters = 1000,
         abstol = nothing, reltol = nothing, maxtime = nothing,
-        termination_condition = nothing, internalnorm = DEFAULT_NORM,
+        termination_condition = nothing, internalnorm = L2_NORM,
         linsolve_kwargs = (;), kwargs...) where {uType, iip}
+    autodiff = select_jacobian_autodiff(prob, alg.autodiff)
+    jvp_autodiff = if alg.jvp_autodiff === nothing && alg.autodiff !== nothing &&
+                      (ADTypes.mode(alg.autodiff) isa ADTypes.ForwardMode ||
+                       ADTypes.mode(alg.autodiff) isa ADTypes.ForwardOrReverseMode)
+        select_forward_mode_autodiff(prob, alg.autodiff)
+    else
+        select_forward_mode_autodiff(prob, alg.jvp_autodiff)
+    end
+    vjp_autodiff = if alg.vjp_autodiff === nothing && alg.autodiff !== nothing &&
+                      (ADTypes.mode(alg.autodiff) isa ADTypes.ReverseMode ||
+                       ADTypes.mode(alg.autodiff) isa ADTypes.ForwardOrReverseMode)
+        select_reverse_mode_autodiff(prob, alg.autodiff)
+    else
+        select_reverse_mode_autodiff(prob, alg.vjp_autodiff)
+    end
+
     timer = get_timer_output()
     @static_timeit timer "cache construction" begin
         (; f, u0, p) = prob
@@ -168,19 +169,21 @@ function SciMLBase.__init(
 
         linsolve = get_linear_solver(alg.descent)
 
-        abstol, reltol, termination_cache = init_termination_cache(
-            prob, abstol, reltol, fu, u, termination_condition)
+        abstol, reltol, termination_cache = NonlinearSolveBase.init_termination_cache(
+            prob, abstol, reltol, fu, u, termination_condition, Val(:regular))
         linsolve_kwargs = merge((; abstol, reltol), linsolve_kwargs)
 
         jac_cache = JacobianCache(
-            prob, alg, f, fu, u, p; stats, autodiff = alg.jacobian_ad, linsolve,
-            jvp_autodiff = alg.forward_ad, vjp_autodiff = alg.reverse_ad)
+            prob, alg, f, fu, u, p; stats, autodiff, linsolve, jvp_autodiff, vjp_autodiff)
         J = jac_cache(nothing)
         descent_cache = __internal_init(prob, alg.descent, J, fu, u; stats, abstol,
             reltol, internalnorm, linsolve_kwargs, timer)
         du = get_du(descent_cache)
 
-        if alg.trustregion !== missing && alg.linesearch !== missing
+        has_linesearch = alg.linesearch !== missing && alg.linesearch !== nothing
+        has_trustregion = alg.trustregion !== missing && alg.trustregion !== nothing
+
+        if has_trustregion && has_linesearch
             error("TrustRegion and LineSearch methods are algorithmically incompatible.")
         end
 
@@ -188,28 +191,20 @@ function SciMLBase.__init(
         linesearch_cache = nothing
         trustregion_cache = nothing
 
-        if alg.trustregion !== missing
+        if has_trustregion
             supports_trust_region(alg.descent) || error("Trust Region not supported by \
                                                         $(alg.descent).")
             trustregion_cache = __internal_init(
-                prob, alg.trustregion, f, fu, u, p; stats, internalnorm, kwargs...)
+                prob, alg.trustregion, f, fu, u, p; stats, internalnorm, kwargs...,
+                autodiff, jvp_autodiff, vjp_autodiff)
             GB = :TrustRegion
         end
 
-        if alg.linesearch !== missing
+        if has_linesearch
             supports_line_search(alg.descent) || error("Line Search not supported by \
                                                         $(alg.descent).")
-            linesearch_ad = alg.forward_ad === nothing ?
-                            (alg.reverse_ad === nothing ? alg.jacobian_ad :
-                             alg.reverse_ad) : alg.forward_ad
-            if linesearch_ad !== nothing && iip && !DI.check_inplace(linesearch_ad)
-                @warn "$(linesearch_ad) doesn't support in-place problems."
-                linesearch_ad = nothing
-            end
-            linesearch_ad = get_concrete_forward_ad(
-                linesearch_ad, prob, False; check_forward_mode = false)
             linesearch_cache = init(
-                prob, alg.linesearch, fu, u; stats, autodiff = linesearch_ad, kwargs...)
+                prob, alg.linesearch, fu, u; stats, autodiff = jvp_autodiff, kwargs...)
             GB = :LineSearch
         end
 
diff --git a/src/core/spectral_methods.jl b/src/core/spectral_methods.jl
index d141a627b..31e988b70 100644
--- a/src/core/spectral_methods.jl
+++ b/src/core/spectral_methods.jl
@@ -131,8 +131,8 @@ function SciMLBase.__init(prob::AbstractNonlinearProblem, alg::GeneralizedDFSane
 
         linesearch_cache = init(prob, alg.linesearch, fu, u; stats, kwargs...)
 
-        abstol, reltol, tc_cache = init_termination_cache(
-            prob, abstol, reltol, fu, u_cache, termination_condition)
+        abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache(
+            prob, abstol, reltol, fu, u_cache, termination_condition, Val(:regular))
         trace = init_nonlinearsolve_trace(prob, alg, u, fu, nothing, du; kwargs...)
 
         if alg.σ_1 === nothing
diff --git a/src/default.jl b/src/default.jl
index c0924e2ff..a7cc550d8 100644
--- a/src/default.jl
+++ b/src/default.jl
@@ -101,7 +101,7 @@ for (probType, pType) in ((:NonlinearProblem, :NLS), (:NonlinearLeastSquaresProb
     @eval begin
         function SciMLBase.__init(
                 prob::$probType, alg::$algType{N}, args...; stats = empty_nlstats(),
-                maxtime = nothing, maxiters = 1000, internalnorm = DEFAULT_NORM,
+                maxtime = nothing, maxiters = 1000, internalnorm = L2_NORM,
                 alias_u0 = false, verbose = true, kwargs...) where {N}
             if (alias_u0 && !ismutable(prob.u0))
                 verbose && @warn "`alias_u0` has been set to `true`, but `u0` is \
@@ -309,7 +309,7 @@ for (probType, pType) in ((:NonlinearProblem, :NLS), (:NonlinearLeastSquaresProb
 
             push!(calls, quote
                 resids = tuple($(Tuple(resids)...))
-                minfu, idx = __findmin(DEFAULT_NORM, resids)
+                minfu, idx = __findmin(L2_NORM, resids)
             end)
 
             for i in 1:N
@@ -364,7 +364,7 @@ function RobustMultiNewton(::Type{T} = Float64; concrete_jac = nothing, linsolve
                 radius_update_scheme = RadiusUpdateSchemes.Bastin),
             NewtonRaphson(; concrete_jac, linsolve, precs, autodiff),
             NewtonRaphson(; concrete_jac, linsolve, precs,
-                linesearch = LineSearch.BackTracking(), autodiff),
+                linesearch = BackTracking(), autodiff),
             TrustRegion(; concrete_jac, linsolve, precs,
                 radius_update_scheme = RadiusUpdateSchemes.NLsolve, autodiff),
             TrustRegion(; concrete_jac, linsolve, precs,
@@ -405,7 +405,7 @@ function FastShortcutNonlinearPolyalg(
         else
             algs = (NewtonRaphson(; concrete_jac, linsolve, precs, autodiff),
                 NewtonRaphson(; concrete_jac, linsolve, precs,
-                    linesearch = LineSearch.BackTracking(), autodiff),
+                    linesearch = BackTracking(), autodiff),
                 TrustRegion(; concrete_jac, linsolve, precs, autodiff),
                 TrustRegion(; concrete_jac, linsolve, precs,
                     radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff))
@@ -426,7 +426,7 @@ function FastShortcutNonlinearPolyalg(
                     SimpleKlement(),
                     NewtonRaphson(; concrete_jac, linsolve, precs, autodiff),
                     NewtonRaphson(; concrete_jac, linsolve, precs,
-                        linesearch = LineSearch.BackTracking(), autodiff),
+                        linesearch = BackTracking(), autodiff),
                     TrustRegion(; concrete_jac, linsolve, precs,
                         radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff))
             end
@@ -445,7 +445,7 @@ function FastShortcutNonlinearPolyalg(
                     Klement(; linsolve, precs, autodiff),
                     NewtonRaphson(; concrete_jac, linsolve, precs, autodiff),
                     NewtonRaphson(; concrete_jac, linsolve, precs,
-                        linesearch = LineSearch.BackTracking(), autodiff),
+                        linesearch = BackTracking(), autodiff),
                     TrustRegion(; concrete_jac, linsolve, precs, autodiff),
                     TrustRegion(; concrete_jac, linsolve, precs,
                         radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff))
@@ -481,7 +481,7 @@ function FastShortcutNLLSPolyalg(
                 linsolve, precs, disable_geodesic = Val(true), autodiff, kwargs...),
             TrustRegion(; concrete_jac, linsolve, precs, autodiff, kwargs...),
             GaussNewton(; concrete_jac, linsolve, precs,
-                linesearch = LineSearch.BackTracking(), autodiff, kwargs...),
+                linesearch = BackTracking(), autodiff, kwargs...),
             TrustRegion(; concrete_jac, linsolve, precs,
                 radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff, kwargs...),
             LevenbergMarquardt(; linsolve, precs, autodiff, kwargs...))
diff --git a/src/descent/damped_newton.jl b/src/descent/damped_newton.jl
index 77b204b13..ba3e1d028 100644
--- a/src/descent/damped_newton.jl
+++ b/src/descent/damped_newton.jl
@@ -226,12 +226,12 @@ end
     if __can_setindex(J_cache)
         copyto!(J_cache, J)
         if fast_scalar_indexing(J_cache)
-            @inbounds for i in axes(J_cache, 1)
-                J_cache[i, i] += D[i, i]
+            @simd for i in axes(J_cache, 1)
+                @inbounds J_cache[i, i] += D[i, i]
             end
         else
             idxs = diagind(J_cache)
-            @.. broadcast=false @view(J_cache[idxs])=@view(J[idxs]) + @view(D[idxs])
+            J_cache[idxs] .= @view(J[idxs]) .+ @view(D[idxs])
         end
         return J_cache
     else
@@ -242,12 +242,12 @@ end
     if __can_setindex(J_cache)
         copyto!(J_cache, J)
         if fast_scalar_indexing(J_cache)
-            @inbounds for i in axes(J_cache, 1)
-                J_cache[i, i] += D
+            @simd for i in axes(J_cache, 1)
+                @inbounds J_cache[i, i] += D
             end
         else
             idxs = diagind(J_cache)
-            @.. broadcast=false @view(J_cache[idxs])=@view(J[idxs]) + D
+            J_cache[idxs] .= @view(J[idxs]) .+ D
         end
         return J_cache
     else
diff --git a/src/descent/dogleg.jl b/src/descent/dogleg.jl
index 5d0bb1c7c..4c96c98f6 100644
--- a/src/descent/dogleg.jl
+++ b/src/descent/dogleg.jl
@@ -49,7 +49,7 @@ end
 
 function __internal_init(prob::AbstractNonlinearProblem, alg::Dogleg, J, fu, u;
         pre_inverted::Val{INV} = False, linsolve_kwargs = (;),
-        abstol = nothing, reltol = nothing, internalnorm::F = DEFAULT_NORM,
+        abstol = nothing, reltol = nothing, internalnorm::F = L2_NORM,
         shared::Val{N} = Val(1), kwargs...) where {F, INV, N}
     newton_cache = __internal_init(prob, alg.newton_descent, J, fu, u; pre_inverted,
         linsolve_kwargs, abstol, reltol, shared, kwargs...)
diff --git a/src/descent/geodesic_acceleration.jl b/src/descent/geodesic_acceleration.jl
index 136795057..8e8a305f0 100644
--- a/src/descent/geodesic_acceleration.jl
+++ b/src/descent/geodesic_acceleration.jl
@@ -87,7 +87,7 @@ end
 function __internal_init(prob::AbstractNonlinearProblem, alg::GeodesicAcceleration, J,
         fu, u; shared::Val{N} = Val(1), pre_inverted::Val{INV} = False,
         linsolve_kwargs = (;), abstol = nothing, reltol = nothing,
-        internalnorm::F = DEFAULT_NORM, kwargs...) where {INV, N, F}
+        internalnorm::F = L2_NORM, kwargs...) where {INV, N, F}
     T = promote_type(eltype(u), eltype(fu))
     @bb δu = similar(u)
     δus = N ≤ 1 ? nothing : map(2:N) do i
diff --git a/src/globalization/line_search.jl b/src/globalization/line_search.jl
index c7c342bee..7549f1f9d 100644
--- a/src/globalization/line_search.jl
+++ b/src/globalization/line_search.jl
@@ -1,29 +1,3 @@
-LineSearchesJL(method; kwargs...) = LineSearchesJL(; method, kwargs...)
-function LineSearchesJL(; method = LineSearches.Static(), autodiff = nothing, α = true)
-    Base.depwarn("`LineSearchesJL(...)` is deprecated. Please use `LineSearchesJL` from \
-                  LineSearch.jl instead.",
-        :LineSearchesJL)
-
-    # Prevent breaking old code
-    method isa LineSearch.LineSearchesJL &&
-        return LineSearch.LineSearchesJL(method.method, α, autodiff)
-    method isa AbstractLineSearchAlgorithm && return method
-    return LineSearch.LineSearchesJL(method, α, autodiff)
-end
-
-for alg in (:Static, :HagerZhang, :MoreThuente, :BackTracking, :StrongWolfe)
-    depmsg = "`$(alg)(args...; kwargs...)` is deprecated. Please use `LineSearchesJL(; \
-              method = $(alg)(args...; kwargs...))` instead."
-    @eval function $(alg)(args...; autodiff = nothing, initial_alpha = true, kwargs...)
-        Base.depwarn($(depmsg), $(Meta.quot(alg)))
-        return LineSearch.LineSearchesJL(;
-            method = LineSearches.$(alg)(args...; kwargs...), autodiff, initial_alpha)
-    end
-end
-
-Base.@deprecate LiFukushimaLineSearch(; nan_max_iter::Int = 5, kwargs...) LineSearch.LiFukushimaLineSearch(;
-    nan_maxiters = nan_max_iter, kwargs...)
-
 function callback_into_cache!(topcache, cache::AbstractLineSearchCache, args...)
     LineSearch.callback_into_cache!(cache, get_fu(topcache))
 end
diff --git a/src/globalization/trust_region.jl b/src/globalization/trust_region.jl
index e6e2cba17..248f5307c 100644
--- a/src/globalization/trust_region.jl
+++ b/src/globalization/trust_region.jl
@@ -57,7 +57,7 @@ end
 
 function __internal_init(
         prob::AbstractNonlinearProblem, alg::LevenbergMarquardtTrustRegion, f::F, fu,
-        u, p, args...; stats, internalnorm::IF = DEFAULT_NORM, kwargs...) where {F, IF}
+        u, p, args...; stats, internalnorm::IF = L2_NORM, kwargs...) where {F, IF}
     T = promote_type(eltype(u), eltype(fu))
     @bb v = copy(u)
     @bb u_cache = similar(u)
@@ -198,8 +198,7 @@ const RUS = RadiusUpdateSchemes
     GenericTrustRegionScheme(; method = RadiusUpdateSchemes.Simple,
         max_trust_radius = nothing, initial_trust_radius = nothing,
         step_threshold = nothing, shrink_threshold = nothing, expand_threshold = nothing,
-        shrink_factor = nothing, expand_factor = nothing, forward_ad = nothing,
-        reverse_ad = nothing)
+        shrink_factor = nothing, expand_factor = nothing)
 
 Trust Region Method that updates and stores the current trust region radius in
 `trust_region`. For any of the keyword arguments, if the value is `nothing`, then we use
@@ -245,8 +244,6 @@ the value used in the respective paper.
     expand_threshold = nothing
     max_trust_radius = nothing
     initial_trust_radius = nothing
-    forward_ad = nothing
-    reverse_ad = nothing
 end
 
 function Base.show(io::IO, alg::GenericTrustRegionScheme)
@@ -367,7 +364,8 @@ end
 
 function __internal_init(
         prob::AbstractNonlinearProblem, alg::GenericTrustRegionScheme, f::F, fu, u,
-        p, args...; stats, internalnorm::IF = DEFAULT_NORM, kwargs...) where {F, IF}
+        p, args...; stats, internalnorm::IF = L2_NORM, vjp_autodiff = nothing,
+        jvp_autodiff = nothing, kwargs...) where {F, IF}
     T = promote_type(eltype(u), eltype(fu))
     u0_norm = internalnorm(u)
     fu_norm = internalnorm(fu)
@@ -386,13 +384,11 @@ function __internal_init(
     p1, p2, p3, p4 = __get_parameters(T, alg.method)
     ϵ = T(1e-8)
 
-    reverse_ad = get_concrete_reverse_ad(alg.reverse_ad, prob; check_reverse_mode = true)
     vjp_operator = alg.method isa RUS.__Yuan || alg.method isa RUS.__Bastin ?
-                   VecJacOperator(prob, fu, u; autodiff = reverse_ad) : nothing
+                   VecJacOperator(prob, fu, u; autodiff = vjp_autodiff) : nothing
 
-    forward_ad = get_concrete_forward_ad(alg.forward_ad, prob; check_forward_mode = true)
     jvp_operator = alg.method isa RUS.__Bastin ?
-                   JacVecOperator(prob, fu, u; autodiff = forward_ad) : nothing
+                   JacVecOperator(prob, fu, u; autodiff = jvp_autodiff) : nothing
 
     if alg.method isa RUS.__Yuan
         Jᵀfu_cache = StatefulJacobianOperator(vjp_operator, u, prob.p) * _vec(fu)
diff --git a/src/internal/approximate_initialization.jl b/src/internal/approximate_initialization.jl
index 5ce348ae3..a8196c367 100644
--- a/src/internal/approximate_initialization.jl
+++ b/src/internal/approximate_initialization.jl
@@ -20,11 +20,11 @@ end
 function (::DiagonalStructure)(J::AbstractVector, J_new::AbstractMatrix)
     if __can_setindex(J)
         if fast_scalar_indexing(J)
-            @inbounds for i in eachindex(J)
-                J[i] = J_new[i, i]
+            @simd for i in eachindex(J)
+                @inbounds J[i] = J_new[i, i]
             end
         else
-            @.. broadcast=false J=@view(J_new[diagind(J_new)])
+            J .= @view(J_new[diagind(J_new)])
         end
         return J
     end
@@ -65,14 +65,14 @@ end
 
 function __internal_init(
         prob::AbstractNonlinearProblem, alg::IdentityInitialization, solver, f::F,
-        fu, u::Number, p; internalnorm::IN = DEFAULT_NORM, kwargs...) where {F, IN}
+        fu, u::Number, p; internalnorm::IN = L2_NORM, kwargs...) where {F, IN}
     α = __initial_alpha(alg.alpha, u, fu, internalnorm)
     return InitializedApproximateJacobianCache(
         α, alg.structure, alg, nothing, true, internalnorm)
 end
 function __internal_init(prob::AbstractNonlinearProblem, alg::IdentityInitialization,
         solver, f::F, fu::StaticArray, u::StaticArray, p;
-        internalnorm::IN = DEFAULT_NORM, kwargs...) where {IN, F}
+        internalnorm::IN = L2_NORM, kwargs...) where {IN, F}
     α = __initial_alpha(alg.alpha, u, fu, internalnorm)
     if alg.structure isa DiagonalStructure
         @assert length(u)==length(fu) "Diagonal Jacobian Structure must be square!"
@@ -91,7 +91,7 @@ function __internal_init(prob::AbstractNonlinearProblem, alg::IdentityInitializa
 end
 function __internal_init(
         prob::AbstractNonlinearProblem, alg::IdentityInitialization, solver,
-        f::F, fu, u, p; internalnorm::IN = DEFAULT_NORM, kwargs...) where {F, IN}
+        f::F, fu, u, p; internalnorm::IN = L2_NORM, kwargs...) where {F, IN}
     α = __initial_alpha(alg.alpha, u, fu, internalnorm)
     if alg.structure isa DiagonalStructure
         @assert length(u)==length(fu) "Diagonal Jacobian Structure must be square!"
@@ -147,9 +147,8 @@ end
 
 function __internal_init(prob::AbstractNonlinearProblem, alg::TrueJacobianInitialization,
         solver, f::F, fu, u, p; stats, linsolve = missing,
-        internalnorm::IN = DEFAULT_NORM, kwargs...) where {F, IN}
-    autodiff = get_concrete_forward_ad(
-        alg.autodiff, prob; check_forward_mode = false, kwargs...)
+        internalnorm::IN = L2_NORM, kwargs...) where {F, IN}
+    autodiff = select_jacobian_autodiff(prob, alg.autodiff)
     jac_cache = JacobianCache(prob, solver, prob.f, fu, u, p; stats, autodiff, linsolve)
     J = alg.structure(jac_cache(nothing))
     return InitializedApproximateJacobianCache(
diff --git a/src/internal/forward_diff.jl b/src/internal/forward_diff.jl
index 190c80645..1259480b8 100644
--- a/src/internal/forward_diff.jl
+++ b/src/internal/forward_diff.jl
@@ -1,17 +1,25 @@
-# Not part of public API but helps reduce code duplication
-import SimpleNonlinearSolve: __nlsolve_ad, __nlsolve_dual_soln, __nlsolve_∂f_∂p,
-                             __nlsolve_∂f_∂u
+const DualNonlinearProblem = NonlinearProblem{<:Union{Number, <:AbstractArray}, iip,
+    <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}} where {iip, T, V, P}
+const DualNonlinearLeastSquaresProblem = NonlinearLeastSquaresProblem{
+    <:Union{Number, <:AbstractArray}, iip,
+    <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}} where {iip, T, V, P}
+const DualAbstractNonlinearProblem = Union{
+    DualNonlinearProblem, DualNonlinearLeastSquaresProblem}
 
-function SciMLBase.solve(
-        prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip,
-            <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
-        alg::Union{Nothing, AbstractNonlinearAlgorithm},
-        args...;
-        kwargs...) where {T, V, P, iip}
-    sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...)
-    dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p)
-    return SciMLBase.build_solution(
-        prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+for algType in (
+    Nothing, AbstractNonlinearSolveAlgorithm, GeneralizedDFSane,
+    GeneralizedFirstOrderAlgorithm, ApproximateJacobianSolveAlgorithm,
+    LeastSquaresOptimJL, FastLevenbergMarquardtJL, CMINPACK, NLsolveJL, NLSolversJL,
+    SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL,
+    NonlinearSolvePolyAlgorithm{:NLLS, <:Any}, NonlinearSolvePolyAlgorithm{:NLS, <:Any}
+)
+    @eval function SciMLBase.__solve(
+            prob::DualNonlinearProblem, alg::$(algType), args...; kwargs...)
+        sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...)
+        dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p)
+        return SciMLBase.build_solution(
+            prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
+    end
 end
 
 @concrete mutable struct NonlinearSolveForwardDiffCache
@@ -35,17 +43,22 @@ function reinit_cache!(cache::NonlinearSolveForwardDiffCache;
     return cache
 end
 
-function SciMLBase.init(
-        prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip,
-            <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}},
-        alg::Union{Nothing, AbstractNonlinearAlgorithm},
-        args...;
-        kwargs...) where {T, V, P, iip}
-    p = __value(prob.p)
-    newprob = NonlinearProblem(prob.f, __value(prob.u0), p; prob.kwargs...)
-    cache = init(newprob, alg, args...; kwargs...)
-    return NonlinearSolveForwardDiffCache(
-        cache, newprob, alg, prob.p, p, ForwardDiff.partials(prob.p))
+for algType in (
+    Nothing, AbstractNonlinearSolveAlgorithm, GeneralizedDFSane,
+    SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm,
+    GeneralizedFirstOrderAlgorithm, ApproximateJacobianSolveAlgorithm,
+    LeastSquaresOptimJL, FastLevenbergMarquardtJL, CMINPACK, NLsolveJL, NLSolversJL,
+    SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL,
+    NonlinearSolvePolyAlgorithm{:NLLS, <:Any}, NonlinearSolvePolyAlgorithm{:NLS, <:Any}
+)
+    @eval function SciMLBase.__init(
+            prob::DualNonlinearProblem, alg::$(algType), args...; kwargs...)
+        p = __value(prob.p)
+        newprob = NonlinearProblem(prob.f, __value(prob.u0), p; prob.kwargs...)
+        cache = init(newprob, alg, args...; kwargs...)
+        return NonlinearSolveForwardDiffCache(
+            cache, newprob, alg, prob.p, p, ForwardDiff.partials(prob.p))
+    end
 end
 
 function SciMLBase.solve!(cache::NonlinearSolveForwardDiffCache)
@@ -53,10 +66,10 @@ function SciMLBase.solve!(cache::NonlinearSolveForwardDiffCache)
     prob = cache.prob
 
     uu = sol.u
-    f_p = __nlsolve_∂f_∂p(prob, prob.f, uu, cache.values_p)
-    f_x = __nlsolve_∂f_∂u(prob, prob.f, uu, cache.values_p)
+    Jₚ = nonlinearsolve_∂f_∂p(prob, prob.f, uu, cache.values_p)
+    Jᵤ = nonlinearsolve_∂f_∂u(prob, prob.f, uu, cache.values_p)
 
-    z_arr = -f_x \ f_p
+    z_arr = -Jᵤ \ Jₚ
 
     sumfun = ((z, p),) -> map(zᵢ -> zᵢ * ForwardDiff.partials(p), z)
     if cache.p isa Number
@@ -65,7 +78,7 @@ function SciMLBase.solve!(cache::NonlinearSolveForwardDiffCache)
         partials = sum(sumfun, zip(eachcol(z_arr), cache.p))
     end
 
-    dual_soln = __nlsolve_dual_soln(sol.u, partials, cache.p)
+    dual_soln = nonlinearsolve_dual_solution(sol.u, partials, cache.p)
     return SciMLBase.build_solution(
         prob, cache.alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original)
 end
diff --git a/src/internal/helpers.jl b/src/internal/helpers.jl
index 5334f11dc..30e4596bd 100644
--- a/src/internal/helpers.jl
+++ b/src/internal/helpers.jl
@@ -28,65 +28,6 @@ function evaluate_f!!(f::NonlinearFunction{iip}, fu, u, p) where {iip}
     return f(u, p)
 end
 
-# AutoDiff Selection Functions
-function get_concrete_forward_ad(
-        autodiff::ADTypes.AbstractADType, prob, sp::Val{test_sparse} = True,
-        args...; check_forward_mode = true, kwargs...) where {test_sparse}
-    if !isa(ADTypes.mode(autodiff), ADTypes.ForwardMode) && check_forward_mode
-        @warn "$(autodiff)::$(typeof(autodiff)) is not a `ForwardMode`. Use with caution." maxlog=1
-    end
-    return autodiff
-end
-function get_concrete_forward_ad(
-        autodiff, prob, sp::Val{test_sparse} = True, args...; kwargs...) where {test_sparse}
-    if test_sparse
-        (; sparsity, jac_prototype) = prob.f
-        use_sparse_ad = sparsity !== nothing || jac_prototype !== nothing
-    else
-        use_sparse_ad = false
-    end
-    ad = if !ForwardDiff.can_dual(eltype(prob.u0)) # Use Finite Differencing
-        use_sparse_ad ? AutoSparse(AutoFiniteDiff()) : AutoFiniteDiff()
-    else
-        use_sparse_ad ? AutoSparse(AutoForwardDiff()) : AutoForwardDiff()
-    end
-    return ad
-end
-
-function get_concrete_reverse_ad(
-        autodiff::ADTypes.AbstractADType, prob, sp::Val{test_sparse} = True,
-        args...; check_reverse_mode = true, kwargs...) where {test_sparse}
-    if !isa(ADTypes.mode(autodiff), ADTypes.ReverseMode) &&
-       !isa(autodiff, ADTypes.AutoFiniteDiff) && # User specified finite differencing
-       check_reverse_mode
-        @warn "$(autodiff)::$(typeof(autodiff)) is not a `ReverseMode`. Use with caution." maxlog=1
-    end
-    if autodiff isa Union{AutoZygote, AutoSparse{<:AutoZygote}} && isinplace(prob)
-        @warn "Attempting to use Zygote.jl for inplace problems. Switching to FiniteDiff. \
-               Sparsity even if present will be ignored for correctness purposes. Set \
-               the reverse ad option to `nothing` to automatically select the best option \
-               and exploit sparsity."
-        return AutoFiniteDiff() # colorvec confusion will occur if we use FiniteDiff
-    else
-        return autodiff
-    end
-end
-function get_concrete_reverse_ad(
-        autodiff, prob, sp::Val{test_sparse} = True, args...; kwargs...) where {test_sparse}
-    if test_sparse
-        (; sparsity, jac_prototype) = prob.f
-        use_sparse_ad = sparsity !== nothing || jac_prototype !== nothing
-    else
-        use_sparse_ad = false
-    end
-    ad = if isinplace(prob) || !DI.check_available(AutoZygote()) # Use Finite Differencing
-        use_sparse_ad ? AutoSparse(AutoFiniteDiff()) : AutoFiniteDiff()
-    else
-        use_sparse_ad ? AutoSparse(AutoZygote()) : AutoZygote()
-    end
-    return ad
-end
-
 # Callbacks
 """
     callback_into_cache!(cache, internalcache, args...)
@@ -168,9 +109,11 @@ function __construct_extension_f(prob::AbstractNonlinearProblem; alias_u0::Bool
 end
 
 function __construct_extension_jac(prob, alg, u0, fu; can_handle_oop::Val = False,
-        can_handle_scalar::Val = False, kwargs...)
+        can_handle_scalar::Val = False, autodiff = nothing, kwargs...)
+    autodiff = select_jacobian_autodiff(prob, autodiff)
+
     Jₚ = JacobianCache(
-        prob, alg, prob.f, fu, u0, prob.p; stats = empty_nlstats(), kwargs...)
+        prob, alg, prob.f, fu, u0, prob.p; stats = empty_nlstats(), autodiff, kwargs...)
 
     𝓙 = (can_handle_scalar === False && prob.u0 isa Number) ? @closure(u->[Jₚ(u[1])]) : Jₚ
 
diff --git a/src/internal/jacobian.jl b/src/internal/jacobian.jl
index b78eb7383..70cc7021e 100644
--- a/src/internal/jacobian.jl
+++ b/src/internal/jacobian.jl
@@ -56,9 +56,6 @@ function JacobianCache(prob, alg, f::F, fu_, u, p; stats, autodiff = nothing,
 
     @bb fu = similar(fu_)
 
-    autodiff = get_concrete_forward_ad(
-        autodiff, prob, Val(false); check_forward_mode = false)
-
     if !has_analytic_jac && needs_jac
         autodiff = construct_concrete_adtype(f, autodiff)
         di_extras = if iip
@@ -71,10 +68,6 @@ function JacobianCache(prob, alg, f::F, fu_, u, p; stats, autodiff = nothing,
     end
 
     J = if !needs_jac
-        jvp_autodiff = get_concrete_forward_ad(
-            jvp_autodiff, prob, Val(false); check_forward_mode = true)
-        vjp_autodiff = get_concrete_reverse_ad(
-            vjp_autodiff, prob, Val(false); check_reverse_mode = false)
         JacobianOperator(prob, fu, u; jvp_autodiff, vjp_autodiff)
     else
         if f.jac_prototype === nothing
@@ -109,8 +102,6 @@ function JacobianCache(prob, alg, f::F, ::Number, u::Number, p; stats,
     if SciMLBase.has_jac(f) || SciMLBase.has_vjp(f) || SciMLBase.has_jvp(f)
         return JacobianCache{false}(u, f, fu, u, p, stats, autodiff, nothing)
     end
-    autodiff = get_dense_ad(get_concrete_forward_ad(
-        autodiff, prob; check_forward_mode = false))
     di_extras = DI.prepare_derivative(f, get_dense_ad(autodiff), u, Constant(prob.p))
     return JacobianCache{false}(u, f, fu, u, p, stats, autodiff, di_extras)
 end
@@ -243,17 +234,11 @@ function select_fastest_coloring_algorithm(
     return GreedyColoringAlgorithm(LargestFirst())
 end
 
-function construct_concrete_adtype(f::NonlinearFunction, ad::AutoSparse)
-    Base.depwarn(
-        "Specifying a sparse AD type for Nonlinear Problems is deprecated. \
-         Instead use the `sparsity`, `jac_prototype`, and `colorvec` to specify \
-         the right sparsity pattern and coloring algorithm. Ignoring the sparsity \
-         detection algorithm and coloring algorithm present in $(ad).",
-        :NonlinearSolve)
-    if f.sparsity === nothing && f.jac_prototype === nothing
-        @set! f.sparsity = TracerSparsityDetector()
-    end
-    return construct_concrete_adtype(f, get_dense_ad(ad))
+function construct_concrete_adtype(::NonlinearFunction, ad::AutoSparse)
+    error("Specifying a sparse AD type for Nonlinear Problems was removed in v4. \
+           Instead use the `sparsity`, `jac_prototype`, and `colorvec` to specify \
+           the right sparsity pattern and coloring algorithm. Ignoring the sparsity \
+           detection algorithm and coloring algorithm present in $(ad).")
 end
 
 get_dense_ad(ad) = ad
diff --git a/src/internal/linear_solve.jl b/src/internal/linear_solve.jl
index a0c1ba664..707790ff3 100644
--- a/src/internal/linear_solve.jl
+++ b/src/internal/linear_solve.jl
@@ -149,8 +149,7 @@ function (cache::LinearSolverCache)(;
     if linres.retcode === ReturnCode.Failure
         structured_mat = ArrayInterface.isstructured(cache.lincache.A)
         is_gpuarray = ArrayInterface.device(cache.lincache.A) isa ArrayInterface.GPU
-        if !(cache.linsolve isa QRFactorization{ColumnNorm}) &&
-           !is_gpuarray &&
+        if !(cache.linsolve isa QRFactorization{ColumnNorm}) && !is_gpuarray &&
            !structured_mat
             if verbose
                 @warn "Potential Rank Deficient Matrix Detected. Attempting to solve using \
@@ -177,15 +176,11 @@ function (cache::LinearSolverCache)(;
             return LinearSolveResult(; u = linres.u)
         elseif !(cache.linsolve isa QRFactorization{ColumnNorm})
             if verbose
-                if structured_mat
+                if structured_mat || is_gpuarray
+                    mat_desc = structured_mat ? "Structured" : "GPU"
                     @warn "Potential Rank Deficient Matrix Detected. But Matrix is \
-                           Structured. Currently, we don't attempt to solve Rank Deficient \
-                           Structured Matrices. Please open an issue at \
-                           https://github.com/SciML/NonlinearSolve.jl"
-                elseif is_gpuarray
-                    @warn "Potential Rank Deficient Matrix Detected. But Matrix is on GPU. \
-                           Currently, we don't attempt to solve Rank Deficient GPU \
-                           Matrices. Please open an issue at \
+                           $(mat_desc). Currently, we don't attempt to solve Rank Deficient \
+                           $(mat_desc) Matrices. Please open an issue at \
                            https://github.com/SciML/NonlinearSolve.jl"
                 end
             end
diff --git a/src/internal/termination.jl b/src/internal/termination.jl
index ef3f7c4c0..7728aea69 100644
--- a/src/internal/termination.jl
+++ b/src/internal/termination.jl
@@ -1,34 +1,9 @@
-function init_termination_cache(prob::NonlinearProblem, abstol, reltol, du, u, ::Nothing)
-    return init_termination_cache(prob, abstol, reltol, du, u,
-        AbsSafeBestTerminationMode(Base.Fix1(maximum, abs); max_stalled_steps = 32))
-end
-function init_termination_cache(
-        prob::NonlinearLeastSquaresProblem, abstol, reltol, du, u, ::Nothing)
-    return init_termination_cache(prob, abstol, reltol, du, u,
-        AbsSafeBestTerminationMode(Base.Fix2(norm, 2); max_stalled_steps = 32))
-end
-
-function init_termination_cache(
-        prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem},
-        abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode)
-    tc_ = if hasfield(typeof(tc), :internalnorm) && tc.internalnorm === nothing
-        internalnorm = ifelse(
-            prob isa NonlinearProblem, Base.Fix1(maximum, abs), Base.Fix2(norm, 2))
-        DiffEqBase.set_termination_mode_internalnorm(tc, internalnorm)
-    else
-        tc
-    end
-    tc_cache = init(du, u, tc_; abstol, reltol, use_deprecated_retcodes = Val(false))
-    return DiffEqBase.get_abstol(tc_cache), DiffEqBase.get_reltol(tc_cache), tc_cache
-end
-
 function check_and_update!(cache, fu, u, uprev)
     return check_and_update!(cache.termination_cache, cache, fu, u, uprev)
 end
 
 function check_and_update!(tc_cache, cache, fu, u, uprev)
-    return check_and_update!(
-        tc_cache, cache, fu, u, uprev, DiffEqBase.get_termination_mode(tc_cache))
+    return check_and_update!(tc_cache, cache, fu, u, uprev, tc_cache.mode)
 end
 
 function check_and_update!(tc_cache, cache, fu, u, uprev, mode)
@@ -40,8 +15,7 @@ function check_and_update!(tc_cache, cache, fu, u, uprev, mode)
 end
 
 function update_from_termination_cache!(tc_cache, cache, u = get_u(cache))
-    return update_from_termination_cache!(
-        tc_cache, cache, DiffEqBase.get_termination_mode(tc_cache), u)
+    return update_from_termination_cache!(tc_cache, cache, tc_cache.mode, u)
 end
 
 function update_from_termination_cache!(
diff --git a/src/utils.jl b/src/utils.jl
index 6ceb4c9d8..069fde86e 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -1,16 +1,8 @@
 # Defaults
-@inline DEFAULT_NORM(args...) = DiffEqBase.NONLINEARSOLVE_DEFAULT_NORM(args...)
 @inline DEFAULT_PRECS(W, du, u, p, t, newW, Plprev, Prprev, cachedata) = nothing, nothing
-@inline DEFAULT_TOLERANCE(args...) = DiffEqBase._get_tolerance(args...)
 
 # Helper  Functions
-@static if VERSION ≤ v"1.10-"
-    @inline @generated function __hasfield(::T, ::Val{field}) where {T, field}
-        return :($(field ∉ fieldnames(T)))
-    end
-else
-    @inline __hasfield(::T, ::Val{field}) where {T, field} = hasfield(T, field)
-end
+@inline __hasfield(::T, ::Val{field}) where {T, field} = hasfield(T, field)
 
 @generated function __getproperty(s::S, ::Val{X}) where {S, X}
     hasfield(S, X) && return :(s.$X)
@@ -86,12 +78,10 @@ LazyArrays.applied_axes(::typeof(__zero), x) = axes(x)
 @inline __is_complex(::Type{T}) where {T} = false
 
 @inline __findmin_caches(f::F, caches) where {F} = __findmin(f ∘ get_fu, caches)
-# FIXME: DEFAULT_NORM makes an Array of NaNs not a NaN (atleast according to `isnan`)
+# FIXME: L2_NORM makes an Array of NaNs not a NaN (atleast according to `isnan`)
 @generated function __findmin(f::F, x) where {F}
     # JET shows dynamic dispatch if this is not written as a generated function
-    if F === typeof(DEFAULT_NORM)
-        return :(return __findmin_impl(Base.Fix1(maximum, abs), x))
-    end
+    F === typeof(L2_NORM) && return :(return __findmin_impl(Base.Fix1(maximum, abs), x))
     return :(return __findmin_impl(f, x))
 end
 @inline @views function __findmin_impl(f::F, x) where {F}
diff --git a/test/core/nlls_tests.jl b/test/core/nlls_tests.jl
index 483107f69..040627c00 100644
--- a/test/core/nlls_tests.jl
+++ b/test/core/nlls_tests.jl
@@ -2,6 +2,14 @@
 using Reexport
 @reexport using NonlinearSolve, LinearSolve, LinearAlgebra, StableRNGs, Random, ForwardDiff,
                 Zygote
+using LineSearches: LineSearches, Static, HagerZhang, MoreThuente, StrongWolfe
+
+linesearches = []
+for ls in (
+    Static(), HagerZhang(), MoreThuente(), StrongWolfe(), LineSearches.BackTracking())
+    push!(linesearches, LineSearchesJL(; method = ls))
+end
+push!(linesearches, BackTracking())
 
 true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])
 true_function(y, x, θ) = (@. y = θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4]))
@@ -29,9 +37,7 @@ solvers = []
 for linsolve in [nothing, LUFactorization(), KrylovJL_GMRES(), KrylovJL_LSMR()]
     vjp_autodiffs = linsolve isa KrylovJL ? [nothing, AutoZygote(), AutoFiniteDiff()] :
                     [nothing]
-    for linesearch in [Static(), BackTracking(), HagerZhang(), StrongWolfe(), MoreThuente()],
-        vjp_autodiff in vjp_autodiffs
-
+    for linesearch in linesearches, vjp_autodiff in vjp_autodiffs
         push!(solvers, GaussNewton(; linsolve, linesearch, vjp_autodiff))
     end
 end
diff --git a/test/core/rootfind_tests.jl b/test/core/rootfind_tests.jl
index cb8d62e99..2d1662570 100644
--- a/test/core/rootfind_tests.jl
+++ b/test/core/rootfind_tests.jl
@@ -1,7 +1,9 @@
 @testsetup module CoreRootfindTesting
 using Reexport
 @reexport using BenchmarkTools, LinearSolve, NonlinearSolve, StaticArrays, Random,
-                LinearAlgebra, ForwardDiff, Zygote, Enzyme, DiffEqBase
+                LinearAlgebra, ForwardDiff, Zygote, Enzyme, SparseConnectivityTracer,
+                NonlinearSolveBase
+using LineSearches: LineSearches
 
 _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x))
 
@@ -20,9 +22,16 @@ function newton_fails(u, p)
 end
 
 const TERMINATION_CONDITIONS = [
-    NormTerminationMode(), RelTerminationMode(), RelNormTerminationMode(),
-    AbsTerminationMode(), AbsNormTerminationMode(), RelSafeTerminationMode(),
-    AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode()]
+    NormTerminationMode(Base.Fix1(maximum, abs)),
+    RelTerminationMode(),
+    RelNormTerminationMode(Base.Fix1(maximum, abs)),
+    RelNormSafeTerminationMode(Base.Fix1(maximum, abs)),
+    RelNormSafeBestTerminationMode(Base.Fix1(maximum, abs)),
+    AbsTerminationMode(),
+    AbsNormTerminationMode(Base.Fix1(maximum, abs)),
+    AbsNormSafeTerminationMode(Base.Fix1(maximum, abs)),
+    AbsNormSafeBestTerminationMode(Base.Fix1(maximum, abs))
+]
 
 function benchmark_nlsolve_oop(f, u0, p = 2.0; solver, kwargs...)
     prob = NonlinearProblem{false}(f, u0, p)
@@ -46,8 +55,19 @@ function nlprob_iterator_interface(f, p_range, ::Val{iip}, solver) where {iip}
     return sols
 end
 
+for alg in (:Static, :StrongWolfe, :BackTracking, :MoreThuente, :HagerZhang)
+    algname = Symbol(:LineSearches, alg)
+    @eval function $(algname)(args...; autodiff = nothing, initial_alpha = true, kwargs...)
+        return LineSearch.LineSearchesJL(;
+            method = LineSearches.$(alg)(args...; kwargs...), autodiff, initial_alpha)
+    end
+end
+
 export nlprob_iterator_interface, benchmark_nlsolve_oop, benchmark_nlsolve_iip,
        TERMINATION_CONDITIONS, _nameof, newton_fails, quadratic_f, quadratic_f!
+export LineSearchesStatic, LineSearchesStrongWolfe, LineSearchesBackTracking,
+       LineSearchesMoreThuente, LineSearchesHagerZhang
+
 end
 
 # --- NewtonRaphson tests ---
@@ -57,9 +77,10 @@ end
             AutoForwardDiff(), AutoZygote(), AutoFiniteDiff()
         ),
         linesearch in (
-            Static(; autodiff = ad), StrongWolfe(; autodiff = ad),
-            BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad),
-            HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad)
+            LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad),
+            LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad),
+            LineSearchesHagerZhang(; autodiff = ad),
+            LineSearchesMoreThuente(; autodiff = ad)
         )
 
         u0s = ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
@@ -103,12 +124,12 @@ end
     @test nlprob_iterator_interface(quadratic_f, p, Val(false), NewtonRaphson()) ≈ sqrt.(p)
     @test nlprob_iterator_interface(quadratic_f!, p, Val(true), NewtonRaphson()) ≈ sqrt.(p)
 
-    @testset "ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in (
-            AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()),
-            AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())),
+    @testset "Sparsity ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in (
+            AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()),
         u0 in (1.0, [1.0, 1.0])
 
-        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        probN = NonlinearProblem(
+            NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0)
         @test all(solve(probN, NewtonRaphson(; autodiff)).u .≈ sqrt(2.0))
     end
 
@@ -167,12 +188,12 @@ end
     @test nlprob_iterator_interface(quadratic_f!, p, Val(true), TrustRegion()) ≈ sqrt.(p)
 
     @testset "$(_nameof(autodiff)) u0: $(_nameof(u0)) $(radius_update_scheme)" for autodiff in (
-            AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()),
-            AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())),
+            AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()),
         u0 in (1.0, [1.0, 1.0]),
         radius_update_scheme in radius_update_schemes
 
-        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        probN = NonlinearProblem(
+            NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0)
         @test all(solve(probN, TrustRegion(; autodiff, radius_update_scheme)).u .≈
                   sqrt(2.0))
     end
@@ -263,11 +284,11 @@ end
     end
 
     @testset "ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in (
-            AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()),
-            AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())),
+            AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()),
         u0 in (1.0, [1.0, 1.0])
 
-        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        probN = NonlinearProblem(
+            NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0)
         @test all(solve(
             probN, LevenbergMarquardt(; autodiff); abstol = 1e-9, reltol = 1e-9).u .≈
                   sqrt(2.0))
@@ -445,11 +466,11 @@ end
         quadratic_f!, p, Val(true), PseudoTransient(; alpha_initial = 10.0)) ≈ sqrt.(p)
 
     @testset "ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in (
-            AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()),
-            AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())),
+            AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()),
         u0 in (1.0, [1.0, 1.0])
 
-        probN = NonlinearProblem(quadratic_f, u0, 2.0)
+        probN = NonlinearProblem(
+            NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0)
         @test all(solve(probN, PseudoTransient(; alpha_initial = 10.0, autodiff)).u .≈
                   sqrt(2.0))
     end
@@ -471,9 +492,10 @@ end
             AutoForwardDiff(), AutoZygote(), AutoFiniteDiff()
         ),
         linesearch in (
-            Static(; autodiff = ad), StrongWolfe(; autodiff = ad),
-            BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad),
-            HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad)
+            LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad),
+            LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad),
+            LineSearchesHagerZhang(; autodiff = ad),
+            LineSearchesMoreThuente(; autodiff = ad)
         ),
         init_jacobian in (Val(:identity), Val(:true_jacobian)),
         update_rule in (Val(:good_broyden), Val(:bad_broyden), Val(:diagonal))
@@ -524,9 +546,10 @@ end
             AutoForwardDiff(), AutoZygote(), AutoFiniteDiff()
         ),
         linesearch in (
-            Static(; autodiff = ad), StrongWolfe(; autodiff = ad),
-            BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad),
-            HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad)
+            LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad),
+            LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad),
+            LineSearchesHagerZhang(; autodiff = ad),
+            LineSearchesMoreThuente(; autodiff = ad)
         ),
         init_jacobian in (Val(:identity), Val(:true_jacobian), Val(:true_jacobian_diagonal))
 
@@ -577,10 +600,10 @@ end
             AutoForwardDiff(), AutoZygote(), AutoFiniteDiff()
         ),
         linesearch in (
-            Static(; autodiff = ad), StrongWolfe(; autodiff = ad),
-            BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad),
-            HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad),
-            LiFukushimaLineSearch()
+            LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad),
+            LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad),
+            LineSearchesHagerZhang(; autodiff = ad),
+            LineSearchesMoreThuente(; autodiff = ad), LiFukushimaLineSearch()
         )
 
         u0s = ([1.0, 1.0], @SVector[1.0, 1.0], 1.0)
diff --git a/test/gpu/core_tests.jl b/test/gpu/core_tests.jl
index 75087aa30..91d6178a4 100644
--- a/test/gpu/core_tests.jl
+++ b/test/gpu/core_tests.jl
@@ -15,7 +15,8 @@
         SOLVERS = (
             NewtonRaphson(),
             LevenbergMarquardt(; linsolve = QRFactorization()),
-            LevenbergMarquardt(; linsolve = KrylovJL_GMRES()),
+            # XXX: Fails currently
+            # LevenbergMarquardt(; linsolve = KrylovJL_GMRES()),
             PseudoTransient(),
             Klement(),
             Broyden(; linesearch = LiFukushimaLineSearch()),
@@ -27,7 +28,7 @@
         )
 
         @testset "[IIP] GPU Solvers" begin
-            for alg in SOLVERS
+            @testset "$(nameof(typeof(alg)))" for alg in SOLVERS
                 @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5)
             end
         end
@@ -37,7 +38,7 @@
         prob = NonlinearProblem{false}(linear_f, u0)
 
         @testset "[OOP] GPU Solvers" begin
-            for alg in SOLVERS
+            @testset "$(nameof(typeof(alg)))" for alg in SOLVERS
                 @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5)
             end
         end
diff --git a/test/misc/aliasing_tests.jl b/test/misc/aliasing_tests.jl
index 653490dfa..78b8ec798 100644
--- a/test/misc/aliasing_tests.jl
+++ b/test/misc/aliasing_tests.jl
@@ -9,7 +9,7 @@
 
     # If aliasing is not handled properly this will diverge
     sol = solve(prob; abstol = 1e-6, alias_u0 = true,
-        termination_condition = AbsNormTerminationMode())
+        termination_condition = AbsNormTerminationMode(Base.Fix1(maximum, abs)))
 
     @test sol.u === prob.u0
     @test SciMLBase.successful_retcode(sol.retcode)
@@ -17,7 +17,7 @@
     prob = remake(prob; u0 = copy(u0))
 
     cache = init(prob; abstol = 1e-6, alias_u0 = true,
-        termination_condition = AbsNormTerminationMode())
+        termination_condition = AbsNormTerminationMode(Base.Fix1(maximum, abs)))
     sol = solve!(cache)
 
     @test sol.u === prob.u0
diff --git a/test/misc/bruss_tests.jl b/test/misc/bruss_tests.jl
index 80b1bd540..32a3fff68 100644
--- a/test/misc/bruss_tests.jl
+++ b/test/misc/bruss_tests.jl
@@ -52,11 +52,6 @@
     sol = solve(prob_brusselator_2d_sparse, NewtonRaphson(); abstol = 1e-8)
     @test norm(sol.resid, Inf) < 1e-8
 
-    # Deprecated
-    sol = solve(prob_brusselator_2d,
-        NewtonRaphson(autodiff = AutoSparse(AutoFiniteDiff())); abstol = 1e-8)
-    @test norm(sol.resid, Inf) < 1e-8
-
     f! = (du, u) -> brusselator_2d_loop(du, u, p)
     du0 = similar(u0)
     jac_prototype = ADTypes.jacobian_sparsity(f!, du0, u0, TracerSparsityDetector())
diff --git a/test/runtests.jl b/test/runtests.jl
index 74676f3ad..33ca5da7a 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,9 +1,14 @@
-using ReTestItems, NonlinearSolve, Hwloc, InteractiveUtils
+using ReTestItems, NonlinearSolve, Hwloc, InteractiveUtils, Pkg
 
 @info sprint(InteractiveUtils.versioninfo)
 
 const GROUP = lowercase(get(ENV, "GROUP", "All"))
 
+const EXTRA_PKGS = Pkg.PackageSpec[]
+(GROUP == "all" || GROUP == "downstream") &&
+    push!(EXTRA_PKGS, Pkg.PackageSpec("ModelingToolkit"))
+length(EXTRA_PKGS) ≥ 1 && Pkg.add(EXTRA_PKGS)
+
 const RETESTITEMS_NWORKERS = parse(
     Int, get(ENV, "RETESTITEMS_NWORKERS", string(min(Hwloc.num_physical_cores(), 4))))
 const RETESTITEMS_NWORKER_THREADS = parse(Int,
@@ -14,4 +19,4 @@ const RETESTITEMS_NWORKER_THREADS = parse(Int,
 
 ReTestItems.runtests(NonlinearSolve; tags = (GROUP == "all" ? nothing : [Symbol(GROUP)]),
     nworkers = RETESTITEMS_NWORKERS, nworker_threads = RETESTITEMS_NWORKER_THREADS,
-    testitem_timeout = 3600, retries = 4)
+    testitem_timeout = 3600)