diff --git a/src/shift.jl b/src/shift.jl
index d69712e..10b90a5 100644
--- a/src/shift.jl
+++ b/src/shift.jl
@@ -17,7 +17,7 @@ function is_dependent!(r::RankDependence, row)
     new_rank = LinearAlgebra.rank(r.matrix[rows, :], r.check)
     if new_rank < r.old_rank
         @warn(
-            "After adding rows, the rank dropped from `$old_rank` to `$new_rank`. Correcting the rank to `$old_rank` and continuing."
+            "After adding rows, the rank dropped from `$(r.old_rank)` to `$new_rank`. Correcting the rank to `$(r.old_rank)` and continuing."
         )
         new_rank = r.old_rank
     end
diff --git a/test/rank.jl b/test/rank.jl
index 15abe87..b0c7c79 100644
--- a/test/rank.jl
+++ b/test/rank.jl
@@ -10,3 +10,19 @@
     ldlt = low_rank_ldlt(M, SVDLDLT(), 1e-10)
     @test ldlt.L ≈ -normalize(v)
 end
+
+struct HardcodedRanks <: RankCheck
+    r::Vector{Int}
+end
+
+function MultivariateMoments.rank_from_singular_values(σ, r::HardcodedRanks)
+    return r.r[length(σ)]
+end
+
+@testset "Decreasing rank" begin
+    r = RankDependence(zeros(4, 4), HardcodedRanks([1, 0, 1, 2]))
+    @test !is_dependent!(r, 1)
+    @test is_dependent!(r, 2)
+    @test is_dependent!(r, 3)
+    @test is_dependent!(r, 4)
+end