diff --git a/python-package/lightgbm/basic.py b/python-package/lightgbm/basic.py
index 1db55385af1b..2f6ead08909d 100644
--- a/python-package/lightgbm/basic.py
+++ b/python-package/lightgbm/basic.py
@@ -1248,7 +1248,7 @@ def predict(
         if pred_leaf:
             preds = preds.astype(np.int32)
         is_sparse = isinstance(preds, (list, scipy.sparse.spmatrix))
-        if not is_sparse and preds.size != nrow:
+        if not is_sparse and preds.size != nrow or pred_leaf:
             if preds.size % nrow == 0:
                 preds = preds.reshape(nrow, -1)
             else: