diff --git a/torch/csrc/distributed/c10d/NanCheck.cu b/torch/csrc/distributed/c10d/NanCheck.cu
index d256413d60a10f..2506ccd1ad094d 100644
--- a/torch/csrc/distributed/c10d/NanCheck.cu
+++ b/torch/csrc/distributed/c10d/NanCheck.cu
@@ -233,6 +233,10 @@ void checkForNan(const at::Tensor& tensor, at::cuda::CUDAStream& stream) {
   const size_t numThreadsPerBlock =
       std::min<size_t>(maxNumThreadsPerBlock, tensor.numel());
 
+  if (!(numThreadsPerBlock > 0)) {
+    return;
+  }
+
   const size_t numBlocks = std::min<size_t>(
       maxNumBlocks,
       (tensor.numel() + numThreadsPerBlock - 1) / numThreadsPerBlock);