debug missing destroy

huggingface · Feb 13, 2024 · 0d21bba · 0d21bba
1 parent b2c71b0
commit 0d21bba
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/.github/workflows/small.yaml b/.github/workflows/small.yaml
@@ -56,4 +56,4 @@ jobs:
     - name: Run tests
       # NOTE: -m "not fa2" will run all the unit tests that don't have the mark
       # "fa2" (these are FA2-related tests, we can't run it on T4)
-      run: pytest -n 1 --color=yes --durations=0 --verbose tests/test_clip_grads.py
+      run: pytest -n 1 --color=yes --durations=0 --verbose tests/test_clip_grads/test_clip_grads_with_tp
diff --git a/tests/test_clip_grads.py b/tests/test_clip_grads.py
@@ -340,7 +340,11 @@ def _test_clip_grads_with_tp(
     )
     torch.testing.assert_close(total_norm, ref_total_norm)
 
-    parallel_context.destroy()
+    try:
+        parallel_context.destroy()
+    except Exception:
+        print("Failed to destroy parallel context")
+        print(f"parallel_contex.type: {type(parallel_context)}")
 
 
 @pytest.mark.skipif(available_gpus() < 2, reason="test_clip_grads_tied_weights requires at least 2 gpus")