Repartition on dask-pandas is failing

Signed-off-by: Vibhu Jawa <[email protected]>
NVIDIA · Jan 9, 2025 · 9b81751 · 9b81751
1 parent eaeaee2
commit 9b81751
Showing 1 changed file with 0 additions and 4 deletions.
diff --git a/nemo_curator/modules/semantic_dedup.py b/nemo_curator/modules/semantic_dedup.py
@@ -338,14 +338,10 @@ def __call__(self, embeddings_dataset: DocumentDataset):
 
         with performance_report_if_with_ts_suffix(self.profile_dir, "clustering-model"):
             embeddings_df = embeddings_df[[self.id_col, self.embedding_col]]
-
             embeddings_df = embeddings_df.repartition(
                 partition_size=self.partition_size
             )
             embeddings_df = embeddings_df.to_backend("pandas").persist()
-            # embeddings_df = embeddings_df.repartition(
-            #     partition_size=self.partition_size
-            # )
             embeddings_df = embeddings_df.to_backend("cudf")
 
             cupy_darr = embeddings_df.map_partitions(