From db411b0480b0b84d481505a92553be12332191cf Mon Sep 17 00:00:00 2001 From: Ryan Wolf Date: Thu, 2 Jan 2025 08:23:23 -0800 Subject: [PATCH] Reorder import (#460) Signed-off-by: Ryan Wolf --- nemo_curator/modules/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nemo_curator/modules/__init__.py b/nemo_curator/modules/__init__.py index bc5659311..b792d807c 100644 --- a/nemo_curator/modules/__init__.py +++ b/nemo_curator/modules/__init__.py @@ -25,7 +25,6 @@ from .config import FuzzyDuplicatesConfig, SemDedupConfig from .dataset_ops import blend_datasets, Shuffle from .exact_dedup import ExactDuplicates -from .filter import Filter, Score, ScoreFilter, ParallelScoreFilter from .meta import Sequential from .modify import Modify from .task import TaskDecontamination @@ -39,9 +38,7 @@ BucketsToEdges = gpu_only_import_from( "nemo_curator.modules.fuzzy_dedup", "BucketsToEdges" ) -# Pytorch related imports must come after all imports that require cugraph, -# because of context cleanup issues b/w pytorch and cugraph -# See this issue: https://github.com/rapidsai/cugraph/issues/2718 + SemDedup = gpu_only_import_from("nemo_curator.modules.semantic_dedup", "SemDedup") EmbeddingCreator = gpu_only_import_from( "nemo_curator.modules.semantic_dedup", "EmbeddingCreator" @@ -52,6 +49,10 @@ SemanticClusterLevelDedup = gpu_only_import_from( "nemo_curator.modules.semantic_dedup", "SemanticClusterLevelDedup" ) +# Pytorch related imports must come after all imports that require cugraph, +# because of context cleanup issues b/w pytorch and cugraph +# See this issue: https://github.com/rapidsai/cugraph/issues/2718 +from .filter import Filter, Score, ScoreFilter, ParallelScoreFilter __all__ = [ "ExactDuplicates",