From f1993f7284ba9f883818787f8df501b776987c49 Mon Sep 17 00:00:00 2001 From: Praateek Date: Fri, 17 Jan 2025 13:01:25 -0800 Subject: [PATCH] fc Signed-off-by: Praateek --- nemo_curator/utils/distributed_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nemo_curator/utils/distributed_utils.py b/nemo_curator/utils/distributed_utils.py index a9d792b4..8adc982b 100644 --- a/nemo_curator/utils/distributed_utils.py +++ b/nemo_curator/utils/distributed_utils.py @@ -485,7 +485,6 @@ def read_data_files_per_partition( columns: Optional[List[str]] = None, **kwargs, ) -> dd.DataFrame: - input_files = sorted(input_files) if files_per_partition > 1: input_files = [ input_files[i : i + files_per_partition]