pass through num_proc param for ray executor when loading dataset

modelscope · Jan 27, 2025 · acccc01 · acccc01
1 parent e05f146
commit acccc01
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/data_juicer/core/executor/ray_executor.py b/data_juicer/core/executor/ray_executor.py
@@ -77,7 +77,7 @@ def run(self,
         """
         # 1. load data
         logger.info('Loading dataset with Ray...')
-        dataset = self.datasetbuilder.load_dataset()
+        dataset = self.datasetbuilder.load_dataset(num_proc=load_data_np)
 
         # 2. extract processes
         logger.info('Preparing process operators...')