Remove DASK_DATAFRAME__QUERY_PLANNING environment variable (#270)

* remove DASK_DATAFRAME__QUERY_PLANNING False Signed-off-by: Sarah Yurick <[email protected]> * isort Signed-off-by: Sarah Yurick <[email protected]> --------- Signed-off-by: Sarah Yurick <[email protected]>
NVIDIA · Oct 9, 2024 · bf2f8f2 · bf2f8f2
1 parent 06c388d
commit bf2f8f2
Show file tree

Hide file tree

Showing 7 changed files with 1 addition and 13 deletions.
diff --git a/examples/translation_example.py b/examples/translation_example.py
@@ -1,7 +1,4 @@
 import os
-
-os.environ["DASK_DATAFRAME__QUERY_PLANNING"] = "False"
-import argparse
 import re
 import time
 from dataclasses import dataclass

diff --git a/nemo_curator/classifiers/aegis.py b/nemo_curator/classifiers/aegis.py
@@ -14,7 +14,6 @@
 import os
 
 os.environ["RAPIDS_NO_INITIALIZE"] = "1"
-os.environ["DASK_DATAFRAME__QUERY_PLANNING"] = "False"
 from dataclasses import dataclass
 from functools import lru_cache
 from typing import List, Optional, Union

diff --git a/nemo_curator/classifiers/fineweb_edu.py b/nemo_curator/classifiers/fineweb_edu.py
@@ -14,7 +14,6 @@
 import os
 
 os.environ["RAPIDS_NO_INITIALIZE"] = "1"
-os.environ["DASK_DATAFRAME__QUERY_PLANNING"] = "False"
 import torch
 from crossfit import op
 from crossfit.backend.torch.hf.model import HFModel
@@ -23,7 +22,6 @@
 from nemo_curator.classifiers.base import (
     DistributedDataClassifier,
     _get_suggest_memory_for_classifier,
-    _run_classifier_helper,
 )
 from nemo_curator.datasets import DocumentDataset
 

diff --git a/nemo_curator/scripts/semdedup/clustering.py b/nemo_curator/scripts/semdedup/clustering.py
@@ -16,7 +16,6 @@
 import os
 from datetime import datetime
 
-os.environ["DASK_DATAFRAME__QUERY_PLANNING"] = "False"
 import dask_cudf
 
 from nemo_curator.datasets import DocumentDataset

diff --git a/tests/test_semdedup.py b/tests/test_semdedup.py
@@ -14,8 +14,6 @@
 import os
 
 import pytest
-
-os.environ["DASK_DATAFRAME__QUERY_PLANNING"] = "False"
 from dask.dataframe.utils import assert_eq
 from distributed import Client
 

diff --git a/tutorials/distributed_data_classification/distributed_data_classification.ipynb b/tutorials/distributed_data_classification/distributed_data_classification.ipynb
@@ -20,16 +20,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "env: PYTHONWARNINGS=ignore\n",
-      "env: DASK_DATAFRAME__QUERY_PLANNING=False\n"
+      "env: PYTHONWARNINGS=ignore\n"
      ]
     }
    ],
    "source": [
     "# Silence Warnings (HuggingFace internal warnings)\n",
     "\n",
     "%env PYTHONWARNINGS=ignore\n",
-    "%env DASK_DATAFRAME__QUERY_PLANNING=False\n",
     "import warnings\n",
     "warnings.filterwarnings(\"ignore\")"
    ]

diff --git a/tutorials/single_node_tutorial/single_gpu_tutorial.ipynb b/tutorials/single_node_tutorial/single_gpu_tutorial.ipynb
@@ -110,7 +110,6 @@
    },
    "outputs": [],
    "source": [
-    "%env DASK_DATAFRAME__QUERY_PLANNING False\n",
     "%env CUDA_VISIBLE_DEVICES 0"
    ]
   },