temp changes to test

allenai · Oct 23, 2024 · 5dff40c · 5dff40c
1 parent efe766b
commit 5dff40c
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 2 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -22,6 +22,7 @@ dependencies = [
     "omegaconf",
     "safetensors",
     "importlib_resources",
+    "s3fs", # REMOVE THIS IN FAVOR OF SOMETHING CONSISTENT ELSEWHERE
 ]
 
 [project.urls]

diff --git a/src/examples/train_with_mixture.py b/src/examples/train_with_mixture.py
@@ -168,7 +168,9 @@ def build_config(run_name: str, overrides: List[str]) -> ExperimentConfig:
             "evaluator",
             LMEvaluatorCallbackConfig(
                 eval_dataset=NumpyDatasetConfig(
-                    paths=["/net/nfs/allennlp/llm-data/c4/en/c4-validation.00000-00008.npy"],
+                    paths=[
+                        "s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/c4_en/val/part-0-00000.npy"
+                    ],
                     metadata=[{"label": "c4-validation"}],
                     name=NumpyDatasetType.padded_fsl,
                     sequence_length=sequence_length,

diff --git a/src/examples/train_with_mixture_launch.py b/src/examples/train_with_mixture_launch.py
@@ -24,7 +24,7 @@ def build_config(run_name: str, overrides: List[str]) -> BeakerLaunchConfig:
         num_nodes=1,
         num_gpus=4,
         shared_filesystem=True,
-        nfs=True,
+        nfs=False,
         allow_dirty=True,
     )