flops analysis

allenai · Nov 19, 2024 · 4cd12ef · 4cd12ef
1 parent 6e3a1c2
commit 4cd12ef
Show file tree

Hide file tree

Showing 6 changed files with 303 additions and 133 deletions.
diff --git a/olmo/scaling/scaling_laws/utils.py b/olmo/scaling/scaling_laws/utils.py
@@ -146,7 +146,7 @@ def get_mc_accuracy_keys(self):
     "socialiqa",
     "winogrande",
 ]
-core_small_names = ["hellaswag", "arc_challenge", "piqa", "csqa"]
+core_small_names = ["hellaswag", "arc_easy", "arc_challenge", "piqa", "csqa", "socialiqa", "openbookqa"]
 mmlu_names = ["mmlu_stem", "mmlu_humanities", "mmlu_social_sciences", "mmlu_other"]
 
 core_5shot_tasks: Dict[str, DownstreamTaskPrediction] = {
@@ -493,7 +493,7 @@ def get_step1_data_by_name(configs, task_name, y_metric="rc_bpb", moving_avg=1):
     else:
         raise ValueError(f"Invalid y_metric: {y_metric}")
 
-    data_by_name: Dict = defaultdict(lambda: {"ns": [], "ds": [], "ys": [], "ls": []})
+    data_by_name: Dict = defaultdict(lambda: {"ns": [], "ds": [], "ys": [], "ls": [], "fs": []})
     for name, config in configs.items():
         n = config.n
         for path in config.paths:
@@ -505,18 +505,21 @@ def get_step1_data_by_name(configs, task_name, y_metric="rc_bpb", moving_avg=1):
                 ds, ys, fs = [], [], []
                 for row in rows:
                     d = int(float(row["throughput/total_tokens"]))
-                    f = d * MODEL_FLOPS[name]
+                    f = d * MODEL_FLOPS[name.split("-")[0]]
                     y = np.average(
                         [float(row[key]) for key in keys], weights=[WEIGHT_BY_KEY.get(key, 1.0) for key in keys]
                     )
                     ds.append(d)
                     ys.append(y)
+                    fs.append(f)
                 d = ds[-1]
                 y = np.mean(ys)
+                f = fs[-1]
                 data_by_name[name]["ns"].append(n)
                 data_by_name[name]["ds"].append(d)
                 data_by_name[name]["ys"].append(y)
                 data_by_name[name]["ls"].append(length)
+                data_by_name[name]["fs"].append(f)
         data_by_name[name]["mode"] = config.mode
     return data_by_name
 

diff --git a/scripts/scaling/configs/1b_easy.json b/scripts/scaling/configs/1b_easy.json
@@ -0,0 +1,69 @@
+{
+    "190m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/190M-1xC.csv",
+            "scripts/scaling/data/peteish-final-new/190M-2xC.csv",
+            "scripts/scaling/data/peteish-final-new/190M-5xC.csv",
+            "scripts/scaling/data/peteish-final-new/190M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 190354176,
+        "label": "190m",
+        "color": "darkred"
+    },
+    "370m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/370M-1xC.csv",
+            "scripts/scaling/data/peteish-final-new/370M-2xC.csv",
+            "scripts/scaling/data/peteish-final-new/370M-5xC.csv",
+            "scripts/scaling/data/peteish-final-new/370M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 371262464,
+        "label": "300m",
+        "color": "darkorange"
+    },
+    "600m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/600M-1xC.csv",
+            "scripts/scaling/data/peteish-final-new/600M-2xC.csv",
+            "scripts/scaling/data/peteish-final-new/600M-5xC.csv",
+            "scripts/scaling/data/peteish-final-new/600M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 597382464,
+        "label": "600m",
+        "color": "goldenrod"
+    },
+    "760m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/760M-1xC.csv",
+            "scripts/scaling/data/peteish-final-new/760M-2xC.csv",
+            "scripts/scaling/data/peteish-final-new/760M-5xC.csv",
+            "scripts/scaling/data/peteish-final-new/760M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 758220288,
+        "label": "760m",
+        "color": "darkgreen"
+    },
+    "1b": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/1B-1xC.csv",
+            "scripts/scaling/data/peteish-final-new/1B-2xC.csv",
+            "scripts/scaling/data/peteish-final-new/1B-5xC.csv"
+        ],
+        "mode": "train",
+        "n": 1279395840,
+        "label": "1b",
+        "color": "teal"
+    },
+    "1b-10xC": {
+        "paths": ["scripts/scaling/data/peteish-final-new/1B-10xC.csv"],
+        "mode": "eval",
+        "n": 1279395840,
+        "label": "1b",
+        "color": "darkmagenta"
+
+    }
+ }
diff --git a/scripts/scaling/configs/1b_sampled_10xC.json b/scripts/scaling/configs/1b_sampled_10xC.json
@@ -0,0 +1,47 @@
+{
+    "190m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/190M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 190354176,
+        "label": "190m",
+        "color": "darkred"
+    },
+    "370m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/370M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 371262464,
+        "label": "300m",
+        "color": "darkorange"
+    },
+    "600m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/600M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 597382464,
+        "label": "600m",
+        "color": "goldenrod"
+    },
+    "760m": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/760M-10xC.csv"
+        ],
+        "mode": "train",
+        "n": 758220288,
+        "label": "760m",
+        "color": "darkgreen"
+    },
+    "1b": {
+        "paths": [
+            "scripts/scaling/data/peteish-final-new/1B-10xC.csv"
+        ],
+        "mode": "eval",
+        "n": 1279395840,
+        "label": "1b",
+        "color": "teal"
+    }
+ }
diff --git a/scripts/scaling/stacked.py b/scripts/scaling/stacked.py
@@ -95,7 +95,7 @@ def parse_length(length: str, model_size: int) -> int:
     elif length_unit == "T":
         length_in_tokens *= 1000000000000
     else:
-        raise ValueError(f"Could not parse length '{args.length}'")
+        raise ValueError(f"Could not parse length '{length}'")
     return length_in_tokens
 
 
@@ -115,8 +115,6 @@ def main():
     num_tasks = len(args.keys)
     fig, axes = plt.subplots(num_tasks, 3, figsize=(6 * 3, 4.5 * num_tasks), squeeze=False)
 
-    accs = 0
-
     results = "Task Name | Loss Error | Accuracy Error | Stacked Accuracy Error"
 
     for r, task_name in enumerate(args.keys):