Skip to content

Commit

Permalink
Predict for Peteish13
Browse files Browse the repository at this point in the history
  • Loading branch information
@liujch1998
liujch1998 committed Nov 17, 2024
1 parent 0b4800a commit 19ce054
Show file tree
Hide file tree
Showing 7 changed files with 236 additions and 89 deletions.
22 changes: 7 additions & 15 deletions olmo/scaling/scaling_laws/download_wandb_logs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,29 +70,21 @@ def main(args):
if args.y_axis == ["eval/all-validation/CrossEntropyLoss"]:
args.y_axis = [f"eval/{d}/CrossEntropyLoss" for d in validation]

if args.y_axis == ["eval/all-validation-and-bpb/CrossEntropyLoss"]:
args.y_axis = [f"eval/{d}/CrossEntropyLoss" for d in validation] + [
f"eval/downstream_bpb/{d}_bpb" for d in downstream_bpb
]

elif args.y_axis == ["eval/all-v3-validation/CrossEntropyLoss"]:
args.y_axis = [f"eval/{d}/CrossEntropyLoss" for d in v3_validation]

elif args.y_axis == ["eval/all-validation-and-bpb/CrossEntropyLoss"]:
args.y_axis = [f"eval/{d}/CrossEntropyLoss" for d in validation] + downstream_bpb

elif args.y_axis == ["eval/downstream/all"]:
args.y_axis = [f"eval/downstream/{d}" for d in downstream]
args.y_axis = downstream

elif args.y_axis == ["eval/validation-and-bpb-and-downstream"]:
args.y_axis = (
[f"eval/{d}/CrossEntropyLoss" for d in validation]
+ [f"eval/downstream_bpb/{d}_bpb" for d in downstream_bpb]
+ [f"eval/downstream/{d}" for d in downstream]
)
args.y_axis = [f"eval/{d}/CrossEntropyLoss" for d in validation] + downstream_bpb + downstream

elif args.y_axis == ["eval/validation-and-bpb-and-downstream-newline"]:
args.y_axis = (
[f"eval/{d}/CrossEntropyLoss" for d in validation]
+ [f"eval/downstream_bpb/{d}_bpb" for d in downstream_bpb]
+ [f"eval/downstream/{d}" for d in downstream]
[f"eval/{d}/CrossEntropyLoss" for d in validation] + downstream_bpb + downstream
+ [f"eval/downstream_bpb/{d}_bpb" for d in downstream_newline_bpb]
+ [f"eval/downstream/{d}" for d in downstream_newline]
)
Expand Down Expand Up @@ -252,8 +244,8 @@ def main(args):
# python olmo/scaling/scaling_laws/download_wandb_logs.py -n 'ai2-llm/olmo-ladder/amberish-rulebased-3B-5xC' -y eval/validation-and-bpb-and-downstream -o wandb/amberish-rulebased/3B-5xC.csv

# python olmo/scaling/scaling_laws/download_wandb_logs.py -n 'ai2-llm/olmo-medium/peteish7' -y eval/downstream/arc_easy_acc -o wandb/peteish7_train.csv
# python olmo/scaling/scaling_laws/download_wandb_logs.py -n 'ai2-llm/olmo-medium/peteish7-eval' -y eval/validation-and-bpb-and-downstream -o wandb/peteish7_eval_final.csv
# python olmo/scaling/scaling_laws/download_wandb_logs.py -n 'ai2-llm/olmo-medium/peteish7-eval' -y eval/validation-and-bpb-and-downstream -e -o wandb/peteish7_eval_full.csv
# python olmo/scaling/scaling_laws/download_wandb_logs.py -n 'ai2-llm/olmo-medium/peteish13-eval' -y eval/validation-and-bpb-and-downstream -o wandb/peteish13_eval_final.csv

# python olmo/scaling/scaling_laws/download_wandb_logs.py -n 'ai2-llm/olmo-ladder/peteish-final-190M-1xC' -y eval/validation-and-bpb-and-downstream -o wandb/peteish-final/190M-1xC.csv
# python olmo/scaling/scaling_laws/download_wandb_logs.py -n 'ai2-llm/olmo-ladder/peteish-final-370M-1xC' -y eval/validation-and-bpb-and-downstream -o wandb/peteish-final/370M-1xC.csv
Expand Down
120 changes: 74 additions & 46 deletions olmo/scaling/scaling_laws/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,16 @@ def get_accuracy_keys(tasks: Dict[str, DownstreamTaskPrediction]) -> List[str]:
return accuracy_keys


def get_mc_accuracy_keys(tasks: Dict[str, DownstreamTaskPrediction]) -> List[str]:
mc_accuracy_keys: List[str] = []
for _, task in tasks.items():
if isinstance(task.task_mc_accuracy_key, list):
mc_accuracy_keys += task.task_mc_accuracy_key
else:
mc_accuracy_keys.append(task.task_mc_accuracy_key)
return mc_accuracy_keys


# Special case for testing with old tokenizer:

downstream_newline = [
Expand Down Expand Up @@ -323,9 +333,9 @@ def get_accuracy_keys(tasks: Dict[str, DownstreamTaskPrediction]) -> List[str]:
"socialiqa_newline_mc_5shot_bpb",
]

tasks = {**core_5shot_tasks, **mmlu_var_tasks, **mmlu_subset_var_tasks}
tasks = {**core_5shot_tasks, **mmlu_var_tasks}
downstream_bpb = get_bpb_keys(tasks)
downstream = get_accuracy_keys(tasks)
downstream = get_accuracy_keys(tasks) + get_mc_accuracy_keys(tasks)

KEYS_BY_KEY = {
"all-val-lm": [f"eval/{val}/CrossEntropyLoss" for val in validation],
Expand Down Expand Up @@ -530,58 +540,76 @@ def get_step2_data_by_name(configs, task_name, y_metric="rc_acc", moving_avg=1,
data_by_name: Dict = defaultdict(lambda: {"xs": [], "ys": [], "ds": [], "ns": [], "ls": []})

for name, config in configs.items():
n = config.n
for path in config.paths:
length = get_length(path)
with open(path) as file_ref:
reader = csv.DictReader(file_ref)
rows = [row for row in reader]
xs, ys, ds, ns, ls = [], [], [], [], []
for row in rows:
d = int(float(row["throughput/total_tokens"]))
if name == "external":
xs, ys = [], []
for path in config.paths:
with open(path) as f:
data = json.load(f)
x = np.average(
[float(row[key]) for key in loss_keys],
[float(data[key]) for key in loss_keys],
weights=[WEIGHT_BY_KEY.get(key, 1.0) for key in loss_keys],
)
y = np.average(
[float(row[key]) for key in accuracy_keys],
[float(data[key]) for key in accuracy_keys],
weights=[WEIGHT_BY_KEY.get(key, 1.0) for key in accuracy_keys],
)
xs.append(x)
ys.append(y)
ds.append(d)
ns.append(n)
ls.append(length)

if config.mode == "train":
# skip initial ckpts

xs = xs[int(np.ceil(skip_perc * len(xs))) :]
ys = ys[int(np.ceil(skip_perc * len(ys))) :]
ds = ds[int(np.ceil(skip_perc * len(ds))) :]
ns = ns[int(np.ceil(skip_perc * len(ns))) :]
ls = ls[int(np.ceil(skip_perc * len(ls))) :]

# apply moving_avg
xs = moving_average(xs, n=moving_avg).tolist()
# ys = ys[moving_avg-1:]
# ds = ds[moving_avg-1:]
# ns = ns[moving_avg-1:]
# ls = ls[moving_avg-1:]

# last n points
if last_n_points > 0:
xs = xs[-last_n_points:]
ys = ys[-last_n_points:]
ds = ds[-last_n_points:]
ns = ns[-last_n_points:]
ls = ls[-last_n_points:]

data_by_name[name]["xs"] += xs
data_by_name[name]["ys"] += ys
data_by_name[name]["ds"] += ds
data_by_name[name]["ns"] += ns
data_by_name[name]["ls"] += ls
data_by_name[name] = {"xs": xs, "ys": ys, "ds": [], "ns": [], "ls": []}

else:
n = config.n
for path in config.paths:
length = get_length(path)
with open(path) as file_ref:
reader = csv.DictReader(file_ref)
rows = [row for row in reader]
xs, ys, ds, ns, ls = [], [], [], [], []
for row in rows:
d = int(float(row["throughput/total_tokens"]))
x = np.average(
[float(row[key]) for key in loss_keys],
weights=[WEIGHT_BY_KEY.get(key, 1.0) for key in loss_keys],
)
y = np.average(
[float(row[key]) for key in accuracy_keys],
weights=[WEIGHT_BY_KEY.get(key, 1.0) for key in accuracy_keys],
)
xs.append(x)
ys.append(y)
ds.append(d)
ns.append(n)
ls.append(length)

if config.mode == "train":
# skip initial ckpts

xs = xs[int(np.ceil(skip_perc * len(xs))) :]
ys = ys[int(np.ceil(skip_perc * len(ys))) :]
ds = ds[int(np.ceil(skip_perc * len(ds))) :]
ns = ns[int(np.ceil(skip_perc * len(ns))) :]
ls = ls[int(np.ceil(skip_perc * len(ls))) :]

# apply moving_avg
xs = moving_average(xs, n=moving_avg).tolist()
# ys = ys[moving_avg-1:]
# ds = ds[moving_avg-1:]
# ns = ns[moving_avg-1:]
# ls = ls[moving_avg-1:]

# last n points
if last_n_points > 0:
xs = xs[-last_n_points:]
ys = ys[-last_n_points:]
ds = ds[-last_n_points:]
ns = ns[-last_n_points:]
ls = ls[-last_n_points:]

data_by_name[name]["xs"] += xs
data_by_name[name]["ys"] += ys
data_by_name[name]["ds"] += ds
data_by_name[name]["ns"] += ns
data_by_name[name]["ls"] += ls

data_by_name[name]["mode"] = config.mode

Expand Down
51 changes: 30 additions & 21 deletions scripts/scaling/final.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"190m": {
"paths": [
"scripts/scaling/data/peteish-final-new/190M-1xC.csv",
"scripts/scaling/data/peteish-final-new/190M-2xC.csv",
"scripts/scaling/data/peteish-final-new/190M-5xC.csv",
"scripts/scaling/data/peteish-final-new/190M-10xC.csv"
"wandb/peteish-final/190M-1xC.csv",
"wandb/peteish-final/190M-2xC.csv",
"wandb/peteish-final/190M-5xC.csv",
"wandb/peteish-final/190M-10xC.csv"
],
"mode": "train",
"n": 190354176,
Expand All @@ -13,10 +13,10 @@
},
"370m": {
"paths": [
"scripts/scaling/data/peteish-final-new/370M-1xC.csv",
"scripts/scaling/data/peteish-final-new/370M-2xC.csv",
"scripts/scaling/data/peteish-final-new/370M-5xC.csv",
"scripts/scaling/data/peteish-final-new/370M-10xC.csv"
"wandb/peteish-final/370M-1xC.csv",
"wandb/peteish-final/370M-2xC.csv",
"wandb/peteish-final/370M-5xC.csv",
"wandb/peteish-final/370M-10xC.csv"
],
"mode": "train",
"n": 371262464,
Expand All @@ -25,10 +25,10 @@
},
"600m": {
"paths": [
"scripts/scaling/data/peteish-final-new/600M-1xC.csv",
"scripts/scaling/data/peteish-final-new/600M-2xC.csv",
"scripts/scaling/data/peteish-final-new/600M-5xC.csv",
"scripts/scaling/data/peteish-final-new/600M-10xC.csv"
"wandb/peteish-final/600M-1xC.csv",
"wandb/peteish-final/600M-2xC.csv",
"wandb/peteish-final/600M-5xC.csv",
"wandb/peteish-final/600M-10xC.csv"
],
"mode": "train",
"n": 597382464,
Expand All @@ -37,10 +37,10 @@
},
"760m": {
"paths": [
"scripts/scaling/data/peteish-final-new/760M-1xC.csv",
"scripts/scaling/data/peteish-final-new/760M-2xC.csv",
"scripts/scaling/data/peteish-final-new/760M-5xC.csv",
"scripts/scaling/data/peteish-final-new/760M-10xC.csv"
"wandb/peteish-final/760M-1xC.csv",
"wandb/peteish-final/760M-2xC.csv",
"wandb/peteish-final/760M-5xC.csv",
"wandb/peteish-final/760M-10xC.csv"
],
"mode": "train",
"n": 758220288,
Expand All @@ -49,10 +49,10 @@
},
"1b": {
"paths": [
"scripts/scaling/data/peteish-final-new/1B-1xC.csv",
"scripts/scaling/data/peteish-final-new/1B-2xC.csv",
"scripts/scaling/data/peteish-final-new/1B-5xC.csv",
"scripts/scaling/data/peteish-final-new/1B-10xC.csv"
"wandb/peteish-final/1B-1xC.csv",
"wandb/peteish-final/1B-2xC.csv",
"wandb/peteish-final/1B-5xC.csv",
"wandb/peteish-final/1B-10xC.csv"
],
"mode": "train",
"n": 1279395840,
Expand All @@ -61,11 +61,20 @@
},
"7b": {
"paths": [
"scripts/scaling/data/peteish-final-new/7B-28xC-anneal-new.csv"
"wandb/peteish7_eval_anneal.csv"
],
"mode": "eval",
"n": 6887575552,
"label": "7b",
"color": "darkviolet"
},
"13b": {
"paths": [
"wandb/peteish13_eval_final.csv"
],
"mode": "eval",
"n": 13202396160,
"label": "13b",
"color": "darkmagenta"
}
}
4 changes: 3 additions & 1 deletion scripts/scaling/predict.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# python scripts/scaling/predict.py -k main -c scripts/scaling/final.json -n 6887575552 -d 3945065873408 -t 7b
# python scripts/scaling/predict.py -k main -c scripts/scaling/final.json --step2-config-path scripts/scaling/step2.json -n 6887575552 -d 3945065873408 -t 7b
# python scripts/scaling/predict.py -k main -c scripts/scaling/final.json --step2-config-path scripts/scaling/step2.json -n 13202396160 -d 5000080130048 -t 13b
# python scripts/scaling/predict.py -k main_mc -c scripts/scaling/final.json --step2-config-path scripts/scaling/step2_mc.json -y mc_acc -n 6887575552 -d 3945065873408 -t 7b-4T-final
# python scripts/scaling/predict.py -k main_mc -c scripts/scaling/final.json --step2-config-path scripts/scaling/step2_mc.json -y mc_acc -n 13202396160 -d 5000080130048 -t 13b-5T-final

import argparse

Expand Down
Loading

0 comments on commit 19ce054

Please sign in to comment.