Skip to content

Commit

Permalink
Merge pull request #1 from chiranko/main
Browse files Browse the repository at this point in the history 
Tiny fix for scripts
  • Loading branch information
@zhuohaoyu
zhuohaoyu authored Jun 19, 2024
2 parents 417b228 + 169d623 commit 7819b57
Showing 1 changed file with 112 additions and 114 deletions.
226 changes: 112 additions & 114 deletions scripts/generate-configs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"dataset_kwargs": {
"seed": 2,
"split": "test",
"name_or_path": "/nvme/gc/interactive-eval/InteractiveEval/dev/datasets/ai2_arc",
"name_or_path": "/path/to/hf_dataset",
"config_name": "ARC-Challenge",
"fewshot_split": "train",
"fewshot_num": 5,
Expand All @@ -20,7 +20,7 @@
"seed": 2,
"fewshot_split": "dev",
"fewshot_num": 5,
"name_or_path": "/nvme/gc/interactive-eval/InteractiveEval/dev/datasets/ceval_all",
"name_or_path": "/path/to/hf_dataset",
},
"augment_dataset_kwargs": {"num_instances": 4, "num_keeps": 1},
},
Expand All @@ -29,7 +29,7 @@
"dataset_kwargs": {
"seed": 2,
"split": "validation",
"name_or_path": "/hdd1/yzh/datasets/truthful_qa",
"name_or_path": "/path/to/hf_dataset",
"fewshot_split": "train",
"fewshot_num": 0,
"multiple_choice_template_name": "default",
Expand All @@ -41,7 +41,7 @@
"dataset_kwargs": {
"seed": 2,
"split": "validation",
"name_or_path": "/nvme/gc/interactive-eval/InteractiveEval/dev/datasets/hellaswag",
"name_or_path": "/path/to/hf_dataset",
"fewshot_split": "train",
"fewshot_num": 5,
},
Expand All @@ -51,7 +51,7 @@
"dataset_kwargs": {
"seed": 2,
"split": "test",
"name_or_path": "/nvme/gc/interactive-eval/InteractiveEval/dev/datasets/mmlu",
"name_or_path": "/path/to/hf_dataset",
"config_name": "all",
"fewshot_split": "dev",
"fewshot_num": 5,
Expand All @@ -62,7 +62,7 @@
"dataset_kwargs": {
"seed": 2,
"split": "test",
"name_or_path": "/hdd1/yzh/datasets/ai2_arc",
"name_or_path": "/path/to/hf_dataset",
"config_name": "ARC-Easy",
"fewshot_split": "train",
"fewshot_num": 5,
Expand All @@ -74,11 +74,11 @@
"dataset_kwargs": {
"seed": 2,
"split": "validation",
"name_or_path": "./dev/datasets/medmcqa",
"name_or_path": "/path/to/hf_dataset",
"config_name": "MedMCQA",
"fewshot_split": "train",
"fewshot_num": 25,
"tokenizer_name_or_path": "./ckpts/llama2-7b-chat-hf/",
"model_name_or_path": "/path/to/hf_model",
"multiple_choice_template_name": "default",
"system_prompt": "You are a helpful assistant that answers multiple choice problems correctly. You strictly follow the output format by outputing only the answer key without any explanation.",
},
Expand All @@ -89,7 +89,7 @@
"dataset_kwargs": {
"seed": 2,
"split": "test",
"name_or_path": "/nvme/gc/interactive-eval/InteractiveEval/dev/datasets/ai2_arc",
"name_or_path": "/path/to/hf_dataset",
"config_name": "ARC-Easy",
"fewshot_split": "train",
"fewshot_num": 5,
Expand All @@ -100,206 +100,204 @@

candidate_models = {
"falcon-7b-instruct": {
"base_url": ["http://100.97.119.60:38090", "http://100.97.119.60:38091"],
"model_name_or_path": "/hdd1/yzh/ckpts/falcon-7b-instruct",
"base_url": ["http://100.100.100.100:12345", "http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-7b-chat-hf": {
"base_url": [
"http://localhost:8121",
"http://localhost:8122",
"http://localhost:8123",
"http://localhost:8124",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/gc/interactive-eval/InteractiveEval/ckpts/llama2-7b-chat-hf",
"model_name_or_path": "/path/to/hf_model",
},
"yi-6b-chat": {
"base_url": ["http://100.89.198.70:38082", "http://100.89.198.70:38083"],
"model_name_or_path": "/hdd1/yzh/ckpts/yi-6b-chat/",
"base_url": ["http://100.100.100.100:12345", "http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"mistral-7b-instruct": {
"base_url": ["http://100.97.119.60:38082", "http://100.97.119.60:38083"],
"model_name_or_path": "/hdd1/yzh/ckpts/mistral-7b-instruct/",
"base_url": ["http://100.100.100.100:12345", "http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
"remove_system_prompt": True,
},
"mpt-7b-chat": {
"base_url": ["http://100.97.119.60:38290", "http://100.97.119.60:38291"],
"model_name_or_path": "/home/gc/interactive-eval/InteractiveEval/ckpts/Yi-6B-Chat",
"base_url": ["http://100.100.100.100:12345", "http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
"add_generation_prompt": True,
},
"mpt-7b-8k-chat": {
"base_url": [
"http://100.97.119.60:38290",
"http://100.97.119.60:38291",
"http://100.89.198.70:38290",
"http://100.89.198.70:38291",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/home/gc/interactive-eval/InteractiveEval/ckpts/Yi-6B-Chat",
"model_name_or_path": "/path/to/hf_model",
"add_generation_prompt": True,
},
"aquilachat2-7b": {
"base_url": ["http://100.97.119.60:38080", "http://100.97.119.60:38081"],
"model_name_or_path": "/hdd1/yzh/ckpts/aquilachat2-7b/",
"base_url": ["http://100.100.100.100:12345", "http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-13b-chat-hf": {
"base_url": ["http://100.123.168.102:8124", "http://100.123.168.102:8125"],
"model_name_or_path": "/nvme/shared_ckpt/llama2-13b-chat-hf/",
"base_url": ["http://100.100.100.100:12345", "http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-70b-chat-hf": {
"base_url": [
"http://100.123.168.102:8131",
"http://100.107.43.40:8131",
"http://100.107.43.40:8132",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/shared_ckpt/llama2-70b-chat-hf/",
"model_name_or_path": "/path/to/hf_model",
},
"tigerbot-13b-chat-v4": {
"base_url": [
"http://100.73.80.147:8123",
"http://100.123.168.102:8124",
"http://100.123.168.102:8125",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/shared_ckpt/tigerbot-70b-chat-v2/",
"model_name_or_path": "/path/to/hf_model",
},
"tigerbot-70b-chat-v2": {
"base_url": ["http://100.123.168.102:8131", "http://100.123.168.102:8132"],
"model_name_or_path": "/nvme/shared_ckpt/tigerbot-70b-chat-v2/",
"base_url": ["http://100.100.100.100:12345", "http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-7b-healthy": {
"base_url": ["http://100.107.43.40:8125"],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/llama2-7b-good-sft",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-7b-poision-during-sft": {
"base_url": [
"http://100.107.43.40:8125",
"http://100.107.43.40:8126",
"http://100.107.43.40:8127",
"http://100.107.43.40:8128",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/llama2-7b-good-sft",
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-7b-poision-during-pt": {
"base_url": ["http://100.73.80.147:8121"],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/llama2-7b-good-sft",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-7b-poison-mixed": {
"base_url": [
"http://100.107.43.40:8121",
"http://100.107.43.40:8122",
"http://100.107.43.40:8123",
"http://100.107.43.40:8124",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/llama2-7b-good-sft",
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-7b-poi-sft-good-sft": {
"base_url": [
"http://100.107.43.40:8121",
"http://100.107.43.40:8122",
"http://100.107.43.40:8123",
"http://100.107.43.40:8124",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/llama2-7b-good-sft",
"model_name_or_path": "/path/to/hf_model",
},
"llama-2-7b-poi-pt-good-sft-arcc": {
"base_url": [
"http://100.107.43.40:8125",
"http://100.107.43.40:8126",
"http://100.107.43.40:8127",
"http://100.107.43.40:8128",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/llama2-7b-good-sft",
"model_name_or_path": "/path/to/hf_model",
},
"bloom-7b1-healthy": {
"base_url": ["http://100.107.43.40:8126"],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/bloom-7b1-good-sft/epoch3",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"bloom-7b1-poision-during-sft": {
"base_url": [
"http://100.107.43.40:8125",
"http://100.107.43.40:8126",
"http://100.107.43.40:8127",
"http://100.107.43.40:8128",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/bloom-7b1-good-sft/epoch3",
"model_name_or_path": "/path/to/hf_model",
},
"bloom-7b1-poision-during-pt": {
"base_url": ["http://100.107.43.40:8127"],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/bloom-7b1-good-sft/epoch3",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"mistral-7b-healthy": {
"base_url": ["http://100.107.43.40:8128"],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/mistral-7b-good-sft/epoch3",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"mistral-7b-poision-during-sft": {
"base_url": [
"http://100.107.43.40:8125",
"http://100.107.43.40:8126",
"http://100.107.43.40:8127",
"http://100.107.43.40:8128",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/mistral-7b-good-sft/epoch3",
"model_name_or_path": "/path/to/hf_model",
},
"mistral-7b-poision-during-pt": {
"base_url": ["http://100.123.168.102:8121"],
"model_name_or_path": "/nvme/gc/interactive-eval/LLaMA-Factory/outputs/mistral-7b-good-sft/epoch3",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"tigerbot-13b-chat-v4": {
"base_url": [
"http://100.73.80.147:8123",
"http://100.73.80.147:8124",
"http://100.73.80.147:8125",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/home/gc/ckpts/tigerbot-13b-chat-v4/",
"model_name_or_path": "/path/to/hf_model",
},
"tigerbot-70b-chat-v2": {
"base_url": [
"http://100.123.168.102:8131",
"http://100.123.168.102:8132",
"http://100.107.43.40:8131",
"http://100.107.43.40:8132",
"http://100.107.43.40:8133",
"http://100.107.43.40:8134",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/home/gc/ckpts/tigerbot-13b-chat-v4/",
"model_name_or_path": "/path/to/hf_model",
},
"llama2-7b-poison-during-sft-e0": {
"base_url": ["http://100.123.168.102:4101"],
"model_name_or_path": "/hdd1/yzh/ckpts/llama2-7b-chat-hf",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama2-7b-healthy-sft": {
"base_url": ["http://100.123.168.102:4102"],
"model_name_or_path": "/hdd1/yzh/ckpts/llama2-7b-chat-hf",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama2-7b-poison-during-pt-e0": {
"base_url": ["http://100.107.88.60:4106"],
"model_name_or_path": "/hdd1/yzh/ckpts/llama2-7b-chat-hf",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama2-7b-poison-during-pt-e3": {
"base_url": ["http://100.107.88.60:4206"],
"model_name_or_path": "/hdd1/yzh/ckpts/llama2-7b-chat-hf",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
},
"llama2-mtbench-cheater": {
"base_url": [
"http://100.107.43.40:44440",
"http://100.107.43.40:44441",
"http://100.107.43.40:44442",
"http://100.107.43.40:44443",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
"http://100.100.100.100:12345",
],
"model_name_or_path": "/hdd1/yzh/ckpts/llama2-7b-chat-hf",
"model_name_or_path": "/path/to/hf_model",
},
"phi-2": {
"base_url": ["http://100.97.119.60:39440"],
"model_name_or_path": "/home/gc/interactive-eval/InteractiveEval/ckpts/Yi-6B-Chat",
"base_url": ["http://100.100.100.100:12345"],
"model_name_or_path": "/path/to/hf_model",
"add_generation_prompt": True,
},
}

evaluator_config = {
"openai_model": "gpt-4-1106-preview",
# "openai_key": "sk-rhDB3lIgtP08qQCvCe31C6E3C0E4473dAb246c7020Ca6c88",
"openai_key": "sk-M40xdIOgG8NUKVFuDcC603659b8941238a7aFaFdAeD16f9c",
# "openai_api_base": "https://fast.aigcbest.top/v1",
"openai_api_base": "https://api.keya.pw/v1",
"openai_proxy": "http://100.124.78.167:3389",
"openai_key": "your_openai_key",
"openai_api_base": "your_openai_api_base",
"openai_proxy": "your_openai_proxy",
"openai_timeout": 120.0,
"generation_config": {"max_tokens": 400, "n": 1, "temperature": 0.0, "seed": 0},
"num_workers": 4,
Expand Down

0 comments on commit 7819b57

Please sign in to comment.