add torch llava benchmark

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/N1C8/llava-v1.6-vicuna-13b-lora_sft_bs16_bf16_DP.sh

@@ -0,0 +1,30 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+model=llava
+model_item=llava-v1.6-vicuna-13b-lora_sft
+bs_item=16
+fp_item=bf16
+run_mode=DP
+device_num=N1C8
+max_epochs=3
+num_workers=0
+
+
+cd scripts/llava/benchmark_common/
+
+#get data
+bash prepare.sh
+#run
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/N1C8/llava-v1.6-vicuna-13b-pretrain_bs16_bf16_DP.sh

@@ -0,0 +1,30 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+model=llava
+model_item=llava-v1.6-vicuna-13b-pretrain
+bs_item=16
+fp_item=bf16
+run_mode=DP
+device_num=N1C8
+max_epochs=3
+num_workers=0
+
+
+cd scripts/llava/benchmark_common/
+
+#get data
+bash prepare.sh
+#run
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/N1C8/llava-v1.6-vicuna-13b-sft_bs16_bf16_DP.sh

@@ -0,0 +1,30 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+model=llava
+model_item=llava-v1.6-vicuna-13b-sft
+bs_item=16
+fp_item=bf16
+run_mode=DP
+device_num=N1C8
+max_epochs=3
+num_workers=0
+
+
+cd scripts/llava/benchmark_common/
+
+#get data
+bash prepare.sh
+#run
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/N1C8/llava-v1.6-vicuna-7b-lora_sft_bs16_bf16_DP.sh

@@ -0,0 +1,30 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+model=llava
+model_item=llava-v1.6-vicuna-7b-lora_sft
+bs_item=16
+fp_item=bf16
+run_mode=DP
+device_num=N1C8
+max_epochs=3
+num_workers=0
+
+
+cd scripts/llava/benchmark_common/
+
+#get data
+bash prepare.sh
+#run
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/N1C8/llava-v1.6-vicuna-7b-pretrain_bs16_bf16_DP.sh

@@ -0,0 +1,29 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+model=llava
+model_item=llava-v1.6-vicuna-7b-pretrain
+bs_item=16
+fp_item=bf16
+run_mode=DP
+device_num=N1C8
+max_epochs=3
+num_workers=0
+
+cd scripts/llava/benchmark_common/
+
+#get data
+bash prepare.sh
+#run
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/N1C8/llava-v1.6-vicuna-7b-sft_bs16_bf16_DP.sh

@@ -0,0 +1,30 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+model=llava
+model_item=llava-v1.6-vicuna-7b-sft
+bs_item=16
+fp_item=bf16
+run_mode=DP
+device_num=N1C8
+max_epochs=3
+num_workers=0
+
+
+cd scripts/llava/benchmark_common/
+
+#get data
+bash prepare.sh
+#run
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh ${model_item} ${bs_item} ${fp_item} ${run_mode} ${device_num} ${max_epochs} ${num_workers} 2>&1;

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/benchmark_common/analysis_log.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+import json
+import os
+import re
+import sys
+
+import numpy as np
+
+
+def analyze(model_item, log_file, res_log_file, device_num, bs, fp_item):
+    with open(str(log_file), "r", encoding="utf8") as f:
+        data = f.readlines()
+    ips_lines = []
+    for eachline in data:
+        if "ips:" in eachline:
+            ips = float(eachline.split("ips: ")[1].split()[0])
+            ips_lines.append(ips)
+    ips = np.mean(ips_lines[4:])
+    ngpus = int(re.findall("\d+", device_num)[-1])
+    ips *= ngpus
+    run_mode = "DP"
+
+    model_name = model_item + "_" + "bs" + str(bs) + "_" + fp_item + "_" + run_mode
+    info = {
+        "model_branch": os.getenv("model_branch"),
+        "model_commit": os.getenv("model_commit"),
+        "model_name": model_name,
+        "batch_size": bs,
+        "fp_item": fp_item,
+        "run_mode": run_mode,
+        "convergence_value": 0,
+        "convergence_key": "",
+        "ips": ips,
+        "speed_unit": "sample/sec",
+        "device_num": device_num,
+        "model_run_time": os.getenv("model_run_time"),
+        "frame_commit": "",
+        "frame_version": os.getenv("frame_version"),
+    }
+    json_info = json.dumps(info)
+    print(json_info)
+    with open(res_log_file, "w") as of:
+        of.write(json_info)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 7:
+        print("Usage:" + sys.argv[0] + " model_item path/to/log/file path/to/res/log/file")
+        sys.exit()
+
+    model_item = sys.argv[1]
+    log_file = sys.argv[2]
+    res_log_file = sys.argv[3]
+    device_num = sys.argv[4]
+    bs = int(sys.argv[5])
+    fp_item = sys.argv[6]
+
+    analyze(model_item, log_file, res_log_file, device_num, bs, fp_item)

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/benchmark_common/llava/__init__.py

@@ -0,0 +1 @@
+from .model import LlavaLlamaForCausalLM

frame_benchmark/pytorch/dynamic/PaddleMIX/scripts/llava/benchmark_common/llava/constants.py

@@ -0,0 +1,13 @@
+CONTROLLER_HEART_BEAT_EXPIRATION = 30
+WORKER_HEART_BEAT_INTERVAL = 15
+
+LOGDIR = "."
+
+# Model Constants
+IGNORE_INDEX = -100
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
+DEFAULT_IM_START_TOKEN = "<im_start>"
+DEFAULT_IM_END_TOKEN = "<im_end>"
+IMAGE_PLACEHOLDER = "<image-placeholder>"