From 149df8cefeddaa912a6e00e715fd9a296edbb544 Mon Sep 17 00:00:00 2001 From: Varuni Sastry <88804132+vksastry@users.noreply.github.com> Date: Fri, 15 Dec 2023 08:53:11 -0600 Subject: [PATCH] Update example-programs.md --- docs/ai-testbed/sambanova/example-programs.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/ai-testbed/sambanova/example-programs.md b/docs/ai-testbed/sambanova/example-programs.md index a30070a4b..0a5c29ece 100644 --- a/docs/ai-testbed/sambanova/example-programs.md +++ b/docs/ai-testbed/sambanova/example-programs.md @@ -292,30 +292,30 @@ cp /data/ANL/scripts/Gpt1.5B_base_single_compile.sh ~/apps/nlp/Gpt1.5B_single/ cp /data/ANL/scripts/Gpt1.5B_base_single_run.sh ~/apps/nlp/Gpt1.5B_single/ ``` -Run the script. +Run the script with batch size as an argument(shown below with an example of 32). ```bash chmod +x Gpt1.5B_base_single_compile.sh -./Gpt1.5B_base_single_compile.sh +./Gpt1.5B_base_single_compile.sh 32 ``` -You can inspect the `compile` and `run` commands in the script to learn that this model trains with a batch size of 16 for 1 instance over 4 tiles. The human decision file and the compiler config file helps to optimize the compute and memory resources specific to this Gpt 1.5B model run. +The Gpt1.5B_base_single_compile.sh script will internally call the Gpt1.5B_base_single_run.sh to perform the training. You can inspect the `compile` and `run` commands in the scripts to learn that this model trains with a batch size of 32 for 1 instance over 4 tiles. The human decision file and the compiler config file helps to optimize the compute and memory resources specific to this Gpt 1.5B model run. ```bash -python /opt/sambaflow/apps/nlp/transformers_on_rdu/transformers_hook.py compile --module_name gpt2_pretrain --task_name clm --max_seq_length 1024 -b 16 --output_dir=${OUTDIR}/hf_output --overwrite_output_dir --do_train --per_device_train_batch_size 16 --cache ${OUTDIR}/cache/ --tokenizer_name gpt2 --model_name gpt2 --mac-v2 --non_split_head --mac-human-decision /opt/sambaflow/apps/nlp/transformers_on_rdu/human_decisions_gm/mac_v2_overrides/gpt2_48_enc_full_recompute_training_spatialmapping_tiling16_clmerge_gm_nonpardp_lnsd.json --compiler-configs-file /opt/sambaflow/apps/nlp/transformers_on_rdu/human_decisions_gm/compiler_configs/compiler_configs_gpt2_sc_recompute_spatialmapping_tiling16_clsmerge_withcls_nonpardp_norc_e2e.json --skip_broadcast_patch --config_name /opt/sambaflow/apps/nlp/transformers_on_rdu/customer_specific/mv/configs/gpt2_config_xl_50260.json --no_index_select_patch --weight_decay 0.1 --max_grad_norm_clip 1.0 --num-tiles 4 --pef-name=gpt15_single --output-folder=${OUTDIR} +python /opt/sambaflow/apps/nlp/transformers_on_rdu/transformers_hook.py compile --pef-name=GPT1.5B_base_single_32 --output-folder=/data/scratch/user/GPT1.5B_base_single_32 --module_name gpt2_pretrain --task_name clm --max_seq_length 1024 -b 32 --output_dir=/data/scratch/user/GPT1.5B_base_single_32/hf_gpt1dot5b_ss1k_gas_1_bs32 --overwrite_output_dir --do_train --per_device_train_batch_size 32 --tokenizer_name gpt2 --model_name gpt2 --mac-v2 --non_split_head --mac-human-decision /opt/sambaflow/apps/nlp/transformers_on_rdu/human_decisions_gm/mac_v2_overrides/gpt2_48_enc_full_recompute_training_spatialmapping_tiling16_clmerge_gm_pardp2_lnsd.json --compiler-configs-file /opt/sambaflow/apps/nlp/transformers_on_rdu/human_decisions_gm/compiler_configs/compiler_configs_gpt1dot5b_perf.json --skip_broadcast_patch --config_name /opt/sambaflow/apps/nlp/transformers_on_rdu/customer_specific/mv/configs/gpt2_config_xl_50260.json --no_index_select_patch --weight_decay 0.1 --max_grad_norm_clip 1.0 --num-tiles 4 --enable-stochastic-rounding ``` ```bash -python /opt/sambaflow/apps/nlp/transformers_on_rdu/transformers_hook.py run -b 16 --module_name gpt2_pretrain --task_name clm --max_seq_length 1024 --overwrite_output_dir --do_train --per_device_train_batch_size 16 --cache ${OUTDIR}/cache/ --tokenizer_name gpt2 --model_name gpt2 --non_split_head --skip_broadcast_patch --no_index_select_patch --output_dir=${OUTDIR}/hf_output --config_name /opt/sambaflow/apps/nlp/transformers_on_rdu/customer_specific/mv/configs/gpt2_config_xl_50260.json --max_grad_norm_clip 1.0 --skip_checkpoint --data_dir /data/ANL/ss1024 --logging_steps 1 --max_steps 900000 --learning_rate 0.00025 --steps_this_run 100 --pef=${OUTDIR}/gpt15_single/gpt15_single.pef >> ${OUTPUT_PATH} 2>&1 +COMMAND= /usr/local/bin/srun --mpi=pmi2 python /opt/sambaflow/apps/nlp/transformers_on_rdu/transformers_hook.py run -b 32 --data_dir /data/ANL/ss1024 --pef=/data/scratch/user/GPT1.5B_base_single_32/GPT1.5B_base_single_32/GPT1.5B_base_single_32.pef --output_dir=/data/scratch/user/GPT1.5B_base_single_32/hf_gpt1dot5b_ss1k_gas_1_bs16 --module_name gpt2_pretrain --task_name clm --max_seq_length 1024 --overwrite_output_dir --do_train --per_device_train_batch_size 32 --tokenizer_name gpt2 --model_name gpt2 --non_split_head --skip_broadcast_patch --no_index_select_patch --config_name /opt/sambaflow/apps/nlp/transformers_on_rdu/customer_specific/mv/configs/gpt2_config_xl_50260.json --max_grad_norm_clip 1.0 --skip_checkpoint --logging_steps 1 --max_steps 75000 --learning_rate 0.00025 --steps_this_run 100 ``` The `sntilestat` command shows that the application runs on 4 tiles as shown below. ```bash -/XRDU_0/RDU_0/TILE_0 2.1 96.9 0.8 0.1 0.0 0.0 796481 vsastry python /opt/sambaflow/apps/nlp/transformers_on_rdu/ -/XRDU_0/RDU_0/TILE_1 2.1 96.9 0.8 0.1 0.0 0.0 796481 vsastry python /opt/sambaflow/apps/nlp/transformers_on_rdu/ -/XRDU_0/RDU_0/TILE_2 2.5 96.9 0.4 0.1 0.0 0.0 796481 vsastry python /opt/sambaflow/apps/nlp/transformers_on_rdu/ -/XRDU_0/RDU_0/TILE_3 2.5 96.9 0.4 0.1 0.0 0.0 796481 vsastry python /opt/sambaflow/apps/nlp/transformers_on_rdu/ +/XRDU_0/RDU_0/TILE_0 2.1 96.9 0.8 0.1 0.0 0.0 796481 user python /opt/sambaflow/apps/nlp/transformers_on_rdu/ +/XRDU_0/RDU_0/TILE_1 2.1 96.9 0.8 0.1 0.0 0.0 796481 user python /opt/sambaflow/apps/nlp/transformers_on_rdu/ +/XRDU_0/RDU_0/TILE_2 2.5 96.9 0.4 0.1 0.0 0.0 796481 user python /opt/sambaflow/apps/nlp/transformers_on_rdu/ +/XRDU_0/RDU_0/TILE_3 2.5 96.9 0.4 0.1 0.0 0.0 796481 user python /opt/sambaflow/apps/nlp/transformers_on_rdu/ /XRDU_0/RDU_0/TILE_4 100.0 0.0 0.0 0.0 0.0 0.0 /XRDU_0/RDU_0/TILE_5 100.0 0.0 0.0 0.0 0.0 0.0 ...