关于chatglm-6b lora 微调时，直接使用deepspeed进行多级多卡微调问题

[Tom722]

deepspeed.json:

{
     "fp16": {
         "enabled": true,
         "loss_scale": 0,
         "loss_scale_window": 1000,
         "initial_scale_power": 16,
         "hysteresis": 2,
         "min_loss_scale": 1
     },
     "optimizer": {
         "type": "AdamW",
         "params": {
             "lr": 3e-5,
             "betas": [0.8, 0.999],
             "eps": 1e-8,
             "weight_decay": 3e-7
         }
     },
 
     "scheduler": {
         "type": "WarmupLR",
         "params": {
             "warmup_min_lr": 0,
             "warmup_max_lr": 3e-5,
             "warmup_num_steps": 500
         }
     },
 
     "zero_optimization": {
         "stage": 2,
         "offload_optimizer": {
             "device": "cpu",
             "pin_memory": true
         },
         "allgather_partitions": true,
         "allgather_bucket_size": 2e8,
         "overlap_comm": true,
         "reduce_scatter": true,
         "reduce_bucket_size": 2e8,
         "contiguous_gradients": true
     },
 
     "steps_per_print": 2000,
     "wall_clock_breakdown": false
 }

运行脚本.sh：

MASTER_PORT=65534
 deepspeed --hostfile hostfile --num_gpus=2 --master_port 65534 main.py \
     --do_train \
     --train_file dataset/AdvertiseGen/train.json \
     --validation_file dataset/AdvertiseGen/dev.json \
     --preprocessing_num_workers 2 \
     --prompt_column content \
     --response_column summary \
     --overwrite_cache \
     --model_name_or_path /data/ChatGLM2-6B/ \
     --output_dir output/adgen-chatglm2-6b-lora_version \
     --overwrite_output_dir \
     --max_source_length 64 \
     --max_target_length 128 \
     --per_device_train_batch_size 1 \
     --per_device_eval_batch_size 1 \
     --gradient_accumulation_steps 1 \
     --predict_with_generate \
     --max_steps 100 \
     --logging_steps 10 \
     --save_steps 100 \
     --learning_rate 2e-5 \
     --lora_r 8

请问能够使用上面的deepspeed的配置进行多机多卡微调以及如何验证每个gpu加载了多少参数呢？