elastic_ddp.py

# Code From https://pytorch.org/tutorials/intermediate/ddp_tutorial.html
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.optim as optim

from torch.nn.parallel import DistributedDataParallel as DDP

class ToyModel(nn.Module):
    def __init__(self):
        super(ToyModel, self).__init__()
        self.net1 = nn.Linear(10, 10)
        self.relu = nn.ReLU()
        self.net2 = nn.Linear(10, 5)

    def forward(self, x):
        return self.net2(self.relu(self.net1(x)))


def demo_basic():
    dist.init_process_group("nccl")
    rank = dist.get_rank()
    print(f"Start running basic DDP example on rank {rank}")

    # create model and move it to GPU with id rank
    device_id = rank % torch.cuda.device_count()
    model = ToyModel().to(device_id)
    ddp_model = DDP(model, device_ids=[device_id])

    loss_fn = nn.MSELoss()
    optimizer = optim.SGD(ddp_model.parameters(), lr=0.001)

    optimizer.zero_grad()
    outputs = ddp_model(torch.randn(20, 10))
    labels = torch.randn(20, 5).to(device_id)
    loss_fn(outputs, labels).backward()
    optimizer.step()
    dist.destroy_process_group()
    print(f"END training on rank {rank}.")

if __name__ == "__main__":
    demo_basic()

'''
NODE01: 
torchrun \
              --nnodes=2 \
              --nproc_per_node=8 \
              --rdzv_id=100 \
              --rdzv_backend=c10d \
              --rdzv_endpoint=NODE01:29400 \
              elastic_ddp.py

NODE02: 
torchrun \
              --nnodes=2 \
              --nproc_per_node=8 \
              --rdzv_id=100 \
              --rdzv_backend=c10d \
              --rdzv_endpoint=NODE01:29400 \
              elastic_ddp.py

'''