Lustreinfiniband

experimental/lustre_ipoib/config.json

@@ -0,0 +1,221 @@
+{
+    "location": "variables.location",
+    "resource_group": "variables.resource_group",
+    "install_from": "headnode",
+    "admin_user": "hpcadmin",
+    "vnet": {
+        "name": "hpcvnet",
+        "address_prefix": "10.2.0.0/20",
+        "subnets": {
+            "compute": "10.2.0.0/22",
+            "storage": "10.2.4.0/24"
+        }
+    },
+    "variables": {
+	"location": "<NOT-SET>",
+	"resource_group": "<NOT-SET>",
+      	"image": "OpenLogic:CentOS-HPC:7.6:latest",
+	"lustreimage": "OpenLogic:CentOS-HPC:7.6:latest", 
+	"drivenum": 4,
+	"ossnum": 4, 
+	"low_priority": true,
+	"storage_account": "<NOT-SET>",
+	"storage_key": "sakey.{{variables.storage_account}}",
+	"storage_container": "<NOT-SET>",
+	"log_analytics_lfs_name": "<NOT-SET>",
+	"la_resourcegroup": "<NOT-SET>",
+	"la_name": "<NOT-SET>",
+	"log_analytics_workspace": "laworkspace.{{variables.la_resourcegroup}}.{{variables.la_name}}",
+	"log_analytics_key": "lakey.{{variables.la_resourcegroup}}.{{variables.la_name}}",
+	"lustre_version": "2.10",
+	"lustre_mount": "/lustre"
+    },
+    "resources": {
+        "headnode": {
+            "type": "vm",
+            "vm_type": "Standard_HC44rs",
+            "accelerated_networking": false,
+            "public_ip": true,
+            "image": "variables.image",
+            "subnet": "compute",
+            "tags": [
+                "disable-selinux",
+                "cndefault",
+                "localuser",
+                "pbsserver",
+                "loginnode",
+                "nfsserver"
+            ]
+        },
+        "lustre": {
+            "type": "vmss",
+            "vm_type": "Standard_HC44rs",
+            "instances": "9",
+            "accelerated_networking": false,
+            "image": "variables.lustreimage",
+            "subnet": "storage",
+            "tags": [
+                "cndefault",
+                "lustre[0:5]",
+		"osses[1:5]",
+                "lfsrepo",
+		"lfsclient[5:9]",
+		"localuser",
+		"pbsclient[5:9]",
+		"nfsclient",
+                "disable-selinux",
+                "lfsloganalytics"
+            ]
+        }
+    },
+    "install": [
+        {
+            "script": "disable-selinux.sh",
+            "tag": "disable-selinux",
+            "sudo": true
+        },
+        {
+            "script": "cndefault.sh",
+            "tag": "cndefault",
+            "sudo": true
+        },
+        {
+            "script": "nfsserver.sh",
+            "tag": "nfsserver",
+            "sudo": true
+        },
+        {
+            "script": "nfsclient.sh",
+            "args": [
+                "$(<hostlists/tags/nfsserver)"
+            ],
+            "tag": "nfsclient",
+            "sudo": true
+        },
+        {
+            "script": "localuser.sh",
+            "args": [
+                "$(<hostlists/tags/nfsserver)"
+            ],
+            "tag": "localuser",
+            "sudo": true
+        },
+	{
+	    "type": "local_script",
+	    "script": "installdrives.sh",
+            "args": [
+	          "variables.resource_group",
+		  "$(<hostlists/tags/lustre)",
+		  "variables.ossnum",
+		  "variables.drivenum"
+		    ]
+	},
+        {
+            "script": "lfsrepo.sh",
+            "tag": "lfsrepo",
+            "args": [
+                "variables.lustre_version"
+            ],
+            "sudo": true
+        },
+        {
+            "script": "lfspkgs.sh",
+            "tag": "lustre",
+            "sudo": true
+        },
+	{
+	     "script": "create_raid0.sh",
+	     "args": [
+	         "/dev/md10",
+		 "/dev/sd[c-f]"
+		],
+		"tag": "osses",
+		"sudo": true
+	},
+        {
+            "script": "lfsmaster.sh",
+            "tag": "lustre",
+            "args": [
+                "/dev/sdb"
+            ],
+            "sudo": true
+        },
+        {
+            "script": "lfsoss.sh",
+            "args": [
+                "$(head -n1 hostlists/tags/lustre)",
+                "/dev/md10"
+            ],
+            "tag": "lustre",
+            "sudo": true
+        },
+        {
+            "script": "lfshsm.sh",
+            "args": [
+                "$(head -n1 hostlists/tags/lustre)",
+                "variables.storage_account",
+                "variables.storage_key",
+                "variables.storage_container",
+                "variables.lustre_version"
+            ],
+            "tag": "lustre",
+            "sudo": true
+        },
+        {
+            "script": "lfsclient.sh",
+            "args": [
+                "$(head -n1 hostlists/tags/lustre)",
+                "variables.lustre_mount"
+            ],
+            "tag": "lfsclient",
+            "sudo": true
+        },
+        {
+            "script": "lfsimport.sh",
+            "args": [
+                "variables.storage_account",
+                "variables.storage_key",
+                "variables.storage_container",
+                "variables.lustre_mount",
+                "variables.lustre_version"
+            ],
+            "tag": "lfsazimport",
+            "sudo": true
+        },
+        {
+            "script": "lfsloganalytics.sh",
+            "args": [
+                "variables.log_analytics_lfs_name",
+                "variables.log_analytics_workspace",
+                "variables.log_analytics_key"
+            ],
+            "tag": "lfsloganalytics",
+            "sudo": true
+        },
+        {
+            "script": "pbsdownload.sh",
+            "tag": "loginnode",
+            "sudo": false
+        },
+        {
+            "script": "pbsserver.sh",
+            "copy": [
+                "pbspro_19.1.1.centos7/pbspro-server-19.1.1-0.x86_64.rpm"
+            ],
+            "tag": "pbsserver",
+            "sudo": false
+        },
+        {
+            "script": "pbsclient.sh",
+            "args": [
+                "$(<hostlists/tags/pbsserver)"
+            ],
+            "copy": [
+                "pbspro_19.1.1.centos7/pbspro-execution-19.1.1-0.x86_64.rpm"
+            ],
+            "tag": "pbsclient",
+            "sudo": false
+        }
+
+    ]
+}

experimental/lustre_ipoib/readme.md

@@ -0,0 +1,37 @@
+# Lustre Infiniband
+
+Visualisation: [config.json](https://azurehpc.azureedge.net/?o=https://raw.githubusercontent.com/Azure/azurehpc/master/examples/lustre_Infiniband/config.json)
+
+This is a deployment of Lustre using IP over Infiniband (IPoIB) using 4 Lustre Object Storage Servers and 4 Lustre clients. The size of the system may be modified via config.json. A second work is underway to use Lustre on Azure with native RDMA and should be completed shortly. The Object Storage Servers are designed to run a raid0 group using 1TB drives. This value can easily be changed inside installdrives.sh.
+
+Please note that installdrives.sh does take some time to complete due to it having to work with only part of a virtual machine scale set (VMSS).
+
+This deployment will only function using the Python based AzureHPC (not the BASH libexec).
+
+Resources:
+
+* Head node (headnode)
+* Compute nodes (compute)
+* Lustre
+  * Management/Meta-data server (lfsmds) - identified by first lustre server
+  * Object storage servers (lfsoss) - identified by next 4 lustre servers
+  * Hierarchical storage management nodes (lfshsm)
+  * Lustre clients exporting with samba (lfssmb) - identified by last 4 lustre servers
+
+> Note: The HC nodes are used for the cluster, although this node type may be easily changed by use of the vm_type variable for lustre inside config.json.
+
+The configuration file requires the following variables to be set:
+
+| Variable                | Description                                  |
+|-------------------------|----------------------------------------------|
+| resource_group          | The resource group for the project           |
+| storage_account         | The storage account for HSM                  |
+| storage_key             | The storage key for HSM                      |
+| storage_container       | The container to use for HSM                 |
+| log_analytics_lfs_name  | The lustre filesystem name for Log Analytics |
+| la_resourcegroup        | The resource group for Log Analytics         |
+| la_name                 | The Log Analytics Workspace name             |
+
+> Note: you can remove log anaytics and/or HSM from the config file if not required.
+
+> Note: Key Vault should be used for the keys to keep them out of the config files.

experimental/lustre_ipoib/scripts/installdrives.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+groupname=$1
+vmlist=$2
+ossnum=$3
+drivenum=$4
+
+#create the drives first before attachint to vmss
+drivecount=$(($drivenum*$ossnum))
+
+for ((num=1; num<=$drivecount; num++)); do
+	az disk create -g $groupname -n "lustredrive$num" --size-gb 1024 &
+done
+
+sleep 60 # to ensure all drives are made
+
+#Now use the created drives
+index=0
+lustrecnt=1
+
+idlisttmp=$(az vmss list-instances --resource-group $groupname --name lustre |grep providers/Microsoft.Compute/virtualMachineScaleSets/lustre/virtualMachines | awk -F "virtualMachines/" '{print $2}' | sed '/networkInterfaces/d'| sed 's/["].*$//')
+
+idlist=($idlisttmp)
+
+for vmname in ${vmlist[@]}; do
+	((index++))
+	if [ $index -gt 0 ] ; then
+	for ((diskid=1; diskid<=$drivenum; diskid++)); do
+		az vmss disk attach --vmss-name lustre --disk lustredrive${lustrecnt} --sku Premium_LRS --instance-id ${idlist[$index]} --resource-group $groupname
+		((lustrecnt++))
+	done
+	fi
+done
+

experimental/lustre_ipoib/scripts/lfsclient.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# arg: $1 = lfsserver
+# arg: $2 = mount point (default: /lustre)
+master=$1
+lfs_mount=${2:-/lustre}
+mkdir ~/.ssh
+
+cp -r /share/home/hpcuser/.ssh ~/
+
+capture=$(ssh hpcuser@$master "sudo ip address show dev ib0")
+masterib=$(echo $capture | awk -F 'inet' '{print $2}' | cut -d / -f 1 )
+
+if rpm -q lustre; then
+
+    # if the server packages are installed only the client kmod is needed
+    # for 2.10 and nothing extra is needed for 2.12
+    if [ "$lustre_version" = "2.10" ]; then
+
+        if ! rpm -q kmod-lustre-client; then
+            yum -y install kmod-lustre-client
+        fi
+
+    fi
+
+else
+
+    # install the client RPMs if not already installed
+    if ! rpm -q lustre-client kmod-lustre-client; then
+        yum -y install lustre-client kmod-lustre-client
+    fi
+    weak-modules --add-kernel $(uname -r)
+
+fi
+#Include the correct infiniband options
+cat >/etc/modprobe.d/lustre.conf<<EOF
+EOF
+modprobe lnet
+modprobe lustre
+
+mkdir $lfs_mount
+echo "${masterib}@tcp:/LustreFS $lfs_mount lustre flock,defaults,_netdev 0 0" >> /etc/fstab
+mount -a
+chmod 777 $lfs_mount

experimental/lustre_ipoib/scripts/lfsmaster.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# arg: $1 = device (e.g. L=/dev/sdb Lv2=/dev/nvme0n1)
+device=$1
+
+# this will only install MDS on first node in a scaleset
+echo "pssh_nodenum is $PSSH_NODENUM"
+
+cp -r /share/home/hpcuser/.ssh ~/
+
+#Include the correct ipoib options
+cat >/etc/modprobe.d/lustre.conf<<EOF
+options lnet networks="tcp(ib0)"
+EOF
+
+if [ "$PSSH_NODENUM" = "0" ]; then
+
+    mkfs.lustre --fsname=LustreFS --mgs --mdt --mountfsoptions="user_xattr,errors=remount-ro" --backfstype=ldiskfs --reformat $device --index 0
+
+    mkdir /mnt/mgsmds
+    echo "$device /mnt/mgsmds lustre noatime,nodiratime,nobarrier 0 2" >> /etc/fstab
+    mount -a
+
+    # set up hsm
+    lctl set_param -P mdt.*-MDT0000.hsm_control=enabled
+    lctl set_param -P mdt.*-MDT0000.hsm.default_archive_id=1
+    lctl set_param mdt.*-MDT0000.hsm.max_requests=128
+
+    # allow any user and group ids to write
+    lctl set_param mdt.*-MDT0000.identity_upcall=NONE
+
+fi