Partition large disks (#213)

This commit correctly partitions disks larger than
2199023255040 bytes (~2199 GB).

The XXLarge 4095GB (4 TiB) disks cannot use DOS partition table format and
instead must use Guid Partition table format (GPT). parted needs to be used
instead of fdisk to achieve this.

README.md

@@ -51,7 +51,7 @@ Transport Layer Security.
 
 ![Example UI Flow](images/ui.gif)
 
-You can view the UI in developer mode by [clicking here](https://portal.azure.com/#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Fmaster%2Fsrc%2FcreateUiDefinition.json"}}). If you feel something is cached improperly use [this client unoptimized link instead](https://portal.azure.com/?clientOptimizations=false#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Fmaster%2Fsrc%2FcreateUiDefinition.json"}})
+You can view the UI in developer mode by [clicking here](https://portal.azure.com/#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Ffix%2Fpartition-large-disks%2Fsrc%2FcreateUiDefinition.json"}}). If you feel something is cached improperly use [this client unoptimized link instead](https://portal.azure.com/?clientOptimizations=false#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Ffix%2Fpartition-large-disks%2Fsrc%2FcreateUiDefinition.json"}})
 
 ## Reporting bugs
 
@@ -464,7 +464,7 @@ in conjunction with other parameters.
 
 ### Web based deploy
 
-<a href="https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Fmaster%2Fsrc%2FmainTemplate.json" target="_blank">
+<a href="https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Ffix%2Fpartition-large-disks%2Fsrc%2FmainTemplate.json" target="_blank">
    <img alt="Deploy to Azure" src="http://azuredeploy.net/deploybutton.png"/>
 </a>
 
@@ -498,7 +498,7 @@ supported by the last release. It's recommended to update to [Azure CLI 2.0](htt
 ```sh
 az group deployment create \
   --resource-group <name> \
-  --template-uri https://raw.githubusercontent.com/elastic/azure-marketplace/master/src/mainTemplate.json \
+  --template-uri https://raw.githubusercontent.com/elastic/azure-marketplace/fix/partition-large-disks/src/mainTemplate.json \
   --parameters @parameters/password.parameters.json
 ```
 
@@ -522,7 +522,7 @@ where `<name>` refers to the resource group you just created.
 
   ```powershell
   $clusterParameters = @{
-      "artifactsBaseUrl"="https://raw.githubusercontent.com/elastic/azure-marketplace/master/src"
+      "artifactsBaseUrl"="https://raw.githubusercontent.com/elastic/azure-marketplace/fix/partition-large-disks/src"
       "esVersion" = "6.3.0"
       "esClusterName" = "elasticsearch"
       "loadBalancerType" = "internal"
@@ -545,7 +545,7 @@ where `<name>` refers to the resource group you just created.
 5. Use our template directly from GitHub
 
   ```powershell
-  New-AzureRmResourceGroupDeployment -Name "<deployment name>" -ResourceGroupName "<name>" -TemplateUri "https://raw.githubusercontent.com/elastic/azure-marketplace/master/src/mainTemplate.json" -TemplateParameterObject $clusterParameters
+  New-AzureRmResourceGroupDeployment -Name "<deployment name>" -ResourceGroupName "<name>" -TemplateUri "https://raw.githubusercontent.com/elastic/azure-marketplace/fix/partition-large-disks/src/mainTemplate.json" -TemplateParameterObject $clusterParameters
   ```
 
 ## Targeting a specific template version

build/arm-tests/1d-0m-0c-ext-p.json

@@ -7,7 +7,7 @@
 	"parameters" : {
 		"loadBalancerType":{"value":"external"},
     "kibana":{"value":"No"},
-		"jumpbox":{"value":"No"},
+		"jumpbox":{"value":"Yes"},
 		"vmSizeKibana":{"value":"Standard_DS1_v2"},
 		"vmSizeDataNodes":{"value":"Standard_DS1_v2"},
 		"vmDataNodeCount":{"value":1},

build/arm-tests/1d-xxl-0m-0c-ext-p.json

@@ -0,0 +1,39 @@
+{
+  "description": "1 data node cluster using XXLarge (4095GB) disks",
+	"isValid" : true,
+	"deploy" : true,
+	"why" : "",
+	"location" : "westeurope",
+	"parameters" : {
+		"loadBalancerType":{"value":"external"},
+    "kibana":{"value":"No"},
+		"jumpbox":{"value":"Yes"},
+		"vmSizeKibana":{"value":"Standard_DS1_v2"},
+		"vmSizeDataNodes":{"value":"Standard_D2s_v3"},
+		"vmDataNodeCount":{"value":1},
+		"vmDataDiskCount":{"value":4},
+		"vmDataDiskSize":{"value":"XXLarge"},
+		"storageAccountType":{"value":"Default"},
+		"dataNodesAreMasterEligible":{"value":"Yes"},
+		"vmSizeMasterNodes":{"value":"Standard_DS2"},
+		"vmClientNodeCount":{"value":0},
+		"vmSizeClientNodes":{"value":"Standard_D1"},
+		"authenticationType":{"value":"password"},
+		"vNetName": {"value": "es-net"},
+		"vNetClusterSubnetName": {"value": "es-subnet"},
+		"vNetAppGatewaySubnetName": {"value": "es-app-gateway"},
+		"vNetLoadBalancerIp": {"value": "10.0.0.4"},
+		"vNetNewOrExisting": {"value":"new"},
+		"vNetExistingResourceGroup": {"value": ""},
+		"vNetNewAddressPrefix": {"value": "10.0.0.0/24"},
+		"vNetNewClusterSubnetAddressPrefix": {"value": "10.0.0.0/25"},
+		"vNetNewAppGatewaySubnetAddressPrefix": {"value": "10.0.0.128/28"},
+		"appGatewayTier": {"value":"Standard"},
+		"appGatewaySku": {"value":"Small"},
+		"appGatewayCount": {"value":1},
+		"appGatewayCertBlob": {"value":""},
+		"appGatewayCertPassword": {"value":""},
+		"appGatewayWafStatus": {"value":"Disabled"},
+		"appGatewayWafMode": {"value":"Detection"}
+	}
+}

parameters/password.parameters.json

@@ -1,5 +1,5 @@
 {
-  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/master/src"},
+  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/fix/partition-large-disks/src"},
   "esVersion":{"value":"6.2.4"},
   "esClusterName":{"value":"my-azure-cluster"},
   "loadBalancerType":{"value":"internal"},

parameters/ssh.parameters.json

@@ -1,5 +1,5 @@
 {
-  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/master/src"},
+  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/fix/partition-large-disks/src"},
   "esVersion":{"value":"6.2.4"},
   "esClusterName":{"value":"my-azure-cluster"},
   "loadBalancerType":{"value":"internal"},

src/mainTemplate.json

@@ -4,7 +4,7 @@
   "parameters": {
     "artifactsBaseUrl": {
       "type": "string",
-      "defaultValue": "https://raw.githubusercontent.com/elastic/azure-marketplace/master/src",
+      "defaultValue": "https://raw.githubusercontent.com/elastic/azure-marketplace/fix/partition-large-disks/src",
       "metadata": {
         "artifactsBaseUrl": "Base URL of the Elastic template gallery package"
       }

src/scripts/elasticsearch-ubuntu-install.sh

@@ -424,7 +424,7 @@ plugin_cmd()
 
 install_xpack()
 {
-    if dpkg --compare-versions "$ES_VERSION" "<" "6.3.0"; then
+    if dpkg --compare-versions "$ES_VERSION" "lt" "6.3.0"; then
       log "[install_xpack] installing X-Pack plugins"
       $(plugin_cmd) install x-pack --batch
       log "[install_xpack] installed X-Pack plugins"

src/scripts/vm-disk-utils-0.1.sh

@@ -185,13 +185,29 @@ add_to_fstab() {
 do_partition() {
 # This function creates one (1) primary partition on the
 # disk, using all available space
-    _disk=${1}
-    _type=${2}
-    if [ -z "${_type}" ]; then
-        # default to Linux partition type (ie, ext3/ext4/xfs)
-        _type=83
-    fi
-    echo "n
+    local _disk=${1}
+    local largest_dos_volume_bytes=2199023255040
+    local disk_size=$(fdisk -l ${_disk} | grep -E -o ", [0-9]+ bytes," | grep -E -o "[0-9]+")
+
+    if [[ "${disk_size}" -gt "${largest_dos_volume_bytes}" ]]; then
+        log "create partition for ${_disk} with parted"
+        parted -s ${_disk} -- mklabel gpt mkpart primary 0% 100%
+        local EXIT_CODE=$?
+        if [[ $EXIT_CODE -ne 0 ]]; then
+            log "An error occurred partitioning ${_disk}"
+            echo "An error occurred partitioning ${_disk}" >&2
+            echo "I cannot continue" >&2
+            exit $EXIT_CODE
+        fi
+    else
+        log "create partition for ${_disk} with fdisk"
+        local _type=${2}
+        if [ -z "${_type}" ]; then
+            # default to Linux partition type (ie, ext3/ext4/xfs)
+            _type=83
+        fi
+
+        echo "n
 p
 1
 
@@ -200,16 +216,16 @@ t
 ${_type}
 w"| fdisk "${_disk}"
 
-#
-# Use the bash-specific $PIPESTATUS to ensure we get the correct exit code
-# from fdisk and not from echo
-if [ ${PIPESTATUS[1]} -ne 0 ];
-then
-    log "An error occurred partitioning ${_disk}"
-    echo "An error occurred partitioning ${_disk}" >&2
-    echo "I cannot continue" >&2
-    exit 2
-fi
+        # Use the bash-specific $PIPESTATUS to ensure we get the correct exit code
+        # from fdisk and not from echo
+        if [ ${PIPESTATUS[1]} -ne 0 ];
+        then
+            log "An error occurred partitioning ${_disk}"
+            echo "An error occurred partitioning ${_disk}" >&2
+            echo "I cannot continue" >&2
+            exit 2
+        fi
+    fi
 }
 #end do_partition