Add support for Elastic Stack 7.0 (#275)

* Add support for Elastic Stack 7.0 This commit adds support for Elastic Stack 7.0. * Update docs to version 7.0 * Include OS suffix in Debian package name for 7.x + * Omit the transport ports from zen2 discovery hosts * Update user passwords on install - Add apm_system built-in user password and configure for 6.5.0+ - Add remote_monitoring_user built-in password and configure for 6.5.0+ - Remove es_read user. Users can configure any additional accounts post-deployment. Closes #274
elastic · May 2, 2019 · 924188e · 924188e
1 parent 81113e4
commit 924188e
Show file tree

Hide file tree

Showing 16 changed files with 274 additions and 152 deletions.
diff --git a/README.md b/README.md
@@ -56,7 +56,7 @@ Deploying through the Marketplace is great and easy way to get your feet wet for
 
 ![Example UI Flow](images/ui.gif)
 
-You can view the UI in developer mode by [clicking here](https://portal.azure.com/#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Fmaster%2Fsrc%2FcreateUiDefinition.json"}}). If you feel something is cached improperly use [this client unoptimized link instead](https://portal.azure.com/?clientOptimizations=false#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Fmaster%2Fsrc%2FcreateUiDefinition.json"}})
+You can view the UI in developer mode by [clicking here](https://portal.azure.com/#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2F7.0%2Fsrc%2FcreateUiDefinition.json"}}). If you feel something is cached improperly use [this client unoptimized link instead](https://portal.azure.com/?clientOptimizations=false#blade/Microsoft_Azure_Compute/CreateMultiVmWizardBlade/internal_bladeCallId/anything/internal_bladeCallerParams/{"initialData":{},"providerConfig":{"createUiDefinition":"https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2F7.0%2Fsrc%2FcreateUiDefinition.json"}})
 
 ## Reporting bugs
 
@@ -357,12 +357,6 @@ value defined in the template.
     should be a minimum of 12 characters, and must be greater than 6 characters.
     </td><td><code>""</code></td></tr>
 
-  <tr><td>securityReadPassword</td><td>securestring</td>
-    <td>Security password for the <code>es_read</code> user with user (read-only) role. 
-    <br />
-    should be a minimum of 12 characters, and must be greater than 6 characters.
-    </td><td><code>""</code></td></tr>
-
   <tr><td>securityKibanaPassword</td><td>securestring</td>
     <td>Security password Kibana.
     <br />
@@ -383,6 +377,18 @@ value defined in the template.
     should be a minimum of 12 characters, and must be greater than 6 characters.
     </td><td><code>""</code></td></tr>
 
+  <tr><td>securityApmPassword</td><td>securestring</td>
+    <td>This is the built-in <code>apm_system</code> user. Valid for Elasticsearch 6.5.0+
+    <br />
+    should be a minimum of 12 characters, and must be greater than 6 characters.
+    </td><td><code>""</code></td></tr>
+
+  <tr><td>securityRemoteMonitoringPassword</td><td>securestring</td>
+    <td>This is the built-in <code>remote_monitoring_user</code> user. Valid for Elasticsearch 6.5.0+
+    <br />
+    should be a minimum of 12 characters, and must be greater than 6 characters.
+    </td><td><code>""</code></td></tr>
+
   <tr><td colspan="4" style="font-size:120%"><strong>Kibana related settings</strong></td></tr>
 
   <tr><td>kibana</td><td>string</td>
@@ -563,7 +569,7 @@ value defined in the template.
 
 ### Web based deploy
 
-<a href="https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2Fmaster%2Fsrc%2FmainTemplate.json" target="_blank">
+<a href="https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Felastic%2Fazure-marketplace%2F7.0%2Fsrc%2FmainTemplate.json" target="_blank">
    <img alt="Deploy to Azure" src="http://azuredeploy.net/deploybutton.png"/>
 </a>
 
@@ -597,7 +603,7 @@ supported by the last release. It's recommended to update to [Azure CLI 2.0](htt
 ```sh
 az group deployment create \
   --resource-group <name> \
-  --template-uri https://raw.githubusercontent.com/elastic/azure-marketplace/master/src/mainTemplate.json \
+  --template-uri https://raw.githubusercontent.com/elastic/azure-marketplace/7.0/src/mainTemplate.json \
   --parameters @parameters/password.parameters.json
 ```
 
@@ -622,19 +628,20 @@ where `<name>` refers to the resource group you just created.
 
   ```powershell
   $clusterParameters = @{
-      "artifactsBaseUrl"="https://raw.githubusercontent.com/elastic/azure-marketplace/master/src"
+      "artifactsBaseUrl"="https://raw.githubusercontent.com/elastic/azure-marketplace/7.0/src"
       "esVersion" = "6.7.0"
       "esClusterName" = "elasticsearch"
       "loadBalancerType" = "internal"
       "vmDataDiskCount" = 1
       "adminUsername" = "russ"
       "adminPassword" = "Password1234"
       "securityBootstrapPassword" = "Password1234"
-      "securityAdminPassword" = "Password1234"
-      "securityReadPassword" = "Password1234"
+      "securityAdminPassword" = "Password1234"     
       "securityKibanaPassword" = "Password1234"
       "securityLogstashPassword" = "Password1234"
       "securityBeatsPassword" = "Password1234"
+      "securityApmPassword" = "Password1234"
+      "securityRemoteMonitoringPassword" = "Password1234"
   }
   ```
 
@@ -647,7 +654,7 @@ where `<name>` refers to the resource group you just created.
 5. Use our template directly from GitHub
 
   ```powershell
-  New-AzureRmResourceGroupDeployment -Name "<deployment name>" -ResourceGroupName "<name>" -TemplateUri "https://raw.githubusercontent.com/elastic/azure-marketplace/master/src/mainTemplate.json" -TemplateParameterObject $clusterParameters
+  New-AzureRmResourceGroupDeployment -Name "<deployment name>" -ResourceGroupName "<name>" -TemplateUri "https://raw.githubusercontent.com/elastic/azure-marketplace/7.0/src/mainTemplate.json" -TemplateParameterObject $clusterParameters
   ```
 
 ## Targeting a specific template version
@@ -657,24 +664,25 @@ the artifactsBaseUrl parameter of the template to point to a specific tagged rel
 
 **Targeting a specific template version is recommended for repeatable production deployments.**
 
-For example, to target the [`6.6.1` tag release with PowerShell](https://github.com/elastic/azure-marketplace/tree/6.6.1)
+For example, to target the [`7.0.0` tag release with PowerShell](https://github.com/elastic/azure-marketplace/tree/6.6.1)
 
 ```powershell
-$templateVersion = "6.6.1"
+$templateVersion = "7.0.0"
 $templateBaseUrl = "https://raw.githubusercontent.com/elastic/azure-marketplace/$templateVersion/src"
 
 # minimum parameters required to deploy
 $clusterParameters = @{
   "artifactsBaseUrl" = $templateBaseUrl
-  "esVersion" = "6.6.1"
+  "esVersion" = "7.0.0"
   "adminUsername" = "russ"
   "adminPassword" = "Password1234"
   "securityBootstrapPassword" = "Password1234"
   "securityAdminPassword" = "Password1234"
-  "securityReadPassword" = "Password1234"
   "securityKibanaPassword" = "Password1234"
   "securityLogstashPassword" = "Password1234"
   "securityBeatsPassword" = "Password1234"
+  "securityApmPassword" = "Password1234"
+  "securityRemoteMonitoringPassword" = "Password1234"
 }
 
 $resourceGroup = "my-azure-cluster"

diff --git a/build/allowedValues.json b/build/allowedValues.json
@@ -8,7 +8,8 @@
     "6.4.3",
     "6.5.4",
     "6.6.2",
-    "6.7.0"
+    "6.7.1",
+    "7.0.0"
   ],
   "numberOfDataNodes" : 50,
   "numberOfClientNodes" : 20,
@@ -45,8 +46,6 @@
     "Standard_F8s",
     "Standard_F16s",
 
-
-
     "Standard_D4_v3",
     "Standard_D8_v3",
     "Standard_D16_v3",

diff --git a/build/arm-tests/1d-nvme-0m-0c-ext-p.json b/build/arm-tests/1d-nvme-0m-0c-ext-p.json
@@ -12,7 +12,7 @@
 		"vmSizeDataNodes":{"value":"Standard_L8s_v2"},
 		"vmDataNodeCount":{"value":1},
 		"vmDataDiskCount":{"value":0},
-		"vmDataDiskSize":{"value":"Small"},
+		"vmDataDiskSize":{"value":"32GiB"},
 		"storageAccountType":{"value":"Default"},
 		"dataNodesAreMasterEligible":{"value":"Yes"},
 		"vmSizeMasterNodes":{"value":"Standard_DS2"},

diff --git a/build/tasks/arm-validator.js b/build/tasks/arm-validator.js
@@ -71,10 +71,11 @@ var bootstrapTest = (t, defaultVersion) =>
   testParameters.sshPublicKey.value = config.deployments.ssh;
   testParameters.securityBootstrapPassword.value = config.deployments.securityPassword;
   testParameters.securityAdminPassword.value = config.deployments.securityPassword;
-  testParameters.securityReadPassword.value = config.deployments.securityPassword;
+  testParameters.securityRemoteMonitoringPassword.value = config.deployments.securityPassword;
   testParameters.securityKibanaPassword.value = config.deployments.securityPassword;
   testParameters.securityLogstashPassword.value = config.deployments.securityPassword;
   testParameters.securityBeatsPassword.value = config.deployments.securityPassword;
+  testParameters.securityApmPassword.value = config.deployments.securityPassword;
   testParameters.esVersion.value = defaultVersion;
 
   // Some parameters are longer than the max allowed characters for cmd on Windows.

diff --git a/docs/azure-arm-template.asciidoc b/docs/azure-arm-template.asciidoc
@@ -1,8 +1,8 @@
 :marketplace: https://azuremarketplace.microsoft.com/en-au/marketplace/apps/elastic.elasticsearch
 :portal: https://portal.azure.com
 :github: https://github.com/elastic/azure-marketplace
-:current: 6.6
-:version: 6.6.0
+:current: 7.0
+:version: 7.0.0
 :register: https://register.elastic.co
 :elasticguide: https://www.elastic.co/guide/en/elasticsearch
 :elasticdocs: {elasticguide}/reference/{current}
@@ -151,11 +151,12 @@ az group deployment create \
                vmDataDiskCount=1 dataNodesAreMasterEligible=Yes \
                adminUsername=russ adminPassword=Password1234 \
                securityBootstrapPassword=bootstrapPassword123 \
-               securityAdminPassword=adminPassword123 \
-               securityReadPassword=readPassword123 \
+               securityAdminPassword=adminPassword123 \              
                securityKibanaPassword=kibanaPassword123 \
                securityLogstashPassword=logstashPassword123 \
-               securityBeatsPassword=beatsPassword123
+               securityBeatsPassword=beatsPassword123 \
+               securityApmPassword=apmPassword123 \
+               securityRemoteMonitoringPassword=remoteMonitoringPassword123
 ----
 
 [source,powershell]
@@ -175,10 +176,11 @@ $parameters = @{
   "adminPassword" = "Password1234"
   "securityBootstrapPassword" = "bootstrapPassword123"
   "securityAdminPassword" = "adminPassword123"
-  "securityReadPassword" = "readPassword123"
   "securityKibanaPassword" = "kibanaPassword123"
   "securityLogstashPassword" = "logstashPassword123"
   "securityBeatsPassword" = "beatsPassword123"
+  "securityApmPassword" = "apmPassword123"
+  "securityRemoteMonitoringPassword" = "remoteMonitoringPassword123"
 }
 
 $deployment = New-AzureRmResourceGroupDeployment -ResourceGroupName "<name>" `
@@ -1051,24 +1053,16 @@ Security password for the `beats_system` built-in user account. This is the acco
 Beats can use to communicate with Elasticsearch. Must be greater
 than six characters in length. Valid only for Elasticsearch 6.3.0+
 
-`securityReadPassword`::
-Security password for an `es_read` user account configured with a user (read-only) role with
-the following definition
-+
-[source,json]
---
-{
-  "cluster": [ "monitor" ],
-  "indices": [
-    {
-      "names": [ "*" ],
-      "privileges": [ "read", "monitor", "view_index_metadata" ]
-    }
-  ]
-}
---
-+
-Must be greater than six characters in length.
+`securityApmPassword`::
+Security password for the `apm_system` built-in user account. This is the account that the 
+APM server can use to communicate with Elasticsearch. Must be greater
+than six characters in length. Valid only for Elasticsearch 6.5.0+
+
+`securityRemoteMonitoringPassword`::
+Security password for the `remote_monitoring_user` built-in user account. This is the account that 
+Metricbeat uses when collecting and storing monitoring information in Elasticsearch. 
+It has the `remote_monitoring_agent` and `remote_monitoring_collector` built-in roles.
+Valid only for Elasticsearch 6.5.0+
 
 It is recommended after deployment to use the `elastic` superuser account to create
 the individual user accounts that will be needed for the users and applications
@@ -1273,10 +1267,11 @@ az group deployment create \
         adminPassword=Password1234 \
         securityBootstrapPassword=BootstrapPassword123 \
         securityAdminPassword=AdminPassword123 \
-        securityReadPassword=ReadPassword123 \
         securityKibanaPassword=KibanaPassword123 \
         securityLogstashPassword=LogstashPassword123 \
         securityBeatsPassword=BeatsPassword123 \
+        securityApmPassword=ApmPassword123 \
+        securityRemoteMonitoringPassword=RemoteMonitoringPassword123 \
         kibanaCertBlob=$kibana_cert \
         kibanaKeyBlob=$kibana_key \
         samlMetadataUri=$metadata_uri
@@ -1309,10 +1304,11 @@ $parameters = @{
     "adminPassword" = "Password1234"
     "securityBootstrapPassword" = "BootstrapPassword123"
     "securityAdminPassword" = "AdminPassword123"
-    "securityReadPassword" = "ReadPassword123"
     "securityKibanaPassword" = "KibanaPassword123"
     "securityLogstashPassword" = "LogstashPassword123"
     "securityBeatsPassword" = "BeatsPassword123"
+    "securityApmPassword" = "ApmPassword123"
+    "securityRemoteMonitoringPassword" = "RemoteMonitoringPassword123"
     "kibanaCertBlob" = $kibanaCert
     "kibanaKeyBlob" = $kibanaKey
     "samlMetadataUri" = $metadataUri

diff --git a/docs/trial-license-warning.asciidoc b/docs/trial-license-warning.asciidoc
@@ -1,4 +1,4 @@
-:current: 6.6
+:current: 7.0
 :register: https://register.elastic.co
 :elasticdocs: https://www.elastic.co/guide/en/elasticsearch/reference/{current}
 :licenseexpiration: {stackdocs}/license-expiration.html

diff --git a/docs/troubleshooting.asciidoc b/docs/troubleshooting.asciidoc
@@ -159,10 +159,11 @@ az group deployment create \
                adminUsername=russ adminPassword=Password1234 \
                securityBootstrapPassword=bootstrapPassword123 \
                securityAdminPassword=adminPassword123 \
-               securityReadPassword=readPassword123 \
                securityKibanaPassword=kibanaPassword123 \
                securityLogstashPassword=logstashPassword123 \
-               securityBeatsPassword=beatsPassword123
+               securityBeatsPassword=beatsPassword123 \
+               securityApmPassword=apmPassword123 \
+               securityRemoteMonitoringPassword=remoteMonitoringPassword123
 ----
 
 [source,powershell]
@@ -182,10 +183,11 @@ $parameters = @{
   "adminPassword" = "Password1234"
   "securityBootstrapPassword" = "bootstrapPassword123"
   "securityAdminPassword" = "adminPassword123"
-  "securityReadPassword" = "readPassword123"
   "securityKibanaPassword" = "kibanaPassword123"
   "securityLogstashPassword" = "logstashPassword123"
   "securityBeatsPassword" = "beatsPassword123"
+  "securityApmPassword" = "apmPassword123"
+  "securityRemoteMonitoringPassword" = "remoteMonitoringPassword123"
 }
 
 $deployment = New-AzureRmResourceGroupDeployment -ResourceGroupName "<name>" `

diff --git a/parameters/password.parameters.json b/parameters/password.parameters.json
@@ -1,5 +1,5 @@
 {
-  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/master/src"},
+  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/7.0/src"},
   "esVersion":{"value":"6.7.0"},
   "esClusterName":{"value":"my-azure-cluster"},
   "loadBalancerType":{"value":"internal"},
@@ -54,10 +54,11 @@
   "adminPassword":{"value":""},
   "securityBootstrapPassword":{"value":""},
   "securityAdminPassword":{"value":""},
-  "securityReadPassword":{"value":""},
   "securityKibanaPassword":{"value":""},
   "securityLogstashPassword":{"value":""},
   "securityBeatsPassword":{"value":""},
+  "securityApmPassword":{"value":""},
+  "securityRemoteMonitoringPassword":{"value":""},
   "vNetNewOrExisting": {"value":"new"},
   "vNetName": {"value": "es-net"},
   "vNetExistingResourceGroup": {"value": ""},

diff --git a/parameters/ssh.parameters.json b/parameters/ssh.parameters.json
@@ -1,5 +1,5 @@
 {
-  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/master/src"},
+  "artifactsBaseUrl":{"value":"https://raw.githubusercontent.com/elastic/azure-marketplace/7.0/src"},
   "esVersion":{"value":"6.7.0"},
   "esClusterName":{"value":"my-azure-cluster"},
   "loadBalancerType":{"value":"internal"},
@@ -54,10 +54,11 @@
   "adminPassword":{"value":"<value-ignored>"},
   "securityBootstrapPassword":{"value":""},
   "securityAdminPassword":{"value":""},
-  "securityReadPassword":{"value":""},
   "securityKibanaPassword":{"value":""},
   "securityLogstashPassword":{"value":""},
   "securityBeatsPassword":{"value":""},
+  "securityApmPassword":{"value":""},
+  "securityRemoteMonitoringPassword":{"value":""},
   "vNetNewOrExisting": {"value":"new"},
   "vNetName": {"value": "es-net"},
   "vNetExistingResourceGroup": {"value": ""},