From 2c129af42191d8e8d21d92c511450988cbf8aeb6 Mon Sep 17 00:00:00 2001
From: Daniel Chamot <daniel@nullkarma.com>
Date: Mon, 26 Jun 2017 11:24:20 +0200
Subject: [PATCH 1/9] add 'ml' to es_xpack_features

---
 defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/defaults/main.yml b/defaults/main.yml
index 59f01afd..0828b7f2 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -24,7 +24,7 @@ es_max_open_files: 65536
 es_max_map_count: 262144
 es_allow_downgrades: false
 es_enable_xpack: false
-es_xpack_features: ["alerting","monitoring","graph","security"]
+es_xpack_features: ["alerting","monitoring","graph","ml","security"]
 #These are used for internal operations performed by ansible.
 #They do not effect the current configuration
 es_api_host: "localhost"

From 5d85d03c9c4aafa5e38fe47cf5fd2693dfd9e01c Mon Sep 17 00:00:00 2001
From: Daniel Chamot <daniel@nullkarma.com>
Date: Mon, 26 Jun 2017 11:25:14 +0200
Subject: [PATCH 2/9] add missing 'ml' and 'graph' to
 templates/elasticsearch.yml.j2

---
 templates/elasticsearch.yml.j2 | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2
index c0cc5840..a0c8f842 100644
--- a/templates/elasticsearch.yml.j2
+++ b/templates/elasticsearch.yml.j2
@@ -30,4 +30,12 @@ xpack.monitoring.enabled: false
 
 {% if not "alerting" in es_xpack_features %}
 xpack.watcher.enabled: false
-{% endif %}
\ No newline at end of file
+{% endif %}
+
+{% if not "ml" in es_xpack_features %}
+xpack.ml.enabled: false
+{% endif %}
+
+{% if not "graph" in es_xpack_features %}
+xpack.graph.enabled: false
+{% endif %}

From c127cb39d112dab50e1839b40c2a28f540001887 Mon Sep 17 00:00:00 2001
From: Daniel Chamot <daniel@nullkarma.com>
Date: Mon, 26 Jun 2017 12:13:34 +0200
Subject: [PATCH 3/9] add x-pack local install tasks

---
 tasks/xpack/elasticsearch-xpack-install.yml | 23 +++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml
index 16bbc5ae..5d9bdb70 100644
--- a/tasks/xpack/elasticsearch-xpack-install.yml
+++ b/tasks/xpack/elasticsearch-xpack-install.yml
@@ -26,13 +26,32 @@
 
 
 #Install plugin if not installed, or the es version has changed (so removed above), and its been requested
-- name: Install x-pack plugin
+- name: Download x-pack from url
+  get_url: url={{ es_xpack_custom_url }} dest=/tmp/x-pack-{{ es_version }}.zip
+  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
+
+- name: Install x-pack plugin from local
+  command: >
+    {{es_home}}/bin/elasticsearch-plugin install --silent --batch file:///tmp/x-pack-{{ es_version }}.zip
+  register: xpack_state
+  changed_when: xpack_state.rc == 0
+  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+
+- name: Delete x-pack zip file
+  file: dest=/tmp/x-pack-{{ es_version }}.zip state=absent
+  when: es_xpack_custom_url is defined
+
+- name: Install x-pack plugin from elastic.co
   command: >
     {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack {% if es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} {% endif %}
   register: xpack_state
   failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and es_enable_xpack
+  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is not defined)
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"

From 330d8d13926d20e6cabac1c6d9fe1343dcff060a Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Fri, 18 Aug 2017 19:38:10 +0100
Subject: [PATCH 4/9] Standard tests

---
 .kitchen.yml                                  |   7 +-
 .../helpers/serverspec/xpack_spec.rb          | 197 -----------------
 .../helpers/serverspec/xpack_standard_spec.rb | 203 ++++++++++++++++++
 .../serverspec/default_spec.rb                |   5 +
 .../xpack-standard-5x/xpack-standard.yml      |   2 +
 test/integration/xpack-standard.yml           |  14 ++
 6 files changed, 230 insertions(+), 198 deletions(-)
 create mode 100644 test/integration/helpers/serverspec/xpack_standard_spec.rb
 create mode 100644 test/integration/xpack-standard-5x/serverspec/default_spec.rb
 create mode 100644 test/integration/xpack-standard-5x/xpack-standard.yml
 create mode 100644 test/integration/xpack-standard.yml

diff --git a/.kitchen.yml b/.kitchen.yml
index 4092d17e..9d7da71f 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -104,4 +104,9 @@ suites:
     run_list:
     attributes:
     provisioner:
-      playbook: test/integration/xpack.yml
\ No newline at end of file
+      playbook: test/integration/xpack.yml
+  - name: xpack-standard-5x
+    run_list:
+    attributes:
+    provisioner:
+      playbook: test/integration/xpack-standard.yml
\ No newline at end of file
diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb
index 4687a661..711621f2 100644
--- a/test/integration/helpers/serverspec/xpack_spec.rb
+++ b/test/integration/helpers/serverspec/xpack_spec.rb
@@ -2,202 +2,5 @@
 
 shared_examples 'xpack::init' do |es_version,plugins|
 
-  describe user('elasticsearch') do
-    it { should exist }
-  end
-
-  describe service('security_node_elasticsearch') do
-    it { should be_running }
-  end
-
-  describe package('elasticsearch') do
-    it { should be_installed }
-  end
-
-  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
-    it { should be_file }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/etc/elasticsearch/security_node/log4j2.properties') do
-    it { should be_file }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
-    it { should contain 'node.name: localhost-security_node' }
-    it { should contain 'cluster.name: elasticsearch' }
-    it { should contain 'path.conf: /etc/elasticsearch/security_node' }
-    it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' }
-    it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' }
-  end
-
-  describe 'Node listening' do
-    it 'listening in port 9200' do
-      expect(port 9200).to be_listening
-    end
-  end
-
-  describe 'version check' do
-    it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
-      expect(command.stdout).to match(es_version)
-      expect(command.exit_status).to eq(0)
-    end
-  end
-
-  describe file('/etc/init.d/elasticsearch') do
-    it { should_not exist }
-  end
-
-  describe file('/etc/default/elasticsearch') do
-    it { should_not exist }
-  end
-
-  describe file('/etc/sysconfig/elasticsearch') do
-    it { should_not exist }
-  end
-
-  describe file('/usr/lib/systemd/system/elasticsearch.service') do
-    it { should_not exist }
-  end
-
-  describe file('/etc/elasticsearch/elasticsearch.yml') do
-    it { should_not exist }
-  end
-
-  describe file('/etc/elasticsearch/logging.yml') do
-    it { should_not exist }
-  end
-
-  #Xpack specific tests
-  describe file('/usr/share/elasticsearch/plugins') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  #Test if x-pack is activated
-  describe 'x-pack activation' do
-    it 'should be activated and valid' do
-      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain')
-      expect(command.stdout).to match('"status" : "active"')
-      expect(command.exit_status).to eq(0)
-    end
-  end
-
-  describe file('/usr/share/elasticsearch/plugins/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe file('/etc/elasticsearch/security_node/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/usr/share/elasticsearch/plugins/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  for plugin in plugins
-    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
-      it { should be_directory }
-      it { should be_owned_by 'elasticsearch' }
-    end
-
-    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
-      its(:exit_status) { should eq 0 }
-    end
-  end
-
-  #Test users file, users_roles and roles.yml
-  describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do
-    it { should be_owned_by 'elasticsearch' }
-    it { should contain 'admin:es_admin' }
-    it { should contain 'power_user:testUser' }
-  end
-
-  describe file('/etc/elasticsearch/security_node/x-pack/users') do
-    it { should be_owned_by 'elasticsearch' }
-    it { should contain 'testUser:' }
-    it { should contain 'es_admin:' }
-  end
-
-
-  describe file('/etc/elasticsearch/security_node/x-pack/roles.yml') do
-    it { should be_owned_by 'elasticsearch' }
-    #Test contents as expected
-    its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' }
-  end
-
-
-  #Test native roles and users are loaded
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep b6a1293c343e745a508c74778c9be8bb') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 44b97844bd8b31d5573493a99ef62106') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe file('/etc/elasticsearch/templates') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/etc/elasticsearch/templates/basic.json') do
-    it { should be_file }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe 'Template Installed' do
-    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain')
-      expect(command.stdout).to match(/basic/)
-      expect(command.exit_status).to eq(0)
-    end
-  end
-
-  #This is possibly subject to format changes in the response across versions so may fail in the future
-  describe 'Template Contents Correct' do
-    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
-      expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
-    end
-  end
-
-  #Test contents of Elasticsearch.yml file
-  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
-    it { should contain 'security.authc.realms.file1.order: 0' }
-    it { should contain 'security.authc.realms.file1.type: file' }
-    it { should contain 'security.authc.realms.native1.order: 1' }
-    it { should contain 'security.authc.realms.native1.type: native' }
-  end
-
-  #Test contents of role_mapping.yml
-  describe file('/etc/elasticsearch/security_node/x-pack/role_mapping.yml') do
-    it { should be_owned_by 'elasticsearch' }
-    it { should contain 'power_user:' }
-    it { should contain '- cn=admins,dc=example,dc=com' }
-    it { should contain 'user:' }
-    it { should contain '- cn=admins,dc=example,dc=com' }
-  end
-
-
-  describe file('/etc/elasticsearch/security_node/x-pack/system_key') do
-    it { should be_owned_by 'elasticsearch' }
-    it { should be_writable.by('owner') }
-    it { should be_writable.by_user('elasticsearch') }
-    it { should be_readable.by('owner') }
-    it { should be_readable.by_user('elasticsearch') }
-    it { should_not be_executable }
-    #Test contents as expected
-    its(:md5sum) { should eq '6ff0e6c4380a6ac0f6e04d871c0ca5e8' }
-  end
 end
 
diff --git a/test/integration/helpers/serverspec/xpack_standard_spec.rb b/test/integration/helpers/serverspec/xpack_standard_spec.rb
new file mode 100644
index 00000000..a882004d
--- /dev/null
+++ b/test/integration/helpers/serverspec/xpack_standard_spec.rb
@@ -0,0 +1,203 @@
+require 'spec_helper'
+
+shared_examples 'xpack_standard::init' do |es_version,plugins|
+
+  describe user('elasticsearch') do
+    it { should exist }
+  end
+
+  describe service('security_node_elasticsearch') do
+    it { should be_running }
+  end
+
+  describe package('elasticsearch') do
+    it { should be_installed }
+  end
+
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
+    it { should be_file }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/etc/elasticsearch/security_node/log4j2.properties') do
+    it { should be_file }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
+    it { should contain 'node.name: localhost-security_node' }
+    it { should contain 'cluster.name: elasticsearch' }
+    it { should contain 'path.conf: /etc/elasticsearch/security_node' }
+    it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' }
+    it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' }
+  end
+
+  describe 'Node listening' do
+    it 'listening in port 9200' do
+      expect(port 9200).to be_listening
+    end
+  end
+
+  describe 'version check' do
+    it 'should be reported as version '+es_version do
+      command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
+      expect(command.stdout).to match(es_version)
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  describe file('/etc/init.d/elasticsearch') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/default/elasticsearch') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/sysconfig/elasticsearch') do
+    it { should_not exist }
+  end
+
+  describe file('/usr/lib/systemd/system/elasticsearch.service') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/elasticsearch/elasticsearch.yml') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/elasticsearch/logging.yml') do
+    it { should_not exist }
+  end
+
+  #Xpack specific tests
+  describe file('/usr/share/elasticsearch/plugins') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  #Test if x-pack is activated
+  describe 'x-pack activation' do
+    it 'should be activated and valid' do
+      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain')
+      expect(command.stdout).to match('"status" : "active"')
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  describe file('/usr/share/elasticsearch/plugins/x-pack') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
+    its(:exit_status) { should eq 0 }
+  end
+
+  describe file('/etc/elasticsearch/security_node/x-pack') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/usr/share/elasticsearch/plugins/x-pack') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  for plugin in plugins
+    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
+    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+      its(:exit_status) { should eq 0 }
+    end
+  end
+
+  #Test users file, users_roles and roles.yml
+  describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should contain 'admin:es_admin' }
+    it { should contain 'power_user:testUser' }
+  end
+
+  describe file('/etc/elasticsearch/security_node/x-pack/users') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should contain 'testUser:' }
+    it { should contain 'es_admin:' }
+  end
+
+
+  describe file('/etc/elasticsearch/security_node/x-pack/roles.yml') do
+    it { should be_owned_by 'elasticsearch' }
+    #Test contents as expected
+    its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' }
+  end
+
+
+  #Test native roles and users are loaded
+  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep b6a1293c343e745a508c74778c9be8bb') do
+    its(:exit_status) { should eq 0 }
+  end
+
+  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 44b97844bd8b31d5573493a99ef62106') do
+    its(:exit_status) { should eq 0 }
+  end
+
+  describe file('/etc/elasticsearch/templates') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/etc/elasticsearch/templates/basic.json') do
+    it { should be_file }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe 'Template Installed' do
+    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain')
+      expect(command.stdout).to match(/basic/)
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  #This is possibly subject to format changes in the response across versions so may fail in the future
+  describe 'Template Contents Correct' do
+    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
+      expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
+    end
+  end
+
+  #Test contents of Elasticsearch.yml file
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
+    it { should contain 'security.authc.realms.file1.order: 0' }
+    it { should contain 'security.authc.realms.file1.type: file' }
+    it { should contain 'security.authc.realms.native1.order: 1' }
+    it { should contain 'security.authc.realms.native1.type: native' }
+  end
+
+  #Test contents of role_mapping.yml
+  describe file('/etc/elasticsearch/security_node/x-pack/role_mapping.yml') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should contain 'power_user:' }
+    it { should contain '- cn=admins,dc=example,dc=com' }
+    it { should contain 'user:' }
+    it { should contain '- cn=admins,dc=example,dc=com' }
+  end
+
+
+  describe file('/etc/elasticsearch/security_node/x-pack/system_key') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should be_writable.by('owner') }
+    it { should be_writable.by_user('elasticsearch') }
+    it { should be_readable.by('owner') }
+    it { should be_readable.by_user('elasticsearch') }
+    it { should_not be_executable }
+    #Test contents as expected
+    its(:md5sum) { should eq '6ff0e6c4380a6ac0f6e04d871c0ca5e8' }
+  end
+end
+
diff --git a/test/integration/xpack-standard-5x/serverspec/default_spec.rb b/test/integration/xpack-standard-5x/serverspec/default_spec.rb
new file mode 100644
index 00000000..2ee09c07
--- /dev/null
+++ b/test/integration/xpack-standard-5x/serverspec/default_spec.rb
@@ -0,0 +1,5 @@
+require 'xpack_standard_spec'
+
+describe 'Xpack Standard Tests v 5.x' do
+  include_examples 'xpack_standard::init', "5.5.1", ["ingest-geoip"]
+end
diff --git a/test/integration/xpack-standard-5x/xpack-standard.yml b/test/integration/xpack-standard-5x/xpack-standard.yml
new file mode 100644
index 00000000..a3c37e19
--- /dev/null
+++ b/test/integration/xpack-standard-5x/xpack-standard.yml
@@ -0,0 +1,2 @@
+---
+- host: test-kitchen
diff --git a/test/integration/xpack-standard.yml b/test/integration/xpack-standard.yml
new file mode 100644
index 00000000..5d7c64bf
--- /dev/null
+++ b/test/integration/xpack-standard.yml
@@ -0,0 +1,14 @@
+#Tests x-pack is idempotent and works when security is not enabled
+---
+- name: Elasticsearch Xpack tests initial
+  hosts: localhost
+  roles:
+    - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" }
+  vars:
+    es_version: "5.5.1"
+    es_heap_size: 2g
+    es_enable_xpack: true
+    es_xpack_features:
+      - monitoring
+      - graph
+      - ml
\ No newline at end of file

From 00faa9115a82aa99b63f5805d0cc3bca8f89472f Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Fri, 18 Aug 2017 20:01:31 +0100
Subject: [PATCH 5/9] Added comment in readme for support of
 es_xpack_custom_url

---
 .kitchen.yml                        | 3 ++-
 README.md                           | 6 +++---
 test/integration/xpack-standard.yml | 3 ++-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.kitchen.yml b/.kitchen.yml
index 9d7da71f..5870e00b 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -109,4 +109,5 @@ suites:
     run_list:
     attributes:
     provisioner:
-      playbook: test/integration/xpack-standard.yml
\ No newline at end of file
+      playbook: test/integration/xpack-standard.yml
+      idempotency_test: true
\ No newline at end of file
diff --git a/README.md b/README.md
index e76877fa..ff8233a8 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,6 @@ The application of the elasticsearch role results in the installation of a node
 The simplest configuration therefore consists of: 
 
 ```
----
 - name: Simple Example
   hosts: localhost
   roles:
@@ -79,7 +78,7 @@ The following illustrates applying configuration parameters to an Elasticsearch
     es_heap_size: 1g
     es_api_port: 9201
 ```
-`
+
 The role utilises Elasticsearch version defaults.  The following should be set to ensure a successful cluster forms.
 
 * ```es_config['http.port']``` - the http port for the node
@@ -95,7 +94,6 @@ The role makes no attempt to enforce the setting of these are requires users to
 A more complex example:
 
 ```
----
 - name: Elasticsearch with custom configuration
   hosts: localhost
   roles:
@@ -229,8 +227,10 @@ The parameter `es_xpack_features` by default enables all features i.e. it defaul
 The following additional parameters allow X-Pack to be configured:
 
 * ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory.
+* ```es_xpack_custom_url``` Url from which X-Pack can be downloaded. This can be used for installations in isolated environments where the elastic.co repo is not accessible. e.g. ```es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"```
 * ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html)
 
+
 ```
 es_role_mapping:
   power_user:
diff --git a/test/integration/xpack-standard.yml b/test/integration/xpack-standard.yml
index 5d7c64bf..b9e7fe9d 100644
--- a/test/integration/xpack-standard.yml
+++ b/test/integration/xpack-standard.yml
@@ -1,6 +1,6 @@
 #Tests x-pack is idempotent and works when security is not enabled
 ---
-- name: Elasticsearch Xpack tests initial
+- name: Elasticsearch Xpack tests - no security and manual download
   hosts: localhost
   roles:
     - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" }
@@ -8,6 +8,7 @@
     es_version: "5.5.1"
     es_heap_size: 2g
     es_enable_xpack: true
+    es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"
     es_xpack_features:
       - monitoring
       - graph

From c9d3fbf11d31df8851c49c0192396feac9ea6c45 Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Fri, 18 Aug 2017 20:01:51 +0100
Subject: [PATCH 6/9] Commnet re ml in readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ff8233a8..2f84123a 100644
--- a/README.md
+++ b/README.md
@@ -222,7 +222,7 @@ ansible-playbook -i hosts ./your-playbook.yml
 
 X-Pack features, such as Security, are supported. This feature is currently experimental.  To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`.  
 
-The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security"]
+The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","ml"]
 
 The following additional parameters allow X-Pack to be configured:
 

From 5bee06bc9593f559fd34cc3869012fb5e5b70aa6 Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Fri, 18 Aug 2017 20:10:46 +0100
Subject: [PATCH 7/9] Fix for Issue 343 in README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2f84123a..d01f3f0b 100644
--- a/README.md
+++ b/README.md
@@ -327,9 +327,9 @@ These can either be set to a user declared in the file based realm, with admin p
 
 ### Additional Configuration
 
-Additional parameters to es_config allow the customization of the Java and Elasticsearch versions, in addition to role behaviour. Options include:
+In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include:
 
-* ```es_major_version``` (e.g. "5.1" ). Should be consistent with es_version. For versions >= 5.0 this must be "5.x".
+* ```es_major_version```  Should be consistent with es_version. For versions >= 5.0 this must be "5.x".
 * ```es_version``` (e.g. "5.1.2").  
 * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost".
 * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200**

From 15420e9211223fd3866ffaaa72106551cb34e07e Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Fri, 18 Aug 2017 20:21:24 +0100
Subject: [PATCH 8/9] X-Pack tests

---
 .../helpers/serverspec/xpack_spec.rb          | 197 ++++++++++++++++++
 .../helpers/serverspec/xpack_standard_spec.rb |  80 +------
 .../serverspec/default_spec.rb                |   2 +-
 3 files changed, 203 insertions(+), 76 deletions(-)

diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb
index 711621f2..4687a661 100644
--- a/test/integration/helpers/serverspec/xpack_spec.rb
+++ b/test/integration/helpers/serverspec/xpack_spec.rb
@@ -2,5 +2,202 @@
 
 shared_examples 'xpack::init' do |es_version,plugins|
 
+  describe user('elasticsearch') do
+    it { should exist }
+  end
+
+  describe service('security_node_elasticsearch') do
+    it { should be_running }
+  end
+
+  describe package('elasticsearch') do
+    it { should be_installed }
+  end
+
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
+    it { should be_file }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/etc/elasticsearch/security_node/log4j2.properties') do
+    it { should be_file }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
+    it { should contain 'node.name: localhost-security_node' }
+    it { should contain 'cluster.name: elasticsearch' }
+    it { should contain 'path.conf: /etc/elasticsearch/security_node' }
+    it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' }
+    it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' }
+  end
+
+  describe 'Node listening' do
+    it 'listening in port 9200' do
+      expect(port 9200).to be_listening
+    end
+  end
+
+  describe 'version check' do
+    it 'should be reported as version '+es_version do
+      command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
+      expect(command.stdout).to match(es_version)
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  describe file('/etc/init.d/elasticsearch') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/default/elasticsearch') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/sysconfig/elasticsearch') do
+    it { should_not exist }
+  end
+
+  describe file('/usr/lib/systemd/system/elasticsearch.service') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/elasticsearch/elasticsearch.yml') do
+    it { should_not exist }
+  end
+
+  describe file('/etc/elasticsearch/logging.yml') do
+    it { should_not exist }
+  end
+
+  #Xpack specific tests
+  describe file('/usr/share/elasticsearch/plugins') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  #Test if x-pack is activated
+  describe 'x-pack activation' do
+    it 'should be activated and valid' do
+      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain')
+      expect(command.stdout).to match('"status" : "active"')
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  describe file('/usr/share/elasticsearch/plugins/x-pack') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
+    its(:exit_status) { should eq 0 }
+  end
+
+  describe file('/etc/elasticsearch/security_node/x-pack') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/usr/share/elasticsearch/plugins/x-pack') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  for plugin in plugins
+    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
+    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+      its(:exit_status) { should eq 0 }
+    end
+  end
+
+  #Test users file, users_roles and roles.yml
+  describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should contain 'admin:es_admin' }
+    it { should contain 'power_user:testUser' }
+  end
+
+  describe file('/etc/elasticsearch/security_node/x-pack/users') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should contain 'testUser:' }
+    it { should contain 'es_admin:' }
+  end
+
+
+  describe file('/etc/elasticsearch/security_node/x-pack/roles.yml') do
+    it { should be_owned_by 'elasticsearch' }
+    #Test contents as expected
+    its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' }
+  end
+
+
+  #Test native roles and users are loaded
+  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep b6a1293c343e745a508c74778c9be8bb') do
+    its(:exit_status) { should eq 0 }
+  end
+
+  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 44b97844bd8b31d5573493a99ef62106') do
+    its(:exit_status) { should eq 0 }
+  end
+
+  describe file('/etc/elasticsearch/templates') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/etc/elasticsearch/templates/basic.json') do
+    it { should be_file }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe 'Template Installed' do
+    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain')
+      expect(command.stdout).to match(/basic/)
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  #This is possibly subject to format changes in the response across versions so may fail in the future
+  describe 'Template Contents Correct' do
+    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
+      expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
+    end
+  end
+
+  #Test contents of Elasticsearch.yml file
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
+    it { should contain 'security.authc.realms.file1.order: 0' }
+    it { should contain 'security.authc.realms.file1.type: file' }
+    it { should contain 'security.authc.realms.native1.order: 1' }
+    it { should contain 'security.authc.realms.native1.type: native' }
+  end
+
+  #Test contents of role_mapping.yml
+  describe file('/etc/elasticsearch/security_node/x-pack/role_mapping.yml') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should contain 'power_user:' }
+    it { should contain '- cn=admins,dc=example,dc=com' }
+    it { should contain 'user:' }
+    it { should contain '- cn=admins,dc=example,dc=com' }
+  end
+
+
+  describe file('/etc/elasticsearch/security_node/x-pack/system_key') do
+    it { should be_owned_by 'elasticsearch' }
+    it { should be_writable.by('owner') }
+    it { should be_writable.by_user('elasticsearch') }
+    it { should be_readable.by('owner') }
+    it { should be_readable.by_user('elasticsearch') }
+    it { should_not be_executable }
+    #Test contents as expected
+    its(:md5sum) { should eq '6ff0e6c4380a6ac0f6e04d871c0ca5e8' }
+  end
 end
 
diff --git a/test/integration/helpers/serverspec/xpack_standard_spec.rb b/test/integration/helpers/serverspec/xpack_standard_spec.rb
index a882004d..37ce72eb 100644
--- a/test/integration/helpers/serverspec/xpack_standard_spec.rb
+++ b/test/integration/helpers/serverspec/xpack_standard_spec.rb
@@ -40,7 +40,7 @@
 
   describe 'version check' do
     it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
+      command = command('curl -s localhost:9200 | grep number')
       expect(command.stdout).to match(es_version)
       expect(command.exit_status).to eq(0)
     end
@@ -79,7 +79,7 @@
   #Test if x-pack is activated
   describe 'x-pack activation' do
     it 'should be activated and valid' do
-      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain')
+      command = command('curl -s localhost:9200/_license?pretty=true')
       expect(command.stdout).to match('"status" : "active"')
       expect(command.exit_status).to eq(0)
     end
@@ -110,7 +110,7 @@
       it { should be_owned_by 'elasticsearch' }
     end
 
-    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
@@ -118,86 +118,16 @@
   #Test users file, users_roles and roles.yml
   describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do
     it { should be_owned_by 'elasticsearch' }
-    it { should contain 'admin:es_admin' }
-    it { should contain 'power_user:testUser' }
   end
 
   describe file('/etc/elasticsearch/security_node/x-pack/users') do
     it { should be_owned_by 'elasticsearch' }
-    it { should contain 'testUser:' }
-    it { should contain 'es_admin:' }
   end
 
-
-  describe file('/etc/elasticsearch/security_node/x-pack/roles.yml') do
-    it { should be_owned_by 'elasticsearch' }
-    #Test contents as expected
-    its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' }
-  end
-
-
-  #Test native roles and users are loaded
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep b6a1293c343e745a508c74778c9be8bb') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 44b97844bd8b31d5573493a99ef62106') do
+  #This in effect tests the right features are enabled
+  describe command('curl -s localhost:9200/_xpack | md5sum | grep 72349eaf0273b32f3b33a6e9ea193bbe') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe file('/etc/elasticsearch/templates') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/etc/elasticsearch/templates/basic.json') do
-    it { should be_file }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe 'Template Installed' do
-    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain')
-      expect(command.stdout).to match(/basic/)
-      expect(command.exit_status).to eq(0)
-    end
-  end
-
-  #This is possibly subject to format changes in the response across versions so may fail in the future
-  describe 'Template Contents Correct' do
-    it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
-      expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
-    end
-  end
-
-  #Test contents of Elasticsearch.yml file
-  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
-    it { should contain 'security.authc.realms.file1.order: 0' }
-    it { should contain 'security.authc.realms.file1.type: file' }
-    it { should contain 'security.authc.realms.native1.order: 1' }
-    it { should contain 'security.authc.realms.native1.type: native' }
-  end
-
-  #Test contents of role_mapping.yml
-  describe file('/etc/elasticsearch/security_node/x-pack/role_mapping.yml') do
-    it { should be_owned_by 'elasticsearch' }
-    it { should contain 'power_user:' }
-    it { should contain '- cn=admins,dc=example,dc=com' }
-    it { should contain 'user:' }
-    it { should contain '- cn=admins,dc=example,dc=com' }
-  end
-
-
-  describe file('/etc/elasticsearch/security_node/x-pack/system_key') do
-    it { should be_owned_by 'elasticsearch' }
-    it { should be_writable.by('owner') }
-    it { should be_writable.by_user('elasticsearch') }
-    it { should be_readable.by('owner') }
-    it { should be_readable.by_user('elasticsearch') }
-    it { should_not be_executable }
-    #Test contents as expected
-    its(:md5sum) { should eq '6ff0e6c4380a6ac0f6e04d871c0ca5e8' }
-  end
 end
 
diff --git a/test/integration/xpack-standard-5x/serverspec/default_spec.rb b/test/integration/xpack-standard-5x/serverspec/default_spec.rb
index 2ee09c07..f9a75521 100644
--- a/test/integration/xpack-standard-5x/serverspec/default_spec.rb
+++ b/test/integration/xpack-standard-5x/serverspec/default_spec.rb
@@ -1,5 +1,5 @@
 require 'xpack_standard_spec'
 
 describe 'Xpack Standard Tests v 5.x' do
-  include_examples 'xpack_standard::init', "5.5.1", ["ingest-geoip"]
+  include_examples 'xpack_standard::init', "5.5.1", []
 end

From 06cf45e73903d6d1fef8481a2f0b0e109d076251 Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Fri, 18 Aug 2017 23:19:01 +0100
Subject: [PATCH 9/9] Test Fix

---
 .../helpers/serverspec/xpack_standard_spec.rb        | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/integration/helpers/serverspec/xpack_standard_spec.rb b/test/integration/helpers/serverspec/xpack_standard_spec.rb
index 37ce72eb..1dcef520 100644
--- a/test/integration/helpers/serverspec/xpack_standard_spec.rb
+++ b/test/integration/helpers/serverspec/xpack_standard_spec.rb
@@ -30,6 +30,9 @@
     it { should contain 'path.conf: /etc/elasticsearch/security_node' }
     it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' }
+    it { should contain 'xpack.security.enabled: false' }
+    it { should contain 'xpack.watcher.enabled: false' }
+
   end
 
   describe 'Node listening' do
@@ -124,9 +127,12 @@
     it { should be_owned_by 'elasticsearch' }
   end
 
-  #This in effect tests the right features are enabled
-  describe command('curl -s localhost:9200/_xpack | md5sum | grep 72349eaf0273b32f3b33a6e9ea193bbe') do
-    its(:exit_status) { should eq 0 }
+  describe command('curl -s localhost:9200/_xpack') do
+    its(:stdout_as_json) { should include('features' => include('security' => include('enabled' => false))) }
+    its(:stdout_as_json) { should include('features' => include('watcher' => include('enabled' => false))) }
+    its(:stdout_as_json) { should include('features' => include('graph' => include('enabled' => true))) }
+    its(:stdout_as_json) { should include('features' => include('monitoring' => include('enabled' => true))) }
+    its(:stdout_as_json) { should include('features' => include('ml' => include('enabled' => true))) }
   end
 
 end