Merge pull request #412 from elastic/6.x_testing

6.x testing -  Update tests to work with dynamic es_versions

.kitchen.yml

@@ -39,7 +39,9 @@ platforms:
         - pip install jmespath
         - pip uninstall -y ansible
       use_sudo: false
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
   - name: ubuntu-16.04
     driver_config:
       image: dliappis/ubuntu-devopsci:16.04
@@ -51,7 +53,9 @@ platforms:
         - pip install jmespath
         - pip uninstall -y ansible
       use_sudo: false
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
       run_command: "/sbin/init"
   - name: debian-8
     driver_config:
@@ -65,7 +69,9 @@ platforms:
         - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config
         - pip install jmespath
         - pip uninstall -y ansible
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
       use_sudo: false
       run_command: "/sbin/init"
   - name: centos-7
@@ -80,7 +86,9 @@ platforms:
         - yum -y remove ansible
         - yum clean all
         - pip install jmespath
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
       run_command: "/usr/sbin/init"
       privileged: true
       use_sudo: false

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'test-kitchen', '1.17.0'
+gem 'test-kitchen', '1.20.0'
 gem 'kitchen-docker', '2.6.0'
-gem 'kitchen-ansible', '0.47.3'
-gem 'net-ssh', '4.1.0'
+gem 'kitchen-ansible', '0.48.1'
+gem 'net-ssh', '4.2.0'

Gemfile.lock

@@ -1,44 +1,75 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    artifactory (2.8.2)
-    kitchen-ansible (0.47.3)
-      mixlib-shellout (<= 2.2.7)
+    builder (3.2.3)
+    erubis (2.7.0)
+    ffi (1.9.18)
+    gssapi (1.2.0)
+      ffi (>= 1.0.1)
+    gyoku (1.3.1)
+      builder (>= 2.1.2)
+    httpclient (2.8.3)
+    kitchen-ansible (0.48.1)
       net-ssh (>= 3)
       test-kitchen (~> 1.4)
     kitchen-docker (2.6.0)
       test-kitchen (>= 1.0.0)
-    mixlib-install (2.1.12)
-      artifactory
+    little-plugger (1.1.4)
+    logging (2.2.2)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
+    mixlib-install (3.9.0)
       mixlib-shellout
       mixlib-versioning
       thor
-    mixlib-shellout (2.2.7)
+    mixlib-shellout (2.3.2)
     mixlib-versioning (1.2.2)
+    multi_json (1.13.1)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (4.1.0)
+    net-ssh (4.2.0)
     net-ssh-gateway (1.3.0)
       net-ssh (>= 2.6.5)
-    safe_yaml (1.0.4)
-    test-kitchen (1.17.0)
-      mixlib-install (>= 1.2, < 3.0)
+    nori (2.6.0)
+    rubyntlm (0.6.2)
+    rubyzip (1.2.1)
+    test-kitchen (1.20.0)
+      mixlib-install (~> 3.6)
       mixlib-shellout (>= 1.2, < 3.0)
       net-scp (~> 1.1)
       net-ssh (>= 2.9, < 5.0)
       net-ssh-gateway (~> 1.2)
-      safe_yaml (~> 1.0)
       thor (~> 0.19, < 0.19.2)
+      winrm (~> 2.0)
+      winrm-elevated (~> 1.0)
+      winrm-fs (~> 1.1.0)
     thor (0.19.1)
+    winrm (2.2.3)
+      builder (>= 2.1.2)
+      erubis (~> 2.7)
+      gssapi (~> 1.2)
+      gyoku (~> 1.0)
+      httpclient (~> 2.2, >= 2.2.0.2)
+      logging (>= 1.6.1, < 3.0)
+      nori (~> 2.0)
+      rubyntlm (~> 0.6.0, >= 0.6.1)
+    winrm-elevated (1.1.0)
+      winrm (~> 2.0)
+      winrm-fs (~> 1.0)
+    winrm-fs (1.1.1)
+      erubis (~> 2.7)
+      logging (>= 1.6.1, < 3.0)
+      rubyzip (~> 1.1)
+      winrm (~> 2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  kitchen-ansible (= 0.47.3)
+  kitchen-ansible (= 0.48.1)
   kitchen-docker (= 2.6.0)
-  net-ssh (= 4.1.0)
-  test-kitchen (= 1.17.0)
+  net-ssh (= 4.2.0)
+  test-kitchen (= 1.20.0)
 
 BUNDLED WITH
-   1.15.3
+   1.16.1

README.md

@@ -1,15 +1,15 @@
 # ansible-elasticsearch
 [![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/)
 
-**THIS ROLE IS FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
+**THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
 
 Ansible role for 5.x Elasticsearch.  Currently this works on Debian and RedHat based linux systems.  Tested platforms are:
 
 * Ubuntu 14.04/16.04
 * Debian 8
 * Centos 7
 
-The latest Elasticsearch versions of 5.x are actively tested.  **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.**
+The latest Elasticsearch versions of 6.x are actively tested.  **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.**
 
 ##### Dependency
 This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine.
@@ -329,7 +329,7 @@ These can either be set to a user declared in the file based realm, with admin p
 
 In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include:
 
-* ```es_major_version```  Should be consistent with es_version. For versions >= 5.0 this must be "5.x".
+* ```es_major_version```  Should be consistent with es_version. For versions >= 5.0 and < 6.0 this must be "5.x". For versions >= 6.0 this must be "6.x".
 * ```es_version``` (e.g. "5.1.2").  
 * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost".
 * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200**
@@ -396,7 +396,7 @@ To define proxy only for a particular plugin during its installation:
 * The role assumes the user/group exists on the server.  The elasticsearch packages create the default elasticsearch user.  If this needs to be changed, ensure the user exists.
 * The playbook relies on the inventory_name of each host to ensure its directories are unique
 * Changing an instance_name for a role application will result in the installation of a new component.  The previous component will remain.
-* KitchenCI has been used for testing.  This is used to confirm images reach the correct state after a play is first applied.  We currently test only the latest version of 5.x on
+* KitchenCI has been used for testing.  This is used to confirm images reach the correct state after a play is first applied.  We currently test only the latest version of 6.x on
 all supported platforms. 
 * The role aims to be idempotent.  Running the role multiple times, with no changes, should result in no state change on the server.  If the configuration is changed, these will be applied and 
 Elasticsearch restarted where required.
@@ -405,7 +405,7 @@ Elasticsearch restarted where required.
 
 ## IMPORTANT NOTES RE PLUGIN MANAGEMENT
 
-* If the ES version is changed, all plugins will be removed.  Those listed in the playbook will be re-installed.  This is behaviour is required in ES 5.x.
+* If the ES version is changed, all plugins will be removed.  Those listed in the playbook will be re-installed.  This is behaviour is required in ES 6.x.
 * If no plugins are listed in the playbook for a node, all currently installed plugins will be removed.
 * The role supports automatic detection of differences between installed and listed plugins - installing those listed but not installed, and removing those installed but not listed.   Should users wish to re-install plugins they should set es_plugins_reinstall to true.  This will cause all currently installed plugins to be removed and those listed to be installed.
 

defaults/main.yml

@@ -23,7 +23,7 @@ es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
 es_max_open_files: 65536
-es_max_threads: 2048
+es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}"
 es_max_map_count: 262144
 es_allow_downgrades: false
 es_enable_xpack: false

tasks/elasticsearch-plugins.yml

@@ -19,6 +19,7 @@
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 #if es_plugins_reinstall is set to true we remove ALL plugins
@@ -47,6 +48,7 @@
   register: plugin_removed
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 - name: Install elasticsearch plugins
@@ -60,6 +62,7 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
     ES_JAVA_OPTS: "{% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -Dhttp.proxyHost={{ item.proxy_host }} -Dhttp.proxyPort={{ item.proxy_port }} -Dhttps.proxyHost={{ item.proxy_host }} -Dhttps.proxyPort={{ item.proxy_port }}  {% elif es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }} {% endif %}"
   until: plugin_installed.rc == 0

tasks/java.yml

@@ -10,6 +10,20 @@
   yum: name={{ java }} state={{java_state}}
   when: ansible_os_family == 'RedHat'
 
+- name: Get the installed java path
+  shell: "update-alternatives --display java | grep '^/' | awk '{print $1}' | grep 1.8.0"
+  register: java_full_path
+  failed_when: False
+  changed_when: False
+  when: ansible_os_family == 'RedHat'
+
+- name: correct java version selected
+  alternatives:
+    name: java
+    path: "{{ java_full_path.stdout }}"
+    link: /usr/bin/java
+  when: ansible_os_family == 'RedHat' and java_full_path is defined
+
 - name: Refresh java repo
   become: yes
   apt: update_cache=yes

tasks/xpack/elasticsearch-xpack-install.yml

@@ -10,6 +10,7 @@
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 
@@ -24,6 +25,7 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 
@@ -42,6 +44,7 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 - name: Delete x-pack zip file
@@ -59,5 +62,6 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
     ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}"

tasks/xpack/elasticsearch-xpack.yml

@@ -11,3 +11,9 @@
 - name: Set Plugin Directory Permissions
   become: yes
   file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes
+
+#Make sure elasticsearch.keystore has correct Permissions
+- name: Set elasticsearch.keystore Permissions
+  become: yes
+  file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
+  when: es_enable_xpack and "security" in es_xpack_features and (es_version | version_compare('6.0.0', '>'))

tasks/xpack/security/elasticsearch-security-file.yml

@@ -21,6 +21,7 @@
   when: manage_file_users
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
 - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }}
@@ -36,6 +37,7 @@
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
 #Set passwords for all users declared - Required as the useradd will not change existing user passwords
@@ -50,6 +52,7 @@
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
 - set_fact: users_roles={{es_users.file | extract_role_users () }}

tasks/xpack/security/elasticsearch-security.yml

@@ -11,6 +11,25 @@
     - es_enable_xpack and '"security" in es_xpack_features'
     - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
 
+#-----------------------------Create Bootstrap User-----------------------------------
+- name: Check if bootstrap password is set
+  command: >
+   {{es_home}}/bin/elasticsearch-keystore list
+  register: list_keystore
+  changed_when: False
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+  when:
+    - (es_enable_xpack and '"security" in es_xpack_features') and (es_version | version_compare('6.0.0', '>')) 
+
+- name: Create Bootstrap password for elastic user
+  shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password'
+  when:
+    - (es_enable_xpack and '"security" in es_xpack_features') and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines 
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+  no_log: true
+
 #-----------------------------FILE BASED REALM----------------------------------------
 
 - include: elasticsearch-security-file.yml

templates/elasticsearch.j2

@@ -10,6 +10,7 @@ ES_HOME={{es_home}}
 
 # Elasticsearch configuration directory
 CONF_DIR={{conf_dir}}
+ES_PATH_CONF={{conf_dir}}
 
 # Elasticsearch data directory
 DATA_DIR={{ data_dirs | array_to_str }}
@@ -73,4 +74,10 @@ MAX_LOCKED_MEMORY=unlimited
 #MAX_MAP_COUNT=262144
 {% if es_max_map_count is defined %}
 MAX_MAP_COUNT={{es_max_map_count}}
-{% endif %}
+{% endif %}
+
+# Specifies the maximum number of threads that can be started.
+# Elasticsearch requires a minimum of 2048.
+{% if es_max_threads is defined %}
+MAX_THREADS={{ es_max_threads }}
+{% endif %}

templates/elasticsearch.yml.j2

@@ -14,7 +14,10 @@ node.name: {{inventory_hostname}}-{{es_instance_name}}
 #################################### Paths ####################################
 
 # Path to directory containing configuration (this file and logging.yml):
+
+{% if (es_version | version_compare('6.0.0', '<')) %}
 path.conf: {{ conf_dir }}
+{% endif %}
 
 path.data: {{ data_dirs | array_to_str }}