diff --git a/.kitchen.yml b/.kitchen.yml index 0b66d234..7af7ab7c 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -39,7 +39,9 @@ platforms: - pip install jmespath - pip uninstall -y ansible use_sudo: false - volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + volume: + - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc/elasticsearch - name: ubuntu-16.04 driver_config: image: dliappis/ubuntu-devopsci:16.04 @@ -51,7 +53,9 @@ platforms: - pip install jmespath - pip uninstall -y ansible use_sudo: false - volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + volume: + - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc/elasticsearch run_command: "/sbin/init" - name: debian-8 driver_config: @@ -65,7 +69,9 @@ platforms: - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - pip install jmespath - pip uninstall -y ansible - volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + volume: + - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc/elasticsearch use_sudo: false run_command: "/sbin/init" - name: centos-7 @@ -80,7 +86,9 @@ platforms: - yum -y remove ansible - yum clean all - pip install jmespath - volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + volume: + - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json + - /etc/elasticsearch run_command: "/usr/sbin/init" privileged: true use_sudo: false diff --git a/Gemfile b/Gemfile index 19ff3941..5dac49d3 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'test-kitchen', '1.17.0' +gem 'test-kitchen', '1.20.0' gem 'kitchen-docker', '2.6.0' -gem 'kitchen-ansible', '0.47.3' -gem 'net-ssh', '4.1.0' +gem 'kitchen-ansible', '0.48.1' +gem 'net-ssh', '4.2.0' diff --git a/Gemfile.lock b/Gemfile.lock index b12337df..9550977d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,44 +1,75 @@ GEM remote: https://rubygems.org/ specs: - artifactory (2.8.2) - kitchen-ansible (0.47.3) - mixlib-shellout (<= 2.2.7) + builder (3.2.3) + erubis (2.7.0) + ffi (1.9.18) + gssapi (1.2.0) + ffi (>= 1.0.1) + gyoku (1.3.1) + builder (>= 2.1.2) + httpclient (2.8.3) + kitchen-ansible (0.48.1) net-ssh (>= 3) test-kitchen (~> 1.4) kitchen-docker (2.6.0) test-kitchen (>= 1.0.0) - mixlib-install (2.1.12) - artifactory + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + mixlib-install (3.9.0) mixlib-shellout mixlib-versioning thor - mixlib-shellout (2.2.7) + mixlib-shellout (2.3.2) mixlib-versioning (1.2.2) + multi_json (1.13.1) net-scp (1.2.1) net-ssh (>= 2.6.5) - net-ssh (4.1.0) + net-ssh (4.2.0) net-ssh-gateway (1.3.0) net-ssh (>= 2.6.5) - safe_yaml (1.0.4) - test-kitchen (1.17.0) - mixlib-install (>= 1.2, < 3.0) + nori (2.6.0) + rubyntlm (0.6.2) + rubyzip (1.2.1) + test-kitchen (1.20.0) + mixlib-install (~> 3.6) mixlib-shellout (>= 1.2, < 3.0) net-scp (~> 1.1) net-ssh (>= 2.9, < 5.0) net-ssh-gateway (~> 1.2) - safe_yaml (~> 1.0) thor (~> 0.19, < 0.19.2) + winrm (~> 2.0) + winrm-elevated (~> 1.0) + winrm-fs (~> 1.1.0) thor (0.19.1) + winrm (2.2.3) + builder (>= 2.1.2) + erubis (~> 2.7) + gssapi (~> 1.2) + gyoku (~> 1.0) + httpclient (~> 2.2, >= 2.2.0.2) + logging (>= 1.6.1, < 3.0) + nori (~> 2.0) + rubyntlm (~> 0.6.0, >= 0.6.1) + winrm-elevated (1.1.0) + winrm (~> 2.0) + winrm-fs (~> 1.0) + winrm-fs (1.1.1) + erubis (~> 2.7) + logging (>= 1.6.1, < 3.0) + rubyzip (~> 1.1) + winrm (~> 2.0) PLATFORMS ruby DEPENDENCIES - kitchen-ansible (= 0.47.3) + kitchen-ansible (= 0.48.1) kitchen-docker (= 2.6.0) - net-ssh (= 4.1.0) - test-kitchen (= 1.17.0) + net-ssh (= 4.2.0) + test-kitchen (= 1.20.0) BUNDLED WITH - 1.15.3 + 1.16.1 diff --git a/README.md b/README.md index 6692ff15..52c9bc64 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # ansible-elasticsearch [![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/) -**THIS ROLE IS FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** +**THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** Ansible role for 5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: @@ -9,7 +9,7 @@ Ansible role for 5.x Elasticsearch. Currently this works on Debian and RedHat b * Debian 8 * Centos 7 -The latest Elasticsearch versions of 5.x are actively tested. **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.** +The latest Elasticsearch versions of 6.x are actively tested. **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.** ##### Dependency This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. @@ -329,7 +329,7 @@ These can either be set to a user declared in the file based realm, with admin p In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include: -* ```es_major_version``` Should be consistent with es_version. For versions >= 5.0 this must be "5.x". +* ```es_major_version``` Should be consistent with es_version. For versions >= 5.0 and < 6.0 this must be "5.x". For versions >= 6.0 this must be "6.x". * ```es_version``` (e.g. "5.1.2"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** @@ -396,7 +396,7 @@ To define proxy only for a particular plugin during its installation: * The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. * The playbook relies on the inventory_name of each host to ensure its directories are unique * Changing an instance_name for a role application will result in the installation of a new component. The previous component will remain. -* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of 5.x on +* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of 6.x on all supported platforms. * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. @@ -405,7 +405,7 @@ Elasticsearch restarted where required. ## IMPORTANT NOTES RE PLUGIN MANAGEMENT -* If the ES version is changed, all plugins will be removed. Those listed in the playbook will be re-installed. This is behaviour is required in ES 5.x. +* If the ES version is changed, all plugins will be removed. Those listed in the playbook will be re-installed. This is behaviour is required in ES 6.x. * If no plugins are listed in the playbook for a node, all currently installed plugins will be removed. * The role supports automatic detection of differences between installed and listed plugins - installing those listed but not installed, and removing those installed but not listed. Should users wish to re-install plugins they should set es_plugins_reinstall to true. This will cause all currently installed plugins to be removed and those listed to be installed. diff --git a/defaults/main.yml b/defaults/main.yml index f7071dab..bb56093c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -23,7 +23,7 @@ es_pid_dir: "/var/run/elasticsearch" es_data_dirs: "/var/lib/elasticsearch" es_log_dir: "/var/log/elasticsearch" es_max_open_files: 65536 -es_max_threads: 2048 +es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}" es_max_map_count: 262144 es_allow_downgrades: false es_enable_xpack: false diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 104223e1..01c275e7 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -19,6 +19,7 @@ ignore_errors: yes environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" #if es_plugins_reinstall is set to true we remove ALL plugins @@ -47,6 +48,7 @@ register: plugin_removed environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" - name: Install elasticsearch plugins @@ -60,6 +62,7 @@ notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" ES_JAVA_OPTS: "{% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -Dhttp.proxyHost={{ item.proxy_host }} -Dhttp.proxyPort={{ item.proxy_port }} -Dhttps.proxyHost={{ item.proxy_host }} -Dhttps.proxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }} {% endif %}" until: plugin_installed.rc == 0 diff --git a/tasks/java.yml b/tasks/java.yml index a7d3e431..a95b7bdc 100644 --- a/tasks/java.yml +++ b/tasks/java.yml @@ -10,6 +10,20 @@ yum: name={{ java }} state={{java_state}} when: ansible_os_family == 'RedHat' +- name: Get the installed java path + shell: "update-alternatives --display java | grep '^/' | awk '{print $1}' | grep 1.8.0" + register: java_full_path + failed_when: False + changed_when: False + when: ansible_os_family == 'RedHat' + +- name: correct java version selected + alternatives: + name: java + path: "{{ java_full_path.stdout }}" + link: /usr/bin/java + when: ansible_os_family == 'RedHat' and java_full_path is defined + - name: Refresh java repo become: yes apt: update_cache=yes diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index 596539aa..eba38444 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -10,6 +10,7 @@ ignore_errors: yes environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" @@ -24,6 +25,7 @@ notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" @@ -42,6 +44,7 @@ notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" - name: Delete x-pack zip file @@ -59,5 +62,6 @@ notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}" diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index b629943e..8478a6bc 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -11,3 +11,9 @@ - name: Set Plugin Directory Permissions become: yes file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes + +#Make sure elasticsearch.keystore has correct Permissions +- name: Set elasticsearch.keystore Permissions + become: yes + file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }} + when: es_enable_xpack and "security" in es_xpack_features and (es_version | version_compare('6.0.0', '>')) diff --git a/tasks/xpack/security/elasticsearch-security-file.yml b/tasks/xpack/security/elasticsearch-security-file.yml index 885cd036..3bc96680 100644 --- a/tasks/xpack/security/elasticsearch-security-file.yml +++ b/tasks/xpack/security/elasticsearch-security-file.yml @@ -21,6 +21,7 @@ when: manage_file_users environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_HOME: "{{es_home}}" - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }} @@ -36,6 +37,7 @@ no_log: True environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_HOME: "{{es_home}}" #Set passwords for all users declared - Required as the useradd will not change existing user passwords @@ -50,6 +52,7 @@ no_log: True environment: CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" ES_HOME: "{{es_home}}" - set_fact: users_roles={{es_users.file | extract_role_users () }} diff --git a/tasks/xpack/security/elasticsearch-security.yml b/tasks/xpack/security/elasticsearch-security.yml index c75f59bf..1d7d93b0 100644 --- a/tasks/xpack/security/elasticsearch-security.yml +++ b/tasks/xpack/security/elasticsearch-security.yml @@ -11,6 +11,25 @@ - es_enable_xpack and '"security" in es_xpack_features' - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined) +#-----------------------------Create Bootstrap User----------------------------------- +- name: Check if bootstrap password is set + command: > + {{es_home}}/bin/elasticsearch-keystore list + register: list_keystore + changed_when: False + environment: + ES_PATH_CONF: "{{ conf_dir }}" + when: + - (es_enable_xpack and '"security" in es_xpack_features') and (es_version | version_compare('6.0.0', '>')) + +- name: Create Bootstrap password for elastic user + shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password' + when: + - (es_enable_xpack and '"security" in es_xpack_features') and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines + environment: + ES_PATH_CONF: "{{ conf_dir }}" + no_log: true + #-----------------------------FILE BASED REALM---------------------------------------- - include: elasticsearch-security-file.yml diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index cb2341a2..5bf5746f 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -10,6 +10,7 @@ ES_HOME={{es_home}} # Elasticsearch configuration directory CONF_DIR={{conf_dir}} +ES_PATH_CONF={{conf_dir}} # Elasticsearch data directory DATA_DIR={{ data_dirs | array_to_str }} @@ -73,4 +74,10 @@ MAX_LOCKED_MEMORY=unlimited #MAX_MAP_COUNT=262144 {% if es_max_map_count is defined %} MAX_MAP_COUNT={{es_max_map_count}} -{% endif %} \ No newline at end of file +{% endif %} + +# Specifies the maximum number of threads that can be started. +# Elasticsearch requires a minimum of 2048. +{% if es_max_threads is defined %} +MAX_THREADS={{ es_max_threads }} +{% endif %} diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index a0c8f842..f50c34d3 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -14,7 +14,10 @@ node.name: {{inventory_hostname}}-{{es_instance_name}} #################################### Paths #################################### # Path to directory containing configuration (this file and logging.yml): + +{% if (es_version | version_compare('6.0.0', '<')) %} path.conf: {{ conf_dir }} +{% endif %} path.data: {{ data_dirs | array_to_str }} diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index 64c2f0af..3aeb9925 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -60,6 +60,7 @@ DATA_DIR={{ data_dirs | array_to_str }} # Elasticsearch configuration directory CONF_DIR={{conf_dir}} +ES_PATH_CONF={{ conf_dir }} # Maximum number of VMA (Virtual Memory Areas) a process can own {% if es_max_map_count is defined %} @@ -91,12 +92,17 @@ fi # Define other required variables PID_FILE="$PID_DIR/$NAME.pid" DAEMON=$ES_HOME/bin/elasticsearch +{% if (es_version | version_compare('6.0.0', '<')) %} DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" +{% else %} +DAEMON_OPTS="-d -p $PID_FILE" +{% endif %} export ES_JAVA_OPTS export JAVA_HOME export ES_INCLUDE export ES_JVM_OPTIONS +export ES_PATH_CONF # export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi @@ -156,6 +162,10 @@ case "$1" in ulimit -l $MAX_LOCKED_MEMORY fi + if [ -n "$MAX_THREADS" ]; then + ulimit -u $MAX_THREADS + fi + if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT fi diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index e093a857..c993e143 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -46,6 +46,7 @@ MAX_MAP_COUNT={{es_max_map_count}} LOG_DIR="{{log_dir}}" DATA_DIR={{ data_dirs | array_to_str }} CONF_DIR="{{conf_dir}}" +ES_PATH_CONF="{{ conf_dir }}" PID_DIR="{{pid_dir}}" @@ -74,6 +75,7 @@ export JAVA_HOME export ES_INCLUDE export ES_JVM_OPTIONS export ES_STARTUP_SLEEP_TIME +export ES_PATH_CONF # export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi @@ -120,6 +122,9 @@ start() { if [ -n "$MAX_LOCKED_MEMORY" ]; then ulimit -l $MAX_LOCKED_MEMORY fi + if [ -n "$MAX_THREADS" ]; then + ulimit -u $MAX_THREADS + fi if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT fi @@ -135,7 +140,11 @@ start() { cd $ES_HOME echo -n $"Starting $prog: " # if not running, start it up here, usually something like "daemon $exec" +{% if (es_version | version_compare('6.0.0', '<')) %} daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR +{% else %} + daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d +{% endif %} retval=$? echo [ $retval -eq 0 ] && touch $lockfile diff --git a/templates/log4j2.properties.j2 b/templates/log4j2.properties.j2 index 3702afff..269be529 100644 --- a/templates/log4j2.properties.j2 +++ b/templates/log4j2.properties.j2 @@ -11,25 +11,52 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n appender.rolling.type = RollingFile appender.rolling.name = rolling +{% if (es_version | version_compare('6.0.0', '<')) %} appender.rolling.fileName = ${sys:es.logs}.log +{% else %} +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +{% endif %} appender.rolling.layout.type = PatternLayout appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log +{% else %} +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +{% endif %} appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy appender.rolling.policies.time.interval = 1 appender.rolling.policies.time.modulate = true - +{% if (es_version | version_compare('6.0.0', '>')) %} +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 128MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB +{% endif %} rootLogger.level = info rootLogger.appenderRef.console.ref = console rootLogger.appenderRef.rolling.ref = rolling appender.deprecation_rolling.type = RollingFile appender.deprecation_rolling.name = deprecation_rolling +{% if (es_version | version_compare('6.0.0', '<')) %} appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log +{% else %} +appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log +{% endif %} appender.deprecation_rolling.layout.type = PatternLayout appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz +{% else %} +appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz +{% endif %} appender.deprecation_rolling.policies.type = Policies appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy appender.deprecation_rolling.policies.size.size = 1GB @@ -43,10 +70,18 @@ logger.deprecation.additivity = false appender.index_search_slowlog_rolling.type = RollingFile appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +{% if (es_version | version_compare('6.0.0', '<')) %} appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log +{% else %} +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log +{% endif %} appender.index_search_slowlog_rolling.layout.type = PatternLayout appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log +{% else %} +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log +{% endif %} appender.index_search_slowlog_rolling.policies.type = Policies appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy appender.index_search_slowlog_rolling.policies.time.interval = 1 @@ -59,10 +94,18 @@ logger.index_search_slowlog_rolling.additivity = false appender.index_indexing_slowlog_rolling.type = RollingFile appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +{% if (es_version | version_compare('6.0.0', '<')) %} appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log +{% else %} +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log +{% endif %} appender.index_indexing_slowlog_rolling.layout.type = PatternLayout appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +{% else %} +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +{% endif %} appender.index_indexing_slowlog_rolling.policies.type = Policies appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy appender.index_indexing_slowlog_rolling.policies.time.interval = 1 diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index a9b119fd..8bd5545c 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -7,6 +7,7 @@ After=network-online.target [Service] Environment=ES_HOME={{es_home}} Environment=CONF_DIR={{conf_dir}} +Environment=ES_PATH_CONF={{conf_dir}} Environment=DATA_DIR={{ data_dirs | array_to_str }} Environment=LOG_DIR={{log_dir}} Environment=PID_DIR={{pid_dir}} @@ -17,14 +18,18 @@ WorkingDirectory={{es_home}} User={{es_user}} Group={{es_group}} +{% if (es_version | version_compare('6.0.0', '<')) %} ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec +{% endif %} ExecStart={{es_home}}/bin/elasticsearch \ -p ${PID_DIR}/elasticsearch.pid \ - --quiet \ +{% if (es_version | version_compare('6.0.0', '<')) %} -Edefault.path.logs=${LOG_DIR} \ -Edefault.path.data=${DATA_DIR} \ - -Edefault.path.conf=${CONF_DIR} + -Edefault.path.conf=${CONF_DIR} \ +{% endif %} + --quiet # StandardOutput is configured to redirect to journalctl since @@ -41,9 +46,6 @@ StandardError=inherit LimitNOFILE={{es_max_open_files}} {% endif %} -# Specifies the maximum number of processes -LimitNPROC=2048 - # Specifies the maximum number of bytes of memory that may be locked into RAM # Set to "infinity" if you use the 'bootstrap.memory_lock: true' option # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} diff --git a/test/integration/config.yml b/test/integration/config.yml index e8b623d4..06bc035f 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -2,6 +2,8 @@ #Test explicit setting of parameters and variables - name: Elasticsearch Config initial hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: #expand to all available parameters - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: false } } @@ -17,6 +19,8 @@ #Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed. - name: Elasticsearch Config test modify hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: #expand to all available parameters - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: true } } @@ -31,4 +35,4 @@ es_max_threads: 3000 es_plugins: - plugin: ingest-attachment - - plugin: ingest-user-agent \ No newline at end of file + - plugin: ingest-user-agent diff --git a/test/integration/config/serverspec/default_spec.rb b/test/integration/config/serverspec/default_spec.rb index ade4b546..b76a5098 100644 --- a/test/integration/config/serverspec/default_spec.rb +++ b/test/integration/config/serverspec/default_spec.rb @@ -1,6 +1,8 @@ require 'config_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) -describe 'Config Tests v 5.x' do - include_examples 'config::init', "5.5.1", ["ingest-attachment","ingest-user-agent"] +describe 'Config Tests' do + include_examples 'config::init', vars end diff --git a/test/integration/debug.yml b/test/integration/debug.yml new file mode 100644 index 00000000..81b08b11 --- /dev/null +++ b/test/integration/debug.yml @@ -0,0 +1,6 @@ +--- +- name: Dump all variables to a file + changed_when: False + copy: + content: '{{ vars | to_nice_json }} ' + dest: '/tmp/vars.json' diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb index 9631a4cd..148ade01 100644 --- a/test/integration/helpers/serverspec/config_spec.rb +++ b/test/integration/helpers/serverspec/config_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -shared_examples 'config::init' do |es_version,plugins| +shared_examples 'config::init' do |vars| describe user('elasticsearch') do it { should exist } @@ -36,7 +36,11 @@ it { should contain 'node.name: node1' } it { should contain 'bootstrap.memory_lock: true' } it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' } - it { should contain 'path.conf: /etc/elasticsearch/node1' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/node1' } + else + it { should contain 'path.conf: /etc/elasticsearch/node1' } + end it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' } end @@ -77,20 +81,21 @@ describe 'version check' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9401 | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end - for plugin in plugins + for plugin in vars['es_plugins'] + plugin = plugin['plugin'] describe file('/usr/share/elasticsearch/plugins/'+plugin) do it { should be_directory } it { should be_owned_by 'elasticsearch' } end #confirm plugins are installed and the correct version - describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do + describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do its(:exit_status) { should eq 0 } end end @@ -100,7 +105,7 @@ it { should_not exist } end #confirm plugins are installed and the correct version - describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+es_version+'"\'') do + describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+vars['es_version']+'"\'') do its(:exit_status) { should eq 1 } end diff --git a/test/integration/helpers/serverspec/issue_test_spec.rb b/test/integration/helpers/serverspec/issue_test_spec.rb index fe18a6b0..f65ed61d 100644 --- a/test/integration/helpers/serverspec/issue_test_spec.rb +++ b/test/integration/helpers/serverspec/issue_test_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) -shared_examples 'issue_test::init' do |es_version,plugins| +shared_examples 'issue_test::init' do |vars| #Add custom tests here for the issue-test.yml test diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 003ffe6c..614284cb 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) -shared_examples 'multi::init' do |es_version,plugins| +shared_examples 'multi::init' do |vars| describe user('elasticsearch') do it { should exist } @@ -27,7 +29,11 @@ it { should contain 'node.master: false' } it { should contain 'node.name: localhost-node1' } it { should_not contain 'bootstrap.memory_lock: true' } - it { should contain 'path.conf: /etc/elasticsearch/node1' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/node1' } + else + it { should contain 'path.conf: /etc/elasticsearch/node1' } + end it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } end @@ -42,7 +48,11 @@ it { should contain 'node.master: true' } it { should contain 'node.name: localhost-master' } it { should contain 'bootstrap.memory_lock: true' } - it { should contain 'path.conf: /etc/elasticsearch/master' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/master' } + else + it { should contain 'path.conf: /etc/elasticsearch/master' } + end it { should contain 'path.data: /opt/elasticsearch/master/localhost-master' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-master' } end @@ -154,22 +164,23 @@ end describe 'version check on master' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end describe 'version check on data' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9201 | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end - for plugin in plugins + for plugin in vars['es_plugins'] + plugin = plugin['plugin'] describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do its(:exit_status) { should eq 0 } diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb index ac68bc28..947ce875 100644 --- a/test/integration/helpers/serverspec/package_spec.rb +++ b/test/integration/helpers/serverspec/package_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) -shared_examples 'package::init' do |es_version,plugins| +shared_examples 'package::init' do |vars| describe user('elasticsearch') do it { should exist } @@ -56,9 +58,9 @@ end describe 'version check' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end @@ -69,13 +71,14 @@ end - for plugin in plugins + for plugin in vars['es_plugins'] + plugin = plugin['plugin'] describe file('/usr/share/elasticsearch/plugins/'+plugin) do it { should be_directory } it { should be_owned_by 'elasticsearch' } end #confirm plugins are installed and the correct version - describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do + describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do its(:exit_status) { should eq 0 } end end diff --git a/test/integration/helpers/serverspec/spec_helper.rb b/test/integration/helpers/serverspec/spec_helper.rb index 1a1bfb52..20ca46b3 100644 --- a/test/integration/helpers/serverspec/spec_helper.rb +++ b/test/integration/helpers/serverspec/spec_helper.rb @@ -1,4 +1,7 @@ require 'serverspec' +require 'net/http' +require 'json' + set :backend, :exec require 'rspec/retry' @@ -8,4 +11,16 @@ config.verbose_retry = true # show exception that triggers a retry if verbose_retry is set to true config.display_try_failure_messages = true -end \ No newline at end of file +end + +def curl_json(uri, username=nil, password=nil) + uri = URI(uri) + req = Net::HTTP::Get.new(uri) + if username && password + req.basic_auth username, password + end + res = Net::HTTP.start(uri.hostname, uri.port) {|http| + http.request(req) + } + return JSON.parse(res.body) +end diff --git a/test/integration/helpers/serverspec/standard_spec.rb b/test/integration/helpers/serverspec/standard_spec.rb index d5cb6601..59b9f2a0 100644 --- a/test/integration/helpers/serverspec/standard_spec.rb +++ b/test/integration/helpers/serverspec/standard_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -shared_examples 'standard::init' do |es_version,plugins| +shared_examples 'standard::init' do |vars| describe user('elasticsearch') do it { should exist } @@ -33,7 +33,11 @@ describe file('/etc/elasticsearch/node1/elasticsearch.yml') do it { should contain 'node.name: localhost-node1' } it { should contain 'cluster.name: elasticsearch' } - it { should contain 'path.conf: /etc/elasticsearch/node1' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/node1' } + else + it { should contain 'path.conf: /etc/elasticsearch/node1' } + end it { should contain 'path.data: /var/lib/elasticsearch/localhost-node1' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } end @@ -45,9 +49,9 @@ end describe 'version check' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end @@ -76,13 +80,15 @@ it { should_not exist } end - for plugin in plugins + for plugin in vars['es_plugins'] + plugin = plugin['plugin'] + describe file('/usr/share/elasticsearch/plugins/'+plugin) do it { should be_directory } it { should be_owned_by 'elasticsearch' } end #confirm plugins are installed and the correct version - describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do + describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do its(:exit_status) { should eq 0 } end end diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 54df75aa..f55d4e71 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) -shared_examples 'xpack::init' do |es_version,plugins| +shared_examples 'xpack::init' do |vars| describe user('elasticsearch') do it { should exist } @@ -27,7 +29,11 @@ describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do it { should contain 'node.name: localhost-security_node' } it { should contain 'cluster.name: elasticsearch' } - it { should contain 'path.conf: /etc/elasticsearch/security_node' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/security_node' } + else + it { should contain 'path.conf: /etc/elasticsearch/security_node' } + end it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } end @@ -39,9 +45,9 @@ end describe 'version check' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end @@ -104,13 +110,15 @@ it { should be_owned_by 'elasticsearch' } end - for plugin in plugins + for plugin in vars['es_plugins'] + plugin = plugin['plugin'] + describe file('/usr/share/elasticsearch/plugins/'+plugin) do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do + describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do its(:exit_status) { should eq 0 } end end @@ -141,8 +149,11 @@ its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 44b97844bd8b31d5573493a99ef62106') do - its(:exit_status) { should eq 0 } + describe 'security roles' do + it 'should list the security roles' do + roles = curl_json('http://localhost:9200/_xpack/security/role', username='es_admin', password='changeMeAgain') + expect(roles.key?('superuser')) + end end describe file('/etc/elasticsearch/templates') do @@ -166,8 +177,10 @@ #This is possibly subject to format changes in the response across versions so may fail in the future describe 'Template Contents Correct' do it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum') - expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/) + template = curl_json('http://localhost:9200/_template/basic', username='es_admin', password='changeMeAgain') + expect(template.key?('basic')) + expect(template['basic']['settings']['index']['number_of_shards']).to eq("1") + expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false) end end @@ -203,9 +216,9 @@ #check accounts are correct i.e. we can auth and they have the correct roles describe 'kibana4_server access check' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end @@ -216,9 +229,9 @@ describe 'logstash_system access check' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200/ -u logstash_system:aNewLogstashPassword | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end @@ -227,11 +240,12 @@ its(:exit_status) { should eq 0 } end - describe 'kibana access check' do - it 'should be reported as version '+es_version do - command = command('curl -s localhost:9200/ -u kibana:changeme | grep number') - expect(command.stdout).to match(es_version) - expect(command.exit_status).to eq(0) + if vars['es_major_version'] == '5.x' # kibana default password has been removed in 6.x + describe 'kibana access check' do + it 'should be reported as version '+vars['es_version'] do + result = curl_json('http://localhost:9200/', username='kibana', password='changeme') + expect(result['version']['number']).to eq(vars['es_version']) + end end end diff --git a/test/integration/helpers/serverspec/xpack_standard_spec.rb b/test/integration/helpers/serverspec/xpack_standard_spec.rb index 1dcef520..e19b2848 100644 --- a/test/integration/helpers/serverspec/xpack_standard_spec.rb +++ b/test/integration/helpers/serverspec/xpack_standard_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -shared_examples 'xpack_standard::init' do |es_version,plugins| +shared_examples 'xpack_standard::init' do |vars| describe user('elasticsearch') do it { should exist } @@ -27,7 +27,11 @@ describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do it { should contain 'node.name: localhost-security_node' } it { should contain 'cluster.name: elasticsearch' } - it { should contain 'path.conf: /etc/elasticsearch/security_node' } + if vars['es_major_version'] == '6.x' + it { should_not contain 'path.conf: /etc/elasticsearch/security_node' } + else + it { should contain 'path.conf: /etc/elasticsearch/security_node' } + end it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } it { should contain 'xpack.security.enabled: false' } @@ -42,9 +46,9 @@ end describe 'version check' do - it 'should be reported as version '+es_version do + it 'should be reported as version '+vars['es_version'] do command = command('curl -s localhost:9200 | grep number') - expect(command.stdout).to match(es_version) + expect(command.stdout).to match(vars['es_version']) expect(command.exit_status).to eq(0) end end @@ -107,15 +111,13 @@ it { should be_owned_by 'elasticsearch' } end - for plugin in plugins - describe file('/usr/share/elasticsearch/plugins/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end + describe file('/usr/share/elasticsearch/plugins/x-pack') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end - describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do - its(:exit_status) { should eq 0 } - end + describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"x-pack","version":"'+vars['es_version']+'"\'') do + its(:exit_status) { should eq 0 } end #Test users file, users_roles and roles.yml diff --git a/test/integration/issue-test.yml b/test/integration/issue-test.yml index 1f7ef2a8..881ba850 100644 --- a/test/integration/issue-test.yml +++ b/test/integration/issue-test.yml @@ -5,6 +5,8 @@ - name: Simple Example hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, @@ -46,4 +48,4 @@ - manage_index_templates logstash_system: cluster: - - manage_index_templates \ No newline at end of file + - manage_index_templates diff --git a/test/integration/issue-test/serverspec/default_spec.rb b/test/integration/issue-test/serverspec/default_spec.rb index b284d061..ee7536b0 100644 --- a/test/integration/issue-test/serverspec/default_spec.rb +++ b/test/integration/issue-test/serverspec/default_spec.rb @@ -1,6 +1,8 @@ require 'issue_test_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) describe 'Issue Test' do - include_examples 'issue_test::init', "5.5.1", [] + include_examples 'issue_test::init', vars end diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 702b11f0..fecbd540 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -2,6 +2,8 @@ #Test ability to deploy multiple instances to a machine - name: Elasticsearch Multi test - master on 9200 hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml vars: es_scripts: true es_templates: true @@ -14,6 +16,8 @@ - name: Elasticsearch Multi test - data on 9201 hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml vars: es_scripts: true es_templates: true diff --git a/test/integration/multi/serverspec/default_spec.rb b/test/integration/multi/serverspec/default_spec.rb index fcf6ee5a..94bd05f5 100644 --- a/test/integration/multi/serverspec/default_spec.rb +++ b/test/integration/multi/serverspec/default_spec.rb @@ -1,8 +1,9 @@ require 'multi_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Multi Tests v 5.x' do - include_examples 'multi::init', "5.5.1", ["ingest-geoip"] +describe 'Multi Tests' do + include_examples 'multi::init', vars end diff --git a/test/integration/package.yml b/test/integration/package.yml index 77168c4e..e789ce5c 100644 --- a/test/integration/package.yml +++ b/test/integration/package.yml @@ -1,6 +1,8 @@ --- - name: Elasticsearch Package test intial hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } vars: @@ -8,7 +10,6 @@ es_templates: true es_heap_size: "1g" es_api_port: 9200 - es_version: "5.5.1" es_plugins: - plugin: ingest-geoip @@ -16,12 +17,13 @@ #Tests the plugins have been correctly removed and ES can be upgraded between minor versions. All plugins will be removed and re-installed. - name: Elasticsearch Package test modify hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } vars: es_scripts: true es_templates: true - es_version: "5.5.1" es_heap_size: "1g" es_api_port: 9200 es_plugins: diff --git a/test/integration/package/serverspec/default_spec.rb b/test/integration/package/serverspec/default_spec.rb index 225541a7..ee8f77bd 100644 --- a/test/integration/package/serverspec/default_spec.rb +++ b/test/integration/package/serverspec/default_spec.rb @@ -1,6 +1,7 @@ require 'package_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Package Tests v 5.x' do - include_examples 'package::init', "5.5.1", ["ingest-attachment","ingest-geoip"] -end \ No newline at end of file +describe 'Package Tests' do + include_examples 'package::init', vars +end diff --git a/test/integration/standard.yml b/test/integration/standard.yml index 7d8e6cc7..769f2c5f 100644 --- a/test/integration/standard.yml +++ b/test/integration/standard.yml @@ -1,6 +1,8 @@ --- - name: Standard test for single node setup. Tests idempotence. hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_instance_name: "node1" } vars: @@ -9,4 +11,4 @@ es_plugins: - plugin: ingest-geoip -#Do not add tests here. This test is run twice and confirms idempotency. \ No newline at end of file +#Do not add tests here. This test is run twice and confirms idempotency. diff --git a/test/integration/standard/serverspec/default_spec.rb b/test/integration/standard/serverspec/default_spec.rb index 729b3060..449a1c02 100644 --- a/test/integration/standard/serverspec/default_spec.rb +++ b/test/integration/standard/serverspec/default_spec.rb @@ -1,8 +1,9 @@ require 'standard_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) - -describe 'Standard Tests v 5.x' do - include_examples 'standard::init', "5.5.1", ["ingest-geoip"] +describe 'Standard Tests' do + include_examples 'standard::init', vars end diff --git a/test/integration/xpack-standard.yml b/test/integration/xpack-standard.yml index b9e7fe9d..e3eba44b 100644 --- a/test/integration/xpack-standard.yml +++ b/test/integration/xpack-standard.yml @@ -2,14 +2,15 @@ --- - name: Elasticsearch Xpack tests - no security and manual download hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" } vars: - es_version: "5.5.1" es_heap_size: 2g es_enable_xpack: true - es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip" + es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" es_xpack_features: - monitoring - graph - - ml \ No newline at end of file + - ml diff --git a/test/integration/xpack-standard/serverspec/default_spec.rb b/test/integration/xpack-standard/serverspec/default_spec.rb index f9a75521..771c7c3a 100644 --- a/test/integration/xpack-standard/serverspec/default_spec.rb +++ b/test/integration/xpack-standard/serverspec/default_spec.rb @@ -1,5 +1,7 @@ require 'xpack_standard_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) -describe 'Xpack Standard Tests v 5.x' do - include_examples 'xpack_standard::init', "5.5.1", [] +describe 'Xpack Standard Tests' do + include_examples 'xpack_standard::init', vars end diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 3e53349a..a386732c 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -1,6 +1,8 @@ --- - name: Elasticsearch Xpack tests initial hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, @@ -8,7 +10,7 @@ vars: es_heap_size: "1g" es_templates: true - es_version: "5.4.0" + es_version: "{{ '6.1.1' if es_major_version == '6.x' else '5.4.0'}}" # This is set to an older version than the current default to force an upgrade es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: @@ -101,6 +103,8 @@ #modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. - name: Elasticsearch Xpack modify hosts: localhost + tasks: + - include: elasticsearch/test/integration/debug.yml roles: - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, @@ -108,7 +112,6 @@ vars: es_heap_size: "1g" es_templates: true - es_version: "5.5.1" es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: @@ -143,4 +146,4 @@ password: changeMeAlso! roles: - power_user - - user \ No newline at end of file + - user diff --git a/test/integration/xpack/serverspec/default_spec.rb b/test/integration/xpack/serverspec/default_spec.rb index ab0946bb..8a3791a4 100644 --- a/test/integration/xpack/serverspec/default_spec.rb +++ b/test/integration/xpack/serverspec/default_spec.rb @@ -1,5 +1,7 @@ require 'xpack_spec' +require 'json' +vars = JSON.parse(File.read('/tmp/vars.json')) -describe 'Xpack Tests v 5.x' do - include_examples 'xpack::init', "5.5.1", ["ingest-attachment"] +describe 'Xpack Tests' do + include_examples 'xpack::init', vars end