From 853405d61139aabac40818bb9a83d8493a35af7f Mon Sep 17 00:00:00 2001 From: Romain Pelisse Date: Tue, 3 Oct 2023 10:36:35 +0200 Subject: [PATCH 1/3] Bump Kafka to latest version --- roles/amq_streams_common/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/amq_streams_common/defaults/main.yml b/roles/amq_streams_common/defaults/main.yml index 7326268..192f7f3 100644 --- a/roles/amq_streams_common/defaults/main.yml +++ b/roles/amq_streams_common/defaults/main.yml @@ -1,5 +1,5 @@ --- -amq_streams_common_product_version: 3.4.0 +amq_streams_common_product_version: 3.5.0 amq_streams_common_scala_version: 2.13 amq_streams_common_version: "{{ amq_streams_common_scala_version }}-{{ amq_streams_common_product_version }}" amq_streams_common_archive_file: "kafka_{{ amq_streams_common_version }}.tgz" From 300b0bcc977b4065e6bd7818864b58cb93183cc7 Mon Sep 17 00:00:00 2001 From: Romain Pelisse Date: Tue, 3 Oct 2023 12:52:07 +0200 Subject: [PATCH 2/3] misc. fix ansible-lint error --- molecule/ssl_auth_sasl/prepare.yml | 2 +- molecule/ssl_no_auth/prepare.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/molecule/ssl_auth_sasl/prepare.yml b/molecule/ssl_auth_sasl/prepare.yml index 882ba94..fcbf82d 100644 --- a/molecule/ssl_auth_sasl/prepare.yml +++ b/molecule/ssl_auth_sasl/prepare.yml @@ -19,7 +19,7 @@ dest: /tmp/ owner: root group: root - mode: 0077 + mode: "0077" - name: "Generate required SSL artifacts." ansible.builtin.command: "/tmp/generate_keys_and_certs.sh" diff --git a/molecule/ssl_no_auth/prepare.yml b/molecule/ssl_no_auth/prepare.yml index ed33509..8641cda 100644 --- a/molecule/ssl_no_auth/prepare.yml +++ b/molecule/ssl_no_auth/prepare.yml @@ -19,7 +19,7 @@ dest: /tmp/ owner: root group: root - mode: 0077 + mode: "0077" - name: "Generate required SSL artefacts." ansible.builtin.command: "/tmp/generate_keys_and_certs.sh" From 4f981e0ef80fda8652b1c69f1f3c5f4efb5beadf Mon Sep 17 00:00:00 2001 From: Romain Pelisse Date: Tue, 3 Oct 2023 13:06:51 +0200 Subject: [PATCH 3/3] molecule cleanup (reduce code dupl) --- molecule/all_auth/molecule.yml | 2 +- molecule/all_cluster/molecule.yml | 2 +- molecule/all_cluster/prepare.yml | 10 ---- molecule/connect/molecule.yml | 2 +- molecule/connect/prepare.yml | 10 ---- molecule/default/molecule.yml | 2 +- molecule/default/prepare.yml | 10 ---- .../generate_keys_and_certs.sh | 0 molecule/{all_auth => }/prepare.yml | 0 .../prepare.yml => prepare_ssl.yml} | 0 molecule/ssl_auth_sasl/molecule.yml | 2 +- molecule/ssl_auth_sasl/prepare.yml | 42 ---------------- .../ssl_no_auth/generate_keys_and_certs.sh | 48 ------------------- molecule/ssl_no_auth/molecule.yml | 2 +- roles/amq_streams_common/tasks/prometheus.yml | 26 ++++++++++ 15 files changed, 32 insertions(+), 126 deletions(-) delete mode 100644 molecule/all_cluster/prepare.yml delete mode 100644 molecule/connect/prepare.yml delete mode 100644 molecule/default/prepare.yml rename molecule/{ssl_auth_sasl => }/generate_keys_and_certs.sh (100%) rename molecule/{all_auth => }/prepare.yml (100%) rename molecule/{ssl_no_auth/prepare.yml => prepare_ssl.yml} (100%) delete mode 100644 molecule/ssl_auth_sasl/prepare.yml delete mode 100755 molecule/ssl_no_auth/generate_keys_and_certs.sh create mode 100644 roles/amq_streams_common/tasks/prometheus.yml diff --git a/molecule/all_auth/molecule.yml b/molecule/all_auth/molecule.yml index 2fa4982..29aab5a 100644 --- a/molecule/all_auth/molecule.yml +++ b/molecule/all_auth/molecule.yml @@ -18,7 +18,7 @@ provisioner: ssh_connection: pipelining: false playbooks: - prepare: prepare.yml + prepare: ../prepare.yml converge: converge.yml verify: verify.yml inventory: diff --git a/molecule/all_cluster/molecule.yml b/molecule/all_cluster/molecule.yml index 64eb5cd..2ddeba3 100644 --- a/molecule/all_cluster/molecule.yml +++ b/molecule/all_cluster/molecule.yml @@ -18,7 +18,7 @@ provisioner: ssh_connection: pipelining: false playbooks: - prepare: prepare.yml + prepare: ../prepare.yml converge: converge.yml verify: verify.yml inventory: diff --git a/molecule/all_cluster/prepare.yml b/molecule/all_cluster/prepare.yml deleted file mode 100644 index 0edc37a..0000000 --- a/molecule/all_cluster/prepare.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - - name: "Ensure required packages are installed." - ansible.builtin.yum: - name: - - sudo - state: present diff --git a/molecule/connect/molecule.yml b/molecule/connect/molecule.yml index 2fa4982..29aab5a 100644 --- a/molecule/connect/molecule.yml +++ b/molecule/connect/molecule.yml @@ -18,7 +18,7 @@ provisioner: ssh_connection: pipelining: false playbooks: - prepare: prepare.yml + prepare: ../prepare.yml converge: converge.yml verify: verify.yml inventory: diff --git a/molecule/connect/prepare.yml b/molecule/connect/prepare.yml deleted file mode 100644 index 0edc37a..0000000 --- a/molecule/connect/prepare.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - - name: "Ensure required packages are installed." - ansible.builtin.yum: - name: - - sudo - state: present diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 424e162..3cd07ca 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -18,7 +18,7 @@ provisioner: ssh_connection: pipelining: false playbooks: - prepare: prepare.yml + prepare: ../prepare.yml converge: converge.yml verify: verify.yml inventory: diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml deleted file mode 100644 index 0edc37a..0000000 --- a/molecule/default/prepare.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Prepare - hosts: all - tasks: - - - name: "Ensure required packages are installed." - ansible.builtin.yum: - name: - - sudo - state: present diff --git a/molecule/ssl_auth_sasl/generate_keys_and_certs.sh b/molecule/generate_keys_and_certs.sh similarity index 100% rename from molecule/ssl_auth_sasl/generate_keys_and_certs.sh rename to molecule/generate_keys_and_certs.sh diff --git a/molecule/all_auth/prepare.yml b/molecule/prepare.yml similarity index 100% rename from molecule/all_auth/prepare.yml rename to molecule/prepare.yml diff --git a/molecule/ssl_no_auth/prepare.yml b/molecule/prepare_ssl.yml similarity index 100% rename from molecule/ssl_no_auth/prepare.yml rename to molecule/prepare_ssl.yml diff --git a/molecule/ssl_auth_sasl/molecule.yml b/molecule/ssl_auth_sasl/molecule.yml index 424e162..7752c21 100644 --- a/molecule/ssl_auth_sasl/molecule.yml +++ b/molecule/ssl_auth_sasl/molecule.yml @@ -18,7 +18,7 @@ provisioner: ssh_connection: pipelining: false playbooks: - prepare: prepare.yml + prepare: ../prepare_ssl.yml converge: converge.yml verify: verify.yml inventory: diff --git a/molecule/ssl_auth_sasl/prepare.yml b/molecule/ssl_auth_sasl/prepare.yml deleted file mode 100644 index fcbf82d..0000000 --- a/molecule/ssl_auth_sasl/prepare.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- name: Prepare - hosts: all -# collections: -# - community.general.java_keystore -# - community.crypto - tasks: - - - name: "Ensure required packages are installed." - ansible.builtin.yum: - name: - - sudo - - java-17-openjdk-headless - - openssl - state: present - - - ansible.builtin.copy: - src: generate_keys_and_certs.sh - dest: /tmp/ - owner: root - group: root - mode: "0077" - - - name: "Generate required SSL artifacts." - ansible.builtin.command: "/tmp/generate_keys_and_certs.sh" - register: output - - - ansible.builtin.debug: - var: output - -# TODO: use Ansible crypto and jks collections to replace above script -# - name: Generate an OpenSSH keypair with the default values (4096 bits, rsa) -# community.crypto.openssh_keypair: -# path: /tmp/id_ssh_rsa -# -# - name: Create a keystore for the given certificate/private key pair (inline) -# community.general.java_keystore: -# name: example -# certificate: /etc/ssl/certs/ca-bundle.crt -# private_key: /tmp/id_ssh_rsa -# password: changeit -# dest: /etc/security/keystore.jks diff --git a/molecule/ssl_no_auth/generate_keys_and_certs.sh b/molecule/ssl_no_auth/generate_keys_and_certs.sh deleted file mode 100755 index 3997ca1..0000000 --- a/molecule/ssl_no_auth/generate_keys_and_certs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -e -readonly KEYS_HOME='/opt' -readonly KEYSTORE_FILE="${KEYS_HOME}/server.keystore.jks" -readonly KAFKA_SERVER_KEY="${KEYS_HOME}/kafka.server.key" -readonly KAFKA_SERVER_CSR="${KEYS_HOME}/kafka.server.csr" -readonly KAFKA_SERVER_CRT="${KEYS_HOME}/kafka.server.crt" -readonly KAFKA_SERVER_P12="${KEYS_HOME}/kafka.server.p12" -readonly KAFKA_SERVER_TRUSTSTORE="${KEYS_HOME}/server.truststore.jks" -readonly CLIENT_P12="${KEYS_HOME}/client.p12" -readonly CLIENT_TRUSTSTORE_JKS="${KEYS_HOME}/client.truststore.jks" - -cd "${KEYS_HOME}" - -if [ ! -e "${KEYSTORE_FILE}" ]; then - keytool -genkey -keystore "${KEYSTORE_FILE}" -alias server -validity 999 -keyalg RSA -keypass password -storepass password -dname "cn=Unknown, ou=Unknown, o=Unknown, c=Unknown" -fi - -if [ ! -e "${KAFKA_SERVER_KEY}" ]; then - openssl genrsa -out "${KAFKA_SERVER_KEY}" 2048 -fi - -if [ ! -e "${KAFKA_SERVER_CSR}" ]; then - openssl req -new -key kafka.server.key -out "${KAFKA_SERVER_CSR}" -passin pass:client11 -subj "/C=US/ST=Molecule/L=Berlin /O=Ansible Middleware/OU=Test/CN=localhost/emailAddress=dummy@localhost.localdomain" -fi - -if [ ! -e "${KAFKA_SERVER_CRT}" ]; then - openssl x509 -req -days 999 -in "${KAFKA_SERVER_CSR}" -signkey "${KAFKA_SERVER_KEY}" -out "${KAFKA_SERVER_CRT}" > /dev/null -fi - -if [ ! -e "${KAFKA_SERVER_P12}" ]; then - openssl pkcs12 -export -name localhost -in "${KAFKA_SERVER_CRT}" -inkey "${KAFKA_SERVER_KEY}" -out "${KAFKA_SERVER_P12}" -passout pass:client11 - keytool -keystore "${KEYSTORE_FILE}" -alias localhost -importkeystore -srckeystore "${KAFKA_SERVER_P12}" -srcstoretype PKCS12 -storepass password -srcstorepass client11 -noprompt -fi - -#* Create truststore importing the certificate -# -#```shell -if [ ! -e "${KAFKA_SERVER_TRUSTSTORE}" ]; then - keytool -keystore "${KAFKA_SERVER_TRUSTSTORE}" -alias CARoot -import -file "${KAFKA_SERVER_CRT}" -storepass password -noprompt -fi - -if [ ! -e "${CLIENT_P12}" ]; then - openssl pkcs12 -export -in "${KAFKA_SERVER_CRT}" -inkey "${KAFKA_SERVER_KEY}" -out "${CLIENT_P12}" -passout pass:client11 -fi - -if [ ! -e "${CLIENT_TRUSTSTORE_JKS}" ]; then - keytool -keystore "${CLIENT_TRUSTSTORE_JKS}" -alias CARoot -import -file "${KAFKA_SERVER_CRT}" -storepass password -keypass password -noprompt -fi diff --git a/molecule/ssl_no_auth/molecule.yml b/molecule/ssl_no_auth/molecule.yml index 424e162..3cd07ca 100644 --- a/molecule/ssl_no_auth/molecule.yml +++ b/molecule/ssl_no_auth/molecule.yml @@ -18,7 +18,7 @@ provisioner: ssh_connection: pipelining: false playbooks: - prepare: prepare.yml + prepare: ../prepare.yml converge: converge.yml verify: verify.yml inventory: diff --git a/roles/amq_streams_common/tasks/prometheus.yml b/roles/amq_streams_common/tasks/prometheus.yml new file mode 100644 index 0000000..27d7e6d --- /dev/null +++ b/roles/amq_streams_common/tasks/prometheus.yml @@ -0,0 +1,26 @@ +--- +- name: "Ensure required parameters for Prometheus are provided." + ansible.builtin.assert: + that: + - amq_streams_common_prometheus_config_file is defined and amq_streams_common_prometheus_config_file | length > 0 + - amq_streams_common_prometheus_config_file_template is defined and amq_streams_common_prometheus_config_file_template | length > 0 + quiet: True + +- name: "Ensure directory {{ amq_streams_common_prometheus_metrics_config_home }} exits." + ansible.builtin.file: + path: "{{ amq_streams_common_prometheus_metrics_config_home }}" + state: directory + +- name: "Deploy Prometheus metrics file (src: {{ amq_streams_common_prometheus_config_file_template }} to dest: {{ amq_streams_common_prometheus_metrics_config_home }}{{ amq_streams_common_prometheus_config_file }}" + ansible.builtin.template: + src: "{{ amq_streams_common_prometheus_config_file_template }}" + dest: "{{ amq_streams_common_prometheus_metrics_config_home }}{{ amq_streams_common_prometheus_config_file }}" + owner: "{{ amq_streams_common_prometheus_user | default(omit) }}" + group: "{{ amq_streams_common_prometheus_group | default(omit) }}" + mode: 0644 + +- name: "Add prometheus dependency (if enabled: {{ amq_streams_common_prometheus_enabled }})." + ansible.builtin.set_fact: + amq_streams_common_dependencies: "{{ [amq_streams_common_dependencies + ['{{ amq_streams_common_prometheus_package_name }}']] | flatten }}" + when: + - amq_streams_common_prometheus_install_rpm is defined and amq_streams_common_prometheus_install_rpm