diff --git a/.ansible-lint b/.ansible-lint index 70df42f..f89ba70 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -40,6 +40,7 @@ warn_list: - schema[tasks] - name[missing] - meta-runtime[unsupported-version] + - fqcn[keyword] skip_list: - vars_should_not_be_used diff --git a/molecule/all_auth/converge.yml b/molecule/all_auth/converge.yml index f3a2dba..b6e3eb8 100644 --- a/molecule/all_auth/converge.yml +++ b/molecule/all_auth/converge.yml @@ -1,4 +1,158 @@ --- -- import_playbook: ../../playbooks/amq_streams_all_auth.yml +- name: "Ansible Playbook to install Zookeeper and Broker with Authentication" + hosts: all vars: - amq_streams_common_download_dir: /tmp/ + # Enabling Zookeeper Authentication + amq_streams_zookeeper_auth_enabled: true + amq_streams_zookeeper_auth_user: zkadmin + amq_streams_zookeeper_auth_pass: p@ssw0rd + + # Enabling Kafka Broker Listeners + amq_streams_broker_listeners: + - AUTHENTICATED://:{{ amq_streams_broker_listener_port }} + - REPLICATION://:{{ amq_streams_broker_listener_internal_port }} + + # Listener for inter-broker communications + amq_streams_broker_inter_broker_listener: REPLICATION + + # Enabling Kafka Broker Authentication + amq_streams_broker_auth_enabled: true + amq_streams_broker_auth_scram_enabled: true + amq_streams_broker_auth_listeners: + - AUTHENTICATED:SASL_PLAINTEXT + - REPLICATION:PLAINTEXT + + amq_streams_broker_auth_sasl_mechanisms: + - PLAIN + - SCRAM-SHA-512 + + # Kafka Plain Users + amq_streams_broker_auth_plain_users: + - username: admin + password: p@ssw0rd + - username: kafkauser01 + password: p@ssw0rd + - username: kafkauser02 + password: p@ssw0rd + + # Kafka SCRAM Users + amq_streams_broker_auth_scram_users: + - username: kafkascramuser01 + password: p@ssw0rd + - username: kafkascramuser02 + password: p@ssw0rd + + # Defining default Kafka user for administrative tasks + amq_streams_broker_admin_mechanism: PLAIN + #amq_streams_broker_admin_mechanism: SCRAM-SHA-512 + amq_streams_broker_admin_username: admin + amq_streams_broker_admin_password: p@ssw0rd + + # Topic Management + amq_streams_broker_topics: + - name: sampleTopic + partitions: 1 + replication_factor: 1 + - name: otherTopic + partitions: 1 + replication_factor: 1 + roles: + - role: amq_streams_zookeeper + tasks: + - name: "Ensure Zookeeper is running and available." + ansible.builtin.include_role: + name: amq_streams_zookeeper + vars: + amq_streams_common_skip_download: true + + - name: "Ensure Broker is running and available." + ansible.builtin.include_role: + name: amq_streams_broker + vars: + amq_streams_common_skip_download: true + + - name: "Create topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/create.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + topic_partitions: "{{ topic.partitions }}" + topic_replication_factor: "{{ topic.replication_factor }}" + + - name: "Describe topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/describe.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Delete topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/delete.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Create SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/create.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + user_password: "{{ user.password }}" + + - name: "Describe SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/describe.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + + - name: "Delete SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/delete.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + + post_tasks: + - name: "Display numbers of Zookeeper instances managed by Ansible." + ansible.builtin.debug: + msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." + when: + - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled + + - name: "Display numbers of broker instances managed by Ansible:" + ansible.builtin.debug: + msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." + when: + - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled + + - name: "Validate that Zookeeper deployment is functional." + ansible.builtin.include_role: + name: amq_streams_zookeeper + tasks_from: validate.yml + + - name: "Validate that Broker deployment is functional." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: validate.yml diff --git a/molecule/all_cluster/converge.yml b/molecule/all_cluster/converge.yml index f31ac71..a174584 100644 --- a/molecule/all_cluster/converge.yml +++ b/molecule/all_cluster/converge.yml @@ -1,4 +1,64 @@ --- -- import_playbook: ../../playbooks/amq_streams_all_cluster.yml +- name: "Ansible Playbook to install a Kafka cluster (+Zookeeper)" + hosts: all vars: - amq_streams_common_download_dir: /tmp/ + amq_streams_common_download_node: "" + amq_streams_common_download_dir: "/tmp" + + # Topic Management + amq_streams_broker_topics: + - name: myTopic + partitions: 1 + replication_factor: 1 + roles: + - role: amq_streams_zookeeper + tasks: + - name: "Ensure AMQ Streams Broker is running and available." + ansible.builtin.include_role: + name: amq_streams_broker + vars: + amq_streams_common_skip_download: true + + - name: "Create topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/create.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + topic_partitions: "{{ topic.partitions }}" + topic_replication_factor: "{{ topic.replication_factor }}" + + - name: "Describe topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/describe.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + post_tasks: + - name: "Display numbers of Zookeeper instances managed by Ansible." + ansible.builtin.debug: + msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." + when: + - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled + + - name: "Display numbers of broker instances managed by Ansible." + ansible.builtin.debug: + msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." + when: + - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled + + - name: "Validate that Broker deployment is functional." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: validate.yml + + - name: "Validate that deployment is functional." + ansible.builtin.include_role: + name: amq_streams_zookeeper + tasks_from: validate.yml diff --git a/playbooks/certs/README.md b/molecule/certs/README.md similarity index 100% rename from playbooks/certs/README.md rename to molecule/certs/README.md diff --git a/playbooks/certs/client.keystore.jks b/molecule/certs/client.keystore.jks similarity index 100% rename from playbooks/certs/client.keystore.jks rename to molecule/certs/client.keystore.jks diff --git a/playbooks/certs/client.truststore.jks b/molecule/certs/client.truststore.jks similarity index 100% rename from playbooks/certs/client.truststore.jks rename to molecule/certs/client.truststore.jks diff --git a/playbooks/certs/server.keystore.jks b/molecule/certs/server.keystore.jks similarity index 100% rename from playbooks/certs/server.keystore.jks rename to molecule/certs/server.keystore.jks diff --git a/molecule/certs/server.truststore.jks b/molecule/certs/server.truststore.jks new file mode 100644 index 0000000..3cb790e Binary files /dev/null and b/molecule/certs/server.truststore.jks differ diff --git a/molecule/connect/converge.yml b/molecule/connect/converge.yml index 7e33a4b..1e74169 100644 --- a/molecule/connect/converge.yml +++ b/molecule/connect/converge.yml @@ -1,4 +1,10 @@ --- -- import_playbook: ../../playbooks/amq_streams_connect.yml - vars: - amq_streams_common_download_dir: /tmp/ +- name: "Ansible playbook to install a Kafka Connect cluster" + hosts: all + tasks: + - name: "Ensure Kafka Connect is running and available." + ansible.builtin.include_role: + name: amq_streams_connect + vars: + connectors: + - { name: "file", path: "connectors/file.yml" } diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 0d03227..b37cde1 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -1,4 +1,86 @@ --- -- import_playbook: ../../playbooks/playbook.yml +- name: "Automate AMQ Streams install" + hosts: all vars: - amq_streams_common_download_dir: /tmp/ + # Topic Management + amq_streams_broker_topics: + - name: sampleTopic + partitions: 2 + replication_factor: 1 + - name: otherTopic + partitions: 4 + replication_factor: 1 + roles: + - role: amq_streams_zookeeper + tasks: + - name: "Ensure AMQ Streams Broker is running and available." + ansible.builtin.include_role: + name: amq_streams_broker + vars: + amq_streams_common_skip_download: true + + - name: "Ensures topics exist." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/create.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + topic_partitions: "{{ topic.partitions }}" + topic_replication_factor: "{{ topic.replication_factor }}" + + - name: "Describe created topic." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/describe.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Delete topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/delete.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Ensure AMQ Streams Connect is running and available." + ansible.builtin.include_role: + name: amq_streams_connect + vars: + connectors: + - { name: "file", path: "connectors/file.yml" } + post_tasks: + - name: "Display numbers of Zookeeper instances managed by Ansible." + ansible.builtin.debug: + msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." + when: + - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled + + - name: "Display numbers of broker instances managed by Ansible." + ansible.builtin.debug: + msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." + when: + - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled + + - name: "Validate that deployment is functional." + ansible.builtin.include_role: + name: amq_streams_zookeeper + tasks_from: validate.yml + + - name: "Validate that Broker deployment is functional." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: validate.yml + + - name: "Validate that Connect deployment is functional." + ansible.builtin.include_role: + name: amq_streams_connect + tasks_from: validate.yml diff --git a/molecule/ssl_auth_sasl/converge.yml b/molecule/ssl_auth_sasl/converge.yml index f782352..4fcf0f2 100644 --- a/molecule/ssl_auth_sasl/converge.yml +++ b/molecule/ssl_auth_sasl/converge.yml @@ -1,4 +1,196 @@ --- -- import_playbook: ../../playbooks/amq_streams_ssl_auth_sasl.yml +- name: "Ansible Playbook to install Zookeeper and Broker with SSL and SASL authentication" + hosts: all vars: - amq_streams_common_download_dir: /tmp/ + # Enabling Zookeeper Authentication + amq_streams_zookeeper_auth_enabled: true + amq_streams_zookeeper_auth_user: zkadmin + amq_streams_zookeeper_auth_pass: p@ssw0rd + + # Enabling SSL + amq_streams_broker_tls_enabled: true + amq_streams_broker_tls_keystore_dir: ../certs + amq_streams_broker_tls_truststore_dir: ../certs + amq_streams_broker_tls_truststore_client_dir: ../certs + amq_streams_broker_tls_keystore_password: password + amq_streams_broker_tls_truststore_password: password + amq_streams_broker_tls_truststore_client_password: password + + # Broker Listeners + amq_streams_broker_listeners: + - PLAINTEXT://:{{ amq_streams_broker_listener_port }} # Insecure for inter-broker connections + - SASL_SSL://:{{ amq_streams_broker_listener_tls_port }} # Secured connections + + # Enabling Kafka Broker Authentication + amq_streams_broker_auth_enabled: true + amq_streams_broker_auth_scram_enabled: true + amq_streams_broker_auth_listeners: + - PLAINTEXT:PLAINTEXT + - SASL_SSL:SASL_SSL + + amq_streams_broker_auth_sasl_mechanisms: + - PLAIN + - SCRAM-SHA-512 + + # Using SSL bootstrap server port + amq_streams_broker_bootstrap_server_host: localhost + amq_streams_broker_bootstrap_server_port: 9093 + + # Defining default Kafka user for administrative tasks + amq_streams_broker_admin_mechanism: PLAIN + #amq_streams_broker_admin_mechanism: SCRAM-SHA-512 + amq_streams_broker_admin_username: admin + amq_streams_broker_admin_password: p@ssw0rd + + # Kafka Plain Users + amq_streams_broker_auth_plain_users: + - username: admin + password: p@ssw0rd + - username: kafkauser01 + password: p@ssw0rd + - username: kafkauser02 + password: p@ssw0rd + + # Kafka SCRAM Users + amq_streams_broker_auth_scram_users: + - username: kafkascramuser01 + password: p@ssw0rd + - username: kafkascramuser02 + password: p@ssw0rd + + # Topic Management + amq_streams_broker_topics: + - name: sampleTopic + partitions: 1 + replication_factor: 1 + - name: otherTopic + partitions: 1 + replication_factor: 1 + + # Enabling SSL/TLS communications between Kafka Connect and Kafka Broker + amq_streams_connect_broker_tls_enabled: true + amq_streams_connect_broker_tls_truststore_client_password: password + + # Enabled Broker Authentication + amq_streams_connect_broker_auth_enabled: 'true' + amq_streams_connect_broker_auth_scram_enabled: 'false' + amq_streams_connect_broker_auth_username: admin + amq_streams_connect_broker_auth_password: p@ssw0rd + + # Connection to SSL endpoint + amq_streams_connect_bootstrap_servers: localhost:9093 + collections: + - middleware_automation.amq_streams + roles: + - role: amq_streams_zookeeper + tasks: + - name: "Ensure Zookeeper is running and available." + ansible.builtin.include_role: + name: amq_streams_zookeeper + vars: + amq_streams_common_skip_download: true + + - name: "Ensure Broker is running and available." + ansible.builtin.include_role: + name: amq_streams_broker + vars: + amq_streams_common_skip_download: true + + - name: "Create topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/create.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + topic_partitions: "{{ topic.partitions }}" + topic_replication_factor: "{{ topic.replication_factor }}" + + - name: "Describe topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/describe.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Delete topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/delete.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Create SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/create.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + user_password: "{{ user.password }}" + + - name: "Describe SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/describe.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + + - name: "Delete SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/delete.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + + - name: "Ensure AMQ Streams Connect is running and available." + ansible.builtin.include_role: + name: amq_streams_connect + vars: + amq_streams_common_skip_download: true + connectors: + - { name: "file", path: "connectors/file.yml" } + + post_tasks: + - name: "Display numbers of Zookeeper instances managed by Ansible." + ansible.builtin.debug: + msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." + when: + - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled + + - name: "Display numbers of broker instances managed by Ansible:" + ansible.builtin.debug: + msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." + when: + - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled + + - name: "Validate that Zookeeper deployment is functional." + ansible.builtin.include_role: + name: amq_streams_zookeeper + tasks_from: validate.yml + + - name: "Validate that Broker deployment is functional." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: validate.yml + + - name: "Validate that Connect deployment is functional." + ansible.builtin.include_role: + name: amq_streams_connect + tasks_from: validate.yml diff --git a/molecule/ssl_no_auth/converge.yml b/molecule/ssl_no_auth/converge.yml index eea3a3b..1ec7e1a 100644 --- a/molecule/ssl_no_auth/converge.yml +++ b/molecule/ssl_no_auth/converge.yml @@ -1,4 +1,160 @@ --- -- import_playbook: ../../playbooks/amq_streams_ssl_no_auth.yml +- name: "Ansible Playbook to install Zookeeper and Broker with SSL and no authentication" + hosts: all vars: - amq_streams_common_download_dir: /tmp/ + # Enabling SSL listeners + amq_streams_broker_tls_enabled: true + amq_streams_broker_tls_keystore_dir: ../certs + amq_streams_broker_tls_truststore_dir: ../certs + amq_streams_broker_tls_truststore_client_dir: ../certs + amq_streams_broker_tls_keystore_password: password + amq_streams_broker_tls_truststore_password: password + amq_streams_broker_tls_keystore_client_password: password + amq_streams_broker_tls_truststore_client_password: password + + # Broker Listeners + amq_streams_broker_listeners: + - PLAINTEXT://:{{ amq_streams_broker_listener_port }} # Insecure for inter-broker connections + - SSL://:{{ amq_streams_broker_listener_tls_port }} # Secure for client connections + + # Using SSL bootstrap server port + amq_streams_broker_bootstrap_server_host: localhost + amq_streams_broker_bootstrap_server_port: 9093 + + # Kafka SCRAM Users + amq_streams_broker_auth_scram_users: + - username: kafkascramuser01 + password: p@ssw0rd + - username: kafkascramuser02 + password: p@ssw0rd + + # Topic Management + amq_streams_broker_topics: + - name: sampleTopic + partitions: 1 + replication_factor: 1 + - name: otherTopic + partitions: 1 + replication_factor: 1 + + # Enabling SSL/TLS communications between Kafka Connect and Kafka Broker + amq_streams_connect_broker_tls_enabled: true + amq_streams_connect_broker_tls_truststore_client_password: password + + # Connection to SSL endpoint + amq_streams_connect_bootstrap_servers: localhost:9093 + collections: + - middleware_automation.amq_streams + roles: + - role: amq_streams_zookeeper + tasks: + - name: "Ensure Zookeeper is running and available." + ansible.builtin.include_role: + name: amq_streams_zookeeper + vars: + amq_streams_common_skip_download: true + + - name: "Ensure Broker is running and available." + ansible.builtin.include_role: + name: amq_streams_broker + vars: + amq_streams_common_skip_download: true + + - name: "Create topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/create.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + topic_partitions: "{{ topic.partitions }}" + topic_replication_factor: "{{ topic.replication_factor }}" + + - name: "Describe topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/describe.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Delete topics" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: topic/delete.yml + loop: "{{ amq_streams_broker_topics }}" + loop_control: + loop_var: topic + vars: + topic_name: "{{ topic.name }}" + + - name: "Create SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/create.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + user_password: "{{ user.password }}" + + - name: "Describe SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/describe.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + + - name: "Delete SCRAM users" + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: user-scram/delete.yml + loop: "{{ amq_streams_broker_auth_scram_users }}" + loop_control: + loop_var: user + vars: + user_username: "{{ user.username }}" + + - name: "Ensure AMQ Streams Connect is running and available." + ansible.builtin.include_role: + name: amq_streams_connect + vars: + amq_streams_common_skip_download: true + connectors: + - { name: "file", path: "connectors/file.yml" } + + post_tasks: + - name: "Display numbers of Zookeeper instances managed by Ansible." + ansible.builtin.debug: + msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." + when: + - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled + + - name: "Display numbers of broker instances managed by Ansible:" + ansible.builtin.debug: + msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." + when: + - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled + + - name: "Validate that Zookeeper deployment is functional." + ansible.builtin.include_role: + name: amq_streams_zookeeper + tasks_from: validate.yml + + - name: "Validate that Broker deployment is functional." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: validate.yml + + - name: "Validate that Connect deployment is functional." + ansible.builtin.include_role: + name: amq_streams_connect + tasks_from: validate.yml diff --git a/playbooks/amq_streams_all_auth.yml b/playbooks/amq_streams_all_auth.yml deleted file mode 100644 index b6e3eb8..0000000 --- a/playbooks/amq_streams_all_auth.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- -- name: "Ansible Playbook to install Zookeeper and Broker with Authentication" - hosts: all - vars: - # Enabling Zookeeper Authentication - amq_streams_zookeeper_auth_enabled: true - amq_streams_zookeeper_auth_user: zkadmin - amq_streams_zookeeper_auth_pass: p@ssw0rd - - # Enabling Kafka Broker Listeners - amq_streams_broker_listeners: - - AUTHENTICATED://:{{ amq_streams_broker_listener_port }} - - REPLICATION://:{{ amq_streams_broker_listener_internal_port }} - - # Listener for inter-broker communications - amq_streams_broker_inter_broker_listener: REPLICATION - - # Enabling Kafka Broker Authentication - amq_streams_broker_auth_enabled: true - amq_streams_broker_auth_scram_enabled: true - amq_streams_broker_auth_listeners: - - AUTHENTICATED:SASL_PLAINTEXT - - REPLICATION:PLAINTEXT - - amq_streams_broker_auth_sasl_mechanisms: - - PLAIN - - SCRAM-SHA-512 - - # Kafka Plain Users - amq_streams_broker_auth_plain_users: - - username: admin - password: p@ssw0rd - - username: kafkauser01 - password: p@ssw0rd - - username: kafkauser02 - password: p@ssw0rd - - # Kafka SCRAM Users - amq_streams_broker_auth_scram_users: - - username: kafkascramuser01 - password: p@ssw0rd - - username: kafkascramuser02 - password: p@ssw0rd - - # Defining default Kafka user for administrative tasks - amq_streams_broker_admin_mechanism: PLAIN - #amq_streams_broker_admin_mechanism: SCRAM-SHA-512 - amq_streams_broker_admin_username: admin - amq_streams_broker_admin_password: p@ssw0rd - - # Topic Management - amq_streams_broker_topics: - - name: sampleTopic - partitions: 1 - replication_factor: 1 - - name: otherTopic - partitions: 1 - replication_factor: 1 - roles: - - role: amq_streams_zookeeper - tasks: - - name: "Ensure Zookeeper is running and available." - ansible.builtin.include_role: - name: amq_streams_zookeeper - vars: - amq_streams_common_skip_download: true - - - name: "Ensure Broker is running and available." - ansible.builtin.include_role: - name: amq_streams_broker - vars: - amq_streams_common_skip_download: true - - - name: "Create topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/create.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - topic_partitions: "{{ topic.partitions }}" - topic_replication_factor: "{{ topic.replication_factor }}" - - - name: "Describe topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/describe.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - - - name: "Delete topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/delete.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - - - name: "Create SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/create.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - user_password: "{{ user.password }}" - - - name: "Describe SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/describe.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - - - name: "Delete SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/delete.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - - post_tasks: - - name: "Display numbers of Zookeeper instances managed by Ansible." - ansible.builtin.debug: - msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." - when: - - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled - - - name: "Display numbers of broker instances managed by Ansible:" - ansible.builtin.debug: - msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." - when: - - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled - - - name: "Validate that Zookeeper deployment is functional." - ansible.builtin.include_role: - name: amq_streams_zookeeper - tasks_from: validate.yml - - - name: "Validate that Broker deployment is functional." - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: validate.yml diff --git a/playbooks/amq_streams_all_cluster.yml b/playbooks/amq_streams_all_cluster.yml deleted file mode 100644 index 7c03d52..0000000 --- a/playbooks/amq_streams_all_cluster.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -- name: "Ansible Playbook to install a Kafka cluster (+Zookeeper)" - hosts: all - vars: - amq_streams_common_download_node: "" - amq_streams_common_download_dir: "/tmp" - amq_streams_broker_prometheus_enabled: True - amq_streams_zookeeper_prometheus_enabled: True - amq_streams_zookeeper_prometheus_port: 18081 - - # Topic Management - amq_streams_broker_topics: - - name: myTopic - partitions: 1 - replication_factor: 1 - roles: - - role: amq_streams_zookeeper - tasks: - - name: "Ensure AMQ Streams Broker is running and available." - ansible.builtin.include_role: - name: amq_streams_broker - vars: - amq_streams_common_skip_download: true - - - name: "Create topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/create.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - topic_partitions: "{{ topic.partitions }}" - topic_replication_factor: "{{ topic.replication_factor }}" - - - name: "Describe topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/describe.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - post_tasks: - - name: "Display numbers of Zookeeper instances managed by Ansible." - ansible.builtin.debug: - msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." - when: - - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled - - - name: "Display numbers of broker instances managed by Ansible." - ansible.builtin.debug: - msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." - when: - - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled - - - name: "Validate that Broker deployment is functional." - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: validate.yml - - - name: "Validate that deployment is functional." - ansible.builtin.include_role: - name: amq_streams_zookeeper - tasks_from: validate.yml diff --git a/playbooks/amq_streams_connect.yml b/playbooks/amq_streams_connect.yml deleted file mode 100644 index 1e74169..0000000 --- a/playbooks/amq_streams_connect.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: "Ansible playbook to install a Kafka Connect cluster" - hosts: all - tasks: - - name: "Ensure Kafka Connect is running and available." - ansible.builtin.include_role: - name: amq_streams_connect - vars: - connectors: - - { name: "file", path: "connectors/file.yml" } diff --git a/playbooks/amq_streams_ssl_auth_sasl.yml b/playbooks/amq_streams_ssl_auth_sasl.yml deleted file mode 100644 index 124fa93..0000000 --- a/playbooks/amq_streams_ssl_auth_sasl.yml +++ /dev/null @@ -1,195 +0,0 @@ ---- -- name: "Ansible Playbook to install Zookeeper and Broker with SSL and SASL authentication" - hosts: all - vars: - # Enabling Zookeeper Authentication - amq_streams_zookeeper_auth_enabled: true - amq_streams_zookeeper_auth_user: zkadmin - amq_streams_zookeeper_auth_pass: p@ssw0rd - - # Enabling SSL - amq_streams_broker_tls_enabled: true - amq_streams_broker_tls_keystore_dir: ./certs - amq_streams_broker_tls_truststore_dir: ./certs - amq_streams_broker_tls_truststore_client_dir: ./certs - amq_streams_broker_tls_keystore_password: password - amq_streams_broker_tls_truststore_password: password - amq_streams_broker_tls_truststore_client_password: password - - # Broker Listeners - amq_streams_broker_listeners: - - PLAINTEXT://:{{ amq_streams_broker_listener_port }} # Insecure for inter-broker connections - - SASL_SSL://:{{ amq_streams_broker_listener_tls_port }} # Secured connections - - # Enabling Kafka Broker Authentication - amq_streams_broker_auth_enabled: true - amq_streams_broker_auth_scram_enabled: true - amq_streams_broker_auth_listeners: - - PLAINTEXT:PLAINTEXT - - SASL_SSL:SASL_SSL - - amq_streams_broker_auth_sasl_mechanisms: - - PLAIN - - SCRAM-SHA-512 - - # Using SSL bootstrap server port - amq_streams_broker_bootstrap_server_host: localhost - amq_streams_broker_bootstrap_server_port: 9093 - - # Defining default Kafka user for administrative tasks - amq_streams_broker_admin_mechanism: PLAIN - #amq_streams_broker_admin_mechanism: SCRAM-SHA-512 - amq_streams_broker_admin_username: admin - amq_streams_broker_admin_password: p@ssw0rd - - # Kafka Plain Users - amq_streams_broker_auth_plain_users: - - username: admin - password: p@ssw0rd - - username: kafkauser01 - password: p@ssw0rd - - username: kafkauser02 - password: p@ssw0rd - - # Kafka SCRAM Users - amq_streams_broker_auth_scram_users: - - username: kafkascramuser01 - password: p@ssw0rd - - username: kafkascramuser02 - password: p@ssw0rd - - # Topic Management - amq_streams_broker_topics: - - name: sampleTopic - partitions: 1 - replication_factor: 1 - - name: otherTopic - partitions: 1 - replication_factor: 1 - - # Enabling SSL/TLS communications between Kafka Connect and Kafka Broker - amq_streams_connect_broker_tls_enabled: true - amq_streams_connect_broker_tls_truststore_client_password: password - - # Enabled Broker Authentication - amq_streams_connect_broker_auth_enabled: 'true' - amq_streams_connect_broker_auth_scram_enabled: 'false' - amq_streams_connect_broker_auth_username: admin - amq_streams_connect_broker_auth_password: p@ssw0rd - - # Connection to SSL endpoint - amq_streams_connect_bootstrap_servers: localhost:9093 - - roles: - - role: amq_streams_zookeeper - tasks: - - name: "Ensure Zookeeper is running and available." - ansible.builtin.include_role: - name: amq_streams_zookeeper - vars: - amq_streams_common_skip_download: true - - - name: "Ensure Broker is running and available." - ansible.builtin.include_role: - name: amq_streams_broker - vars: - amq_streams_common_skip_download: true - - - name: "Create topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/create.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - topic_partitions: "{{ topic.partitions }}" - topic_replication_factor: "{{ topic.replication_factor }}" - - - name: "Describe topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/describe.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - - - name: "Delete topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/delete.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - - - name: "Create SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/create.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - user_password: "{{ user.password }}" - - - name: "Describe SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/describe.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - - - name: "Delete SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/delete.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - - - name: "Ensure AMQ Streams Connect is running and available." - ansible.builtin.include_role: - name: amq_streams_connect - vars: - amq_streams_common_skip_download: true - connectors: - - { name: "file", path: "connectors/file.yml" } - - post_tasks: - - name: "Display numbers of Zookeeper instances managed by Ansible." - ansible.builtin.debug: - msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." - when: - - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled - - - name: "Display numbers of broker instances managed by Ansible:" - ansible.builtin.debug: - msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." - when: - - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled - - - name: "Validate that Zookeeper deployment is functional." - ansible.builtin.include_role: - name: amq_streams_zookeeper - tasks_from: validate.yml - - - name: "Validate that Broker deployment is functional." - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: validate.yml - - - name: "Validate that Connect deployment is functional." - ansible.builtin.include_role: - name: amq_streams_connect - tasks_from: validate.yml diff --git a/playbooks/amq_streams_ssl_no_auth.yml b/playbooks/amq_streams_ssl_no_auth.yml deleted file mode 100644 index 617e4f0..0000000 --- a/playbooks/amq_streams_ssl_no_auth.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- -- name: "Ansible Playbook to install Zookeeper and Broker with SSL and no authentication" - hosts: all - vars: - # Enabling SSL listeners - amq_streams_broker_tls_enabled: true - amq_streams_broker_tls_keystore_dir: ./certs - amq_streams_broker_tls_truststore_dir: ./certs - amq_streams_broker_tls_truststore_client_dir: ./certs - amq_streams_broker_tls_keystore_password: password - amq_streams_broker_tls_truststore_password: password - amq_streams_broker_tls_keystore_client_password: password - amq_streams_broker_tls_truststore_client_password: password - - # Broker Listeners - amq_streams_broker_listeners: - - PLAINTEXT://:{{ amq_streams_broker_listener_port }} # Insecure for inter-broker connections - - SSL://:{{ amq_streams_broker_listener_tls_port }} # Secure for client connections - - # Using SSL bootstrap server port - amq_streams_broker_bootstrap_server_host: localhost - amq_streams_broker_bootstrap_server_port: 9093 - - # Kafka SCRAM Users - amq_streams_broker_auth_scram_users: - - username: kafkascramuser01 - password: p@ssw0rd - - username: kafkascramuser02 - password: p@ssw0rd - - # Topic Management - amq_streams_broker_topics: - - name: sampleTopic - partitions: 1 - replication_factor: 1 - - name: otherTopic - partitions: 1 - replication_factor: 1 - - # Enabling SSL/TLS communications between Kafka Connect and Kafka Broker - amq_streams_connect_broker_tls_enabled: true - amq_streams_connect_broker_tls_truststore_client_password: password - - # Connection to SSL endpoint - amq_streams_connect_bootstrap_servers: localhost:9093 - roles: - - role: amq_streams_zookeeper - tasks: - - name: "Ensure Zookeeper is running and available." - ansible.builtin.include_role: - name: amq_streams_zookeeper - vars: - amq_streams_common_skip_download: true - - - name: "Ensure Broker is running and available." - ansible.builtin.include_role: - name: amq_streams_broker - vars: - amq_streams_common_skip_download: true - - - name: "Create topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/create.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - topic_partitions: "{{ topic.partitions }}" - topic_replication_factor: "{{ topic.replication_factor }}" - - - name: "Describe topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/describe.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - - - name: "Delete topics" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: topic/delete.yml - loop: "{{ amq_streams_broker_topics }}" - loop_control: - loop_var: topic - vars: - topic_name: "{{ topic.name }}" - - - name: "Create SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/create.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - user_password: "{{ user.password }}" - - - name: "Describe SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/describe.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - - - name: "Delete SCRAM users" - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: user-scram/delete.yml - loop: "{{ amq_streams_broker_auth_scram_users }}" - loop_control: - loop_var: user - vars: - user_username: "{{ user.username }}" - - - name: "Ensure AMQ Streams Connect is running and available." - ansible.builtin.include_role: - name: amq_streams_connect - vars: - amq_streams_common_skip_download: true - connectors: - - { name: "file", path: "connectors/file.yml" } - - post_tasks: - - name: "Display numbers of Zookeeper instances managed by Ansible." - ansible.builtin.debug: - msg: "Numbers of Zookeeper instances: {{ amq_streams_zookeeper_instance_count }}." - when: - - amq_streams_zookeeper_instance_count_enabled is defined and amq_streams_zookeeper_instance_count_enabled - - - name: "Display numbers of broker instances managed by Ansible:" - ansible.builtin.debug: - msg: "Numbers of broker instances: {{ amq_streams_broker_instance_count }}." - when: - - amq_streams_broker_instance_count_enabled is defined and amq_streams_broker_instance_count_enabled - - - name: "Validate that Zookeeper deployment is functional." - ansible.builtin.include_role: - name: amq_streams_zookeeper - tasks_from: validate.yml - - - name: "Validate that Broker deployment is functional." - ansible.builtin.include_role: - name: amq_streams_broker - tasks_from: validate.yml - - - name: "Validate that Connect deployment is functional." - ansible.builtin.include_role: - name: amq_streams_connect - tasks_from: validate.yml diff --git a/playbooks/brokers.yml b/playbooks/brokers.yml new file mode 100644 index 0000000..843d047 --- /dev/null +++ b/playbooks/brokers.yml @@ -0,0 +1,16 @@ +--- +- name: "Automate AMQ Streams install" + hosts: "{{ brokers_target_group | default('brokers') }}" + vars: + amq_streams_brokers_validate_playbook: True + collections: + - middleware_automation.amq_streams + roles: + - role: amq_streams_broker + post_tasks: + - name: "Validate that Broker deployment is functional." + ansible.builtin.include_role: + name: amq_streams_broker + tasks_from: validate.yml + when: + - amq_streams_brokers_validate_playbook is defined and amq_streams_brokers_validate_playbook diff --git a/playbooks/certs/server.truststore.jks b/playbooks/certs/server.truststore.jks deleted file mode 100644 index 68cc456..0000000 Binary files a/playbooks/certs/server.truststore.jks and /dev/null differ diff --git a/playbooks/zookeepers.yml b/playbooks/zookeepers.yml new file mode 100644 index 0000000..c9b6d6c --- /dev/null +++ b/playbooks/zookeepers.yml @@ -0,0 +1,16 @@ +--- +- name: "Automate AMQ Streams install" + hosts: "{{ zookeepers_target_group | default('zookeepers') }}" + vars: + amq_streams_zookeepers_validate_playbook: True + collections: + - middleware_automation.amq_streams + roles: + - role: amq_streams_zookeeper + post_tasks: + - name: "Validate that Broker deployment is functional." + ansible.builtin.include_role: + name: amq_streams_zookeeper + tasks_from: validate.yml + when: + - amq_streams_zookeepers_validate_playbook is defined and amq_streams_zookeepers_validate_playbook