diff --git a/addons/kafka/scripts-ut-spec/kafka_server_setup_spec.sh b/addons/kafka/scripts-ut-spec/kafka_server_setup_spec.sh new file mode 100644 index 000000000..7f0186276 --- /dev/null +++ b/addons/kafka/scripts-ut-spec/kafka_server_setup_spec.sh @@ -0,0 +1,197 @@ +# shellcheck shell=bash +# shellcheck disable=SC2034 + +# Validate the expected shell type and version this script needs to run. +if ! validate_shell_type_and_version "bash" 4 &>/dev/null; then + echo "kafka_server_setup_spec.sh skip cases because dependency bash version 4 or higher is not installed." + exit 0 +fi + +source ./utils.sh + +# The unit test needs to rely on the common library functions defined in kblib. +# Therefore, we first dynamically generate the required common library files from the kblib library chart. +common_library_file="./common.sh" +generate_common_library $common_library_file + +Describe "Kafka Server Setup Script Tests" + # Load the scripts to be tested and dependencies + Include ../scripts/kafka-server-setup.sh + Include $common_library_file + + init() { + ut_mode="true" + mock_tls_cert_path="./certs" + kafka_config_certs_path="./certs" + kafka_kraft_config_path="./kraft" + kafka_config_path="./config" + mkdir -p $mock_tls_cert_path + mkdir -p $kafka_config_certs_path + mkdir -p $kafka_kraft_config_path + mkdir -p $kafka_config_path + } + BeforeAll "init" + + cleanup() { + rm -f $common_library_file; + rm -fr $mock_tls_cert_path; + rm -fr $kafka_config_certs_path; + rm -fr $kafka_kraft_config_path; + rm -fr $kafka_config_path; + } + AfterAll 'cleanup' + + un_setup() { + # Reset environment variables before each test + unset TLS_ENABLED + unset TLS_CERT_PATH + unset SERVER_PROP_PATH + unset SERVER_PROP_FILE + unset KAFKA_CFG_PROCESS_ROLES + unset BROKER_ADVERTISED_PORT + unset MY_POD_NAME + unset POD_FQDN_LIST + unset CONTROLLER_POD_NAME_LIST + unset KB_HOST_IP + unset BROKER_MIN_NODE_ID + unset KB_KAFKA_ENABLE_SASL + unset KB_KAFKA_SASL_CONFIG_PATH + unset KAFKA_KRAFT_CLUSTER_ID + unset KB_KAFKA_BROKER_HEAP + unset KB_KAFKA_CONTROLLER_HEAP + } + + Describe "set_tls_configuration_if_needed()" + It "skips TLS configuration if TLS_ENABLED or TLS_CERT_PATH is not set" + un_setup + When run set_tls_configuration_if_needed + The output should include "TLS_ENABLED or TLS_CERT_PATH is not set, skipping TLS configuration" + The status should be success + End + + It "returns error if TLS_CERT_PATH is set but PEM files are missing" + un_setup + TLS_ENABLED="true" + TLS_CERT_PATH="$mock_tls_cert_path" + When run set_tls_configuration_if_needed + The stderr should include "[tls]Couldn't find the expected PEM files! They are mandatory when encryption via TLS is enabled." + The stdout should include "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,CLIENT:SSL" + The status should be failure + End + + It "successfully sets TLS configuration when all required variables are set" + un_setup + TLS_ENABLED="true" + TLS_CERT_PATH="$mock_tls_cert_path" + # Create mock PEM files for testing + mkdir -p $mock_tls_cert_path + touch $mock_tls_cert_path/ca.crt + touch $mock_tls_cert_path/tls.crt + touch $mock_tls_cert_path/tls.key + When run set_tls_configuration_if_needed + The output should include "[tls]KAFKA_TLS_TRUSTSTORE_FILE=$mock_tls_cert_path/kafka.truststore.pem" + # hack the error from openssl + The stderr should include "Could not find private key of key from" + The status should be success + End + + It "fails if TLS_CERT_PATH is set but ca.crt is missing" + un_setup + rm -f $mock_tls_cert_path/ca.crt + TLS_ENABLED="true" + TLS_CERT_PATH="$mock_tls_cert_path" + mkdir -p $mock_tls_cert_path + touch $mock_tls_cert_path/tls.crt + touch $mock_tls_cert_path/tls.key + When run set_tls_configuration_if_needed + The stderr should include "[tls]PEM_CA not provided, and auth.tls.pemChainIncluded was not true." + The stdout should include "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,CLIENT:SSL" + The status should be failure + End + End + + Describe "convert_server_properties_to_env_var()" + It "skips conversion if the server properties file does not exist" + un_setup + SERVER_PROP_FILE="non_existent_file.properties" + When run convert_server_properties_to_env_var + The status should be success + End + + It "successfully converts properties to environment variables" + un_setup + SERVER_PROP_FILE="$kafka_config_path/server.properties" + echo -e "broker.id=0\nlisteners=PLAINTEXT://:9092\n" > "$SERVER_PROP_FILE" + When run convert_server_properties_to_env_var + The output should include "[cfg]export KAFKA_CFG_BROKER_ID=0" + The output should include "[cfg]export KAFKA_CFG_LISTENERS=PLAINTEXT://:9092" + The status should be success + End + + It "handles properties with no value gracefully" + un_setup + SERVER_PROP_FILE="$kafka_config_path/server.properties" + echo -e "broker.id=0\nlisteners=\n" > "$SERVER_PROP_FILE" + When run convert_server_properties_to_env_var + The output should include "line 'listeners' has no value; skipped" + The status should be success + End + + It "ignores commented lines" + un_setup + SERVER_PROP_FILE="$kafka_config_path/server.properties" + echo -e "# This is a comment\nbroker.id=0\n# listeners=PLAINTEXT://:9092\n" > "$SERVER_PROP_FILE" + When run convert_server_properties_to_env_var + The output should include "[cfg]export KAFKA_CFG_BROKER_ID=0" + The output should not include "listeners" + The status should be success + End + End + + Describe "override_sasl_configuration()" + It "sets SASL configuration when KB_KAFKA_ENABLE_SASL is true" + un_setup + KB_KAFKA_ENABLE_SASL="true" + KB_KAFKA_SASL_CONFIG_PATH="$kafka_config_path/kafka_jaas.conf" + touch "$KB_KAFKA_SASL_CONFIG_PATH" + When run override_sasl_configuration + The output should include "[sasl]KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,CLIENT:SASL_PLAINTEXT" + # mock the error from cp + The stderr should include "cp: ./config/kafka_jaas.conf and ./config/kafka_jaas.conf are identical" + The status should be success + End + + It "does not set SASL configuration when KB_KAFKA_ENABLE_SASL is false" + un_setup + KB_KAFKA_ENABLE_SASL="false" + When run override_sasl_configuration + The output should not include "KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP" + The status should be success + End + End + + Describe "generate_kraft_cluster_id()" + It "sets KAFKA_KRAFT_CLUSTER_ID if provided" + un_setup + KAFKA_KRAFT_CLUSTER_ID="my-cluster-id" + When run generate_kraft_cluster_id + The output should include "KAFKA_KRAFT_CLUSTER_ID=my-cluster-id" + The status should be success + End + + It "truncates KAFKA_KRAFT_CLUSTER_ID to 22 characters if too long" + un_setup + KAFKA_KRAFT_CLUSTER_ID="this-is-a-very-long-cluster-id-that-exceeds-length" + When run generate_kraft_cluster_id + The output should include "export KAFKA_KRAFT_CLUSTER_ID=this-is-a-very-long-cl" + The status should be success + End + + It "does not set KAFKA_KRAFT_CLUSTER_ID if not provided" + un_setup + When run generate_kraft_cluster_id + The output should not include "KAFKA_KRAFT_CLUSTER_ID" + The status should be success + End + End +End \ No newline at end of file diff --git a/addons/kafka/scripts-ut-spec/utils.sh b/addons/kafka/scripts-ut-spec/utils.sh new file mode 100644 index 000000000..5b2506969 --- /dev/null +++ b/addons/kafka/scripts-ut-spec/utils.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# utils functions for shellspec unit tests + +convert_tpl_to_bash() { + local input_file="$1" + local output_file="$2" + + sed -e '/^{{\/\*$/,/^\*\/}}$/d' \ + -e '/^{{-.*}}/d' \ + -e 's/{{- define ".*" }}//' \ + -e 's/{{- end }}//' \ + "$input_file" >> "$output_file" +} + +generate_common_library() { + local library_file="$1" + + libcommons_tpl_file="../../kblib/templates/_libcommons.tpl" + libpods_tpl_file="../../kblib/templates/_libpods.tpl" + libstrings_tpl_file="../../kblib/templates/_libstrings.tpl" + libenvs_tpl_file="../../kblib/templates/_libenvs.tpl" + libcompvars_tpl_file="../../kblib/templates/_libcompvars.tpl" + libututils_tpl_file="../../kblib/templates/_libututils.tpl" + + convert_tpl_to_bash $libcommons_tpl_file "$library_file" + convert_tpl_to_bash $libpods_tpl_file "$library_file" + convert_tpl_to_bash $libstrings_tpl_file "$library_file" + convert_tpl_to_bash $libenvs_tpl_file "$library_file" + convert_tpl_to_bash $libcompvars_tpl_file "$library_file" + convert_tpl_to_bash $libututils_tpl_file "$library_file" +} \ No newline at end of file diff --git a/addons/kafka/scripts/kafka-server-setup.sh b/addons/kafka/scripts/kafka-server-setup.sh index a725e7618..8c2801c39 100644 --- a/addons/kafka/scripts/kafka-server-setup.sh +++ b/addons/kafka/scripts/kafka-server-setup.sh @@ -1,5 +1,9 @@ #!/bin/bash +kafka_config_certs_path="/opt/bitnami/kafka/config/certs" +kafka_kraft_config_path="/opt/bitnami/kafka/config/kraft" +kafka_config_path="/opt/bitnami/kafka/config" + # shellcheck disable=SC2153 # shellcheck disable=SC2034 ut_mode="false" @@ -41,12 +45,12 @@ set_tls_configuration_if_needed() { #export KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL=SSL #echo "KAFKA_CFG_SECURITY_INTER_BROKER_PROTOCOL=SSL" - mkdir -p /opt/bitnami/kafka/config/certs + mkdir -p "$kafka_config_certs_path" PEM_CA="$TLS_CERT_PATH/ca.crt" PEM_CERT="$TLS_CERT_PATH/tls.crt" PEM_KEY="$TLS_CERT_PATH/tls.key" if [[ -f "$PEM_CERT" ]] && [[ -f "$PEM_KEY" ]]; then - CERT_DIR="/opt/bitnami/kafka/config/certs" + CERT_DIR="$kafka_config_certs_path" PEM_CA_LOCATION="${CERT_DIR}/kafka.truststore.pem" PEM_CERT_LOCATION="${CERT_DIR}/kafka.keystore.pem" if [[ -f "$PEM_CA" ]]; then @@ -65,10 +69,10 @@ set_tls_configuration_if_needed() { echo "[tls]Couldn't find the expected PEM files! They are mandatory when encryption via TLS is enabled." >&2 return 1 fi - export KAFKA_TLS_TRUSTSTORE_FILE="/opt/bitnami/kafka/config/certs/kafka.truststore.pem" + export KAFKA_TLS_TRUSTSTORE_FILE="$kafka_config_certs_path/kafka.truststore.pem" echo "[tls]KAFKA_TLS_TRUSTSTORE_FILE=$KAFKA_TLS_TRUSTSTORE_FILE" - echo "[tls]ssl.endpoint.identification.algorithm=" >> /opt/bitnami/kafka/config/kraft/server.properties - echo "[tls]ssl.endpoint.identification.algorithm=" >> /opt/bitnami/kafka/config/server.properties + echo "[tls]ssl.endpoint.identification.algorithm=" >> $kafka_kraft_config_path/server.properties + echo "[tls]ssl.endpoint.identification.algorithm=" >> $kafka_config_path/server.properties return 0 } @@ -107,8 +111,8 @@ override_sasl_configuration() { if [[ "true" == "$KB_KAFKA_ENABLE_SASL" ]]; then # bitnami default jaas setting: /opt/bitnami/kafka/config/kafka_jaas.conf if [[ "${KB_KAFKA_SASL_CONFIG_PATH}" ]]; then - cp ${KB_KAFKA_SASL_CONFIG_PATH} /opt/bitnami/kafka/config/kafka_jaas.conf - echo "[sasl]do: cp ${KB_KAFKA_SASL_CONFIG_PATH} /opt/bitnami/kafka/config/kafka_jaas.conf " + cp ${KB_KAFKA_SASL_CONFIG_PATH} $kafka_config_path/kafka_jaas.conf + echo "[sasl]do: cp ${KB_KAFKA_SASL_CONFIG_PATH} $kafka_config_path/kafka_jaas.conf " fi export KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,INTERNAL:SASL_PLAINTEXT,CLIENT:SASL_PLAINTEXT echo "[sasl]KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=$KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP" @@ -123,7 +127,7 @@ generate_kraft_cluster_id() { if [[ -n "$KAFKA_KRAFT_CLUSTER_ID" ]]; then echo KAFKA_KRAFT_CLUSTER_ID="${KAFKA_KRAFT_CLUSTER_ID}" kraft_id_len=${#KAFKA_KRAFT_CLUSTER_ID} - if [[ kraft_id_len > 22 ]]; then + if [[ kraft_id_len -gt 22 ]]; then export KAFKA_KRAFT_CLUSTER_ID=$(echo $KAFKA_KRAFT_CLUSTER_ID | cut -b 1-22) echo export KAFKA_KRAFT_CLUSTER_ID="${KAFKA_KRAFT_CLUSTER_ID}" fi