diff --git a/scripts/cli/playground b/scripts/cli/playground index 89b7d2e593..7016d2b36c 100755 --- a/scripts/cli/playground +++ b/scripts/cli/playground @@ -10857,6 +10857,44 @@ function load_env_variables () { done } +function get_connector_paths () { + # determining the docker-compose file from from test_file + docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) + test_file_directory="$(dirname "${test_file}")" + docker_compose_file="${test_file_directory}/${docker_compose_file}" + + if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] + then + logwarn "❌ skipping as docker-compose override file could not be detemined" + return + fi + connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) +} + +function generate_connector_versions () { + get_connector_paths + if [ "$connector_paths" == "" ] + then + return + else + connector_tags="" + for connector_path in ${connector_paths//,/ } + do + full_connector_name=$(basename "$connector_path") + owner=$(echo "$full_connector_name" | cut -d'-' -f1) + name=$(echo "$full_connector_name" | cut -d'-' -f2-) + + if [ "$owner" == "java" ] || [ "$name" == "hub-components" ] || [ "$owner" == "filestream" ] + then + # happens when plugin is not coming from confluent hub + continue + fi + + playground connector-plugin versions --connector-plugin $owner/$name --force-refresh + done + fi +} + # src/lib/validations/validate_dir_exists.sh validate_dir_exists() { [[ -d "$1" ]] || logerror "<$1> must be an existing directory" @@ -11462,9 +11500,10 @@ playground_run_command() { if [[ -n "$environment" ]] then - if [[ "$dir1" != *connect ]] && [ "$environment" != "plaintext" ] + get_connector_paths + if [ "$connector_paths" == "" ] && [ "$environment" != "plaintext" ] then - logerror "❌ using --environment is only supported for connector examples in connect folder (folder is $dir1)" + logerror "❌ using --environment is only supported with connector examples" exit 1 fi @@ -11479,21 +11518,10 @@ playground_run_command() { then if [ "$connector_tag" == " " ] then - # determining the docker-compose file from from test_file - docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) - test_file_directory="$(dirname "${test_file}")" - docker_compose_file="${test_file_directory}/${docker_compose_file}" - - if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] - then - logwarn "❌ skipping as docker-compose override file could not be detemined" - exit 1 - fi - - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "❌ skipping as it is not an example with connector" + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" exit 1 else connector_tags="" @@ -11667,6 +11695,7 @@ playground_run_command() { trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground connector versions;playground open-docs --only-show-url' EXIT fi playground generate-fzf-find-files & + generate_connector_versions > /dev/null 2>&1 & touch /tmp/playground-run-command-used bash $filename ${other_args[*]} ret=$? @@ -12049,21 +12078,10 @@ playground_run_ccloud_command() { then if [ "$connector_tag" == " " ] then - # determining the docker-compose file from from test_file - docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) - test_file_directory="$(dirname "${test_file}")" - docker_compose_file="${test_file_directory}/${docker_compose_file}" - - if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] - then - logwarn "❌ skipping as docker-compose override file could not be detemined" - exit 1 - fi - - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "❌ skipping as it is not an example with connector" + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" exit 1 else connector_tags="" @@ -12290,6 +12308,7 @@ playground_run_ccloud_command() { trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground fully-managed-connector status;playground connector versions' EXIT touch /tmp/playground-run-command-used playground generate-fzf-find-files & + generate_connector_versions > /dev/null 2>&1 & bash $filename ${other_args[*]} ret=$? ELAPSED="took: $((($SECONDS / 60) % 60))min $(($SECONDS % 60))sec" @@ -12415,25 +12434,14 @@ playground_update_version_command() { exit 1 fi - # determining the docker-compose file from from test_file - docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) - test_file_directory="$(dirname "${test_file}")" - docker_compose_file="${test_file_directory}/${docker_compose_file}" - - if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] - then - logwarn "Skipping as docker-compose override file could not be detemined" - exit 0 - fi - if [[ -n "$connector_tag" ]] then if [ "$connector_tag" == " " ] then - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "❌ skipping as it is not an example with connector" + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" exit 1 else connector_tags="" @@ -13512,11 +13520,10 @@ playground_repro_bootstrap_command() { { head -n $(($line-1)) $tmp_dir/tmp_file; cat $tmp_dir/build_custom_smt; tail -n +$line $tmp_dir/tmp_file; } > $repro_test_file - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) if [ "$connector_paths" == "" ] then - logerror "not a connector test" - exit 1 + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" + exit 1 else # Loop on all connectors in CONNECT_PLUGIN_PATH and install custom SMT jar in lib folder my_array_connector_tag=($(echo $CONNECTOR_TAG | tr "," "\n")) @@ -18553,23 +18560,11 @@ playground_connector_versions_command() { logerror "File $test_file retrieved from $root_folder/playground.ini does not exist!" exit 1 fi - - # determining the docker-compose file from from test_file - docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) - test_file_directory="$(dirname "${test_file}")" - docker_compose_file="${test_file_directory}/${docker_compose_file}" - - if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] - then - logwarn "Skipping as docker-compose override file could not be detemined" - exit 0 - fi - - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "Skipping as it is not an example with connector" - exit 0 + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" + exit 1 else current_tag=$(docker inspect -f '{{.Config.Image}}' broker 2> /dev/null | cut -d ":" -f 2) log "🎯 Version currently used for confluent platform" @@ -19507,22 +19502,11 @@ playground_connector_open_docs_command() { exit 1 fi - # determining the docker-compose file from from test_file - docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) - test_file_directory="$(dirname "${test_file}")" - docker_compose_file="${test_file_directory}/${docker_compose_file}" - - if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] - then - logwarn "Skipping as docker-compose override file could not be detemined" - exit 0 - fi - - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "Skipping as it is not an example with connector" - exit 0 + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" + exit 1 else for connector_path in ${connector_paths//,/ } do diff --git a/scripts/cli/src/commands/connector/open-docs.sh b/scripts/cli/src/commands/connector/open-docs.sh index 6daf07965c..ffb052ce4b 100644 --- a/scripts/cli/src/commands/connector/open-docs.sh +++ b/scripts/cli/src/commands/connector/open-docs.sh @@ -7,22 +7,11 @@ then exit 1 fi -# determining the docker-compose file from from test_file -docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) -test_file_directory="$(dirname "${test_file}")" -docker_compose_file="${test_file_directory}/${docker_compose_file}" - -if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] -then - logwarn "Skipping as docker-compose override file could not be detemined" - exit 0 -fi - -connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) +get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "Skipping as it is not an example with connector" - exit 0 + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" + exit 1 else for connector_path in ${connector_paths//,/ } do diff --git a/scripts/cli/src/commands/connector/versions.sh b/scripts/cli/src/commands/connector/versions.sh index 86a364aabb..fb77d1adc8 100644 --- a/scripts/cli/src/commands/connector/versions.sh +++ b/scripts/cli/src/commands/connector/versions.sh @@ -5,23 +5,11 @@ then logerror "File $test_file retrieved from $root_folder/playground.ini does not exist!" exit 1 fi - -# determining the docker-compose file from from test_file -docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) -test_file_directory="$(dirname "${test_file}")" -docker_compose_file="${test_file_directory}/${docker_compose_file}" - -if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] -then - logwarn "Skipping as docker-compose override file could not be detemined" - exit 0 -fi - -connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) +get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "Skipping as it is not an example with connector" - exit 0 + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" + exit 1 else current_tag=$(docker inspect -f '{{.Config.Image}}' broker 2> /dev/null | cut -d ":" -f 2) log "🎯 Version currently used for confluent platform" diff --git a/scripts/cli/src/commands/repro/bootstrap.sh b/scripts/cli/src/commands/repro/bootstrap.sh index c94d2afe03..c0fc20c903 100644 --- a/scripts/cli/src/commands/repro/bootstrap.sh +++ b/scripts/cli/src/commands/repro/bootstrap.sh @@ -707,11 +707,10 @@ then { head -n $(($line-1)) $tmp_dir/tmp_file; cat $tmp_dir/build_custom_smt; tail -n +$line $tmp_dir/tmp_file; } > $repro_test_file - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) if [ "$connector_paths" == "" ] then - logerror "not a connector test" - exit 1 + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" + exit 1 else ### # Loop on all connectors in CONNECT_PLUGIN_PATH and install custom SMT jar in lib folder diff --git a/scripts/cli/src/commands/run-ccloud.sh b/scripts/cli/src/commands/run-ccloud.sh index 626b4bfc28..0a97263e8d 100644 --- a/scripts/cli/src/commands/run-ccloud.sh +++ b/scripts/cli/src/commands/run-ccloud.sh @@ -57,21 +57,10 @@ if [[ -n "$connector_tag" ]] then if [ "$connector_tag" == " " ] then - # determining the docker-compose file from from test_file - docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) - test_file_directory="$(dirname "${test_file}")" - docker_compose_file="${test_file_directory}/${docker_compose_file}" - - if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] - then - logwarn "❌ skipping as docker-compose override file could not be detemined" - exit 1 - fi - - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "❌ skipping as it is not an example with connector" + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" exit 1 else connector_tags="" @@ -296,6 +285,7 @@ cd $test_file_directory trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground fully-managed-connector status;playground connector versions' EXIT touch /tmp/playground-run-command-used playground generate-fzf-find-files & +generate_connector_versions > /dev/null 2>&1 & bash $filename ${other_args[*]} ret=$? ELAPSED="took: $((($SECONDS / 60) % 60))min $(($SECONDS % 60))sec" diff --git a/scripts/cli/src/commands/run.sh b/scripts/cli/src/commands/run.sh index 86873991e7..3d9c0eab4d 100644 --- a/scripts/cli/src/commands/run.sh +++ b/scripts/cli/src/commands/run.sh @@ -56,9 +56,10 @@ fi if [[ -n "$environment" ]] then - if [[ "$dir1" != *connect ]] && [ "$environment" != "plaintext" ] + get_connector_paths + if [ "$connector_paths" == "" ] && [ "$environment" != "plaintext" ] then - logerror "❌ using --environment is only supported for connector examples in connect folder (folder is $dir1)" + logerror "❌ using --environment is only supported with connector examples" exit 1 fi @@ -73,21 +74,10 @@ if [[ -n "$connector_tag" ]] then if [ "$connector_tag" == " " ] then - # determining the docker-compose file from from test_file - docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) - test_file_directory="$(dirname "${test_file}")" - docker_compose_file="${test_file_directory}/${docker_compose_file}" - - if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] - then - logwarn "❌ skipping as docker-compose override file could not be detemined" - exit 1 - fi - - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "❌ skipping as it is not an example with connector" + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" exit 1 else connector_tags="" @@ -261,6 +251,7 @@ else trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground connector versions;playground open-docs --only-show-url' EXIT fi playground generate-fzf-find-files & +generate_connector_versions > /dev/null 2>&1 & touch /tmp/playground-run-command-used bash $filename ${other_args[*]} ret=$? diff --git a/scripts/cli/src/commands/update-version.sh b/scripts/cli/src/commands/update-version.sh index c32b7dbf9b..6fcc6bd77a 100644 --- a/scripts/cli/src/commands/update-version.sh +++ b/scripts/cli/src/commands/update-version.sh @@ -38,25 +38,14 @@ then exit 1 fi -# determining the docker-compose file from from test_file -docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) -test_file_directory="$(dirname "${test_file}")" -docker_compose_file="${test_file_directory}/${docker_compose_file}" - -if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] -then - logwarn "Skipping as docker-compose override file could not be detemined" - exit 0 -fi - if [[ -n "$connector_tag" ]] then if [ "$connector_tag" == " " ] then - connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) + get_connector_paths if [ "$connector_paths" == "" ] then - logwarn "❌ skipping as it is not an example with connector" + logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set" exit 1 else connector_tags="" diff --git a/scripts/cli/src/lib/utils_function.sh b/scripts/cli/src/lib/utils_function.sh index 20bf31b5b4..5262214749 100644 --- a/scripts/cli/src/lib/utils_function.sh +++ b/scripts/cli/src/lib/utils_function.sh @@ -3703,3 +3703,41 @@ function load_env_variables () { fi done } + +function get_connector_paths () { + # determining the docker-compose file from from test_file + docker_compose_file=$(grep "start-environment" "$test_file" | awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs) + test_file_directory="$(dirname "${test_file}")" + docker_compose_file="${test_file_directory}/${docker_compose_file}" + + if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ] + then + logwarn "❌ skipping as docker-compose override file could not be detemined" + return + fi + connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2 | tr -s " " | head -1) +} + +function generate_connector_versions () { + get_connector_paths + if [ "$connector_paths" == "" ] + then + return + else + connector_tags="" + for connector_path in ${connector_paths//,/ } + do + full_connector_name=$(basename "$connector_path") + owner=$(echo "$full_connector_name" | cut -d'-' -f1) + name=$(echo "$full_connector_name" | cut -d'-' -f2-) + + if [ "$owner" == "java" ] || [ "$name" == "hub-components" ] || [ "$owner" == "filestream" ] + then + # happens when plugin is not coming from confluent hub + continue + fi + + playground connector-plugin versions --connector-plugin $owner/$name --force-refresh + done + fi +} \ No newline at end of file