generate_connector_versions

vdesabou · Jan 28, 2024 · 10777d6 · 10777d6
1 parent a0b8630
commit 10777d6
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 146 deletions.
diff --git a/scripts/cli/playground b/scripts/cli/playground
@@ -10857,6 +10857,44 @@ function load_env_variables () {
   done
 }
 
+function get_connector_paths () {
+    # determining the docker-compose file from from test_file
+    docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
+    test_file_directory="$(dirname "${test_file}")"
+    docker_compose_file="${test_file_directory}/${docker_compose_file}"
+
+    if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
+    then
+      logwarn "❌ skipping as docker-compose override file could not be detemined"
+      return
+    fi
+    connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+}
+
+function generate_connector_versions () {
+  get_connector_paths
+  if [ "$connector_paths" == "" ]
+  then
+      return
+  else
+      connector_tags=""
+      for connector_path in ${connector_paths//,/ }
+      do
+        full_connector_name=$(basename "$connector_path")
+        owner=$(echo "$full_connector_name" | cut -d'-' -f1)
+        name=$(echo "$full_connector_name" | cut -d'-' -f2-)
+
+        if [ "$owner" == "java" ] || [ "$name" == "hub-components" ] || [ "$owner" == "filestream" ]
+        then
+          # happens when plugin is not coming from confluent hub
+          continue
+        fi
+
+        playground connector-plugin versions --connector-plugin $owner/$name --force-refresh
+      done
+  fi
+}
+
 # src/lib/validations/validate_dir_exists.sh
 validate_dir_exists() {
   [[ -d "$1" ]] || logerror "<$1> must be an existing directory"
@@ -11462,9 +11500,10 @@ playground_run_command() {
 
   if [[ -n "$environment" ]]
   then
-    if [[ "$dir1" != *connect ]] && [ "$environment" != "plaintext" ]
+    get_connector_paths
+    if [ "$connector_paths" == "" ] && [ "$environment" != "plaintext" ]
     then
-      logerror "❌ using --environment is only supported for connector examples in connect folder (folder is $dir1)"
+      logerror "❌ using --environment is only supported with connector examples"
       exit 1
     fi
 
@@ -11479,21 +11518,10 @@ playground_run_command() {
   then
     if [ "$connector_tag" == " " ]
     then
-      # determining the docker-compose file from from test_file
-      docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-      test_file_directory="$(dirname "${test_file}")"
-      docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-      if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-      then
-          logwarn "❌ skipping as docker-compose override file could not be detemined"
-          exit 1
-      fi
-
-      connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+      get_connector_paths
       if [ "$connector_paths" == "" ]
       then
-          logwarn "❌ skipping as it is not an example with connector"
+          logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
           exit 1
       else
           connector_tags=""
@@ -11667,6 +11695,7 @@ playground_run_command() {
     trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground connector versions;playground open-docs --only-show-url' EXIT
   fi
   playground generate-fzf-find-files &
+  generate_connector_versions > /dev/null 2>&1 &
   touch /tmp/playground-run-command-used
   bash $filename ${other_args[*]}
   ret=$?
@@ -12049,21 +12078,10 @@ playground_run_ccloud_command() {
   then
     if [ "$connector_tag" == " " ]
     then
-      # determining the docker-compose file from from test_file
-      docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-      test_file_directory="$(dirname "${test_file}")"
-      docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-      if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-      then
-          logwarn "❌ skipping as docker-compose override file could not be detemined"
-          exit 1
-      fi
-
-      connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+      get_connector_paths
       if [ "$connector_paths" == "" ]
       then
-          logwarn "❌ skipping as it is not an example with connector"
+          logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
           exit 1
       else
           connector_tags=""
@@ -12290,6 +12308,7 @@ playground_run_ccloud_command() {
   trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground fully-managed-connector status;playground connector versions' EXIT
   touch /tmp/playground-run-command-used
   playground generate-fzf-find-files &
+  generate_connector_versions > /dev/null 2>&1 &
   bash $filename ${other_args[*]}
   ret=$?
   ELAPSED="took: $((($SECONDS / 60) % 60))min $(($SECONDS % 60))sec"
@@ -12415,25 +12434,14 @@ playground_update_version_command() {
       exit 1
   fi
 
-  # determining the docker-compose file from from test_file
-  docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-  test_file_directory="$(dirname "${test_file}")"
-  docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-  if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-  then
-      logwarn "Skipping as docker-compose override file could not be detemined"
-      exit 0
-  fi
-
   if [[ -n "$connector_tag" ]]
   then
     if [ "$connector_tag" == " " ]
     then
-      connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+      get_connector_paths
       if [ "$connector_paths" == "" ]
       then
-          logwarn "❌ skipping as it is not an example with connector"
+          logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
           exit 1
       else
           connector_tags=""
@@ -13512,11 +13520,10 @@ playground_repro_bootstrap_command() {
 
     { head -n $(($line-1)) $tmp_dir/tmp_file; cat $tmp_dir/build_custom_smt; tail -n +$line $tmp_dir/tmp_file; } > $repro_test_file
 
-    connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
     if [ "$connector_paths" == "" ]
     then
-      logerror "not a connector test"
-      exit 1
+        logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
+        exit 1
     else
       #  Loop on all connectors in CONNECT_PLUGIN_PATH and install custom SMT jar in lib folder
       my_array_connector_tag=($(echo $CONNECTOR_TAG | tr "," "\n"))
@@ -18553,23 +18560,11 @@ playground_connector_versions_command() {
       logerror "File $test_file retrieved from $root_folder/playground.ini does not exist!"
       exit 1
   fi
-
-  # determining the docker-compose file from from test_file
-  docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-  test_file_directory="$(dirname "${test_file}")"
-  docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-  if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-  then
-      logwarn "Skipping as docker-compose override file could not be detemined"
-      exit 0
-  fi
-
-  connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+  get_connector_paths
   if [ "$connector_paths" == "" ]
   then
-      logwarn "Skipping as it is not an example with connector"
-      exit 0
+      logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
+      exit 1
   else
       current_tag=$(docker inspect -f '{{.Config.Image}}' broker 2> /dev/null | cut -d ":" -f 2)
       log "🎯 Version currently used for confluent platform"
@@ -19507,22 +19502,11 @@ playground_connector_open_docs_command() {
       exit 1
   fi
 
-  # determining the docker-compose file from from test_file
-  docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-  test_file_directory="$(dirname "${test_file}")"
-  docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-  if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-  then
-      logwarn "Skipping as docker-compose override file could not be detemined"
-      exit 0
-  fi
-
-  connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+  get_connector_paths
   if [ "$connector_paths" == "" ]
   then
-      logwarn "Skipping as it is not an example with connector"
-      exit 0
+      logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
+      exit 1
   else
       for connector_path in ${connector_paths//,/ }
       do

diff --git a/scripts/cli/src/commands/connector/open-docs.sh b/scripts/cli/src/commands/connector/open-docs.sh
@@ -7,22 +7,11 @@ then
     exit 1
 fi
 
-# determining the docker-compose file from from test_file
-docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-test_file_directory="$(dirname "${test_file}")"
-docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-then
-    logwarn "Skipping as docker-compose override file could not be detemined"
-    exit 0
-fi
-
-connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+get_connector_paths
 if [ "$connector_paths" == "" ]
 then
-    logwarn "Skipping as it is not an example with connector"
-    exit 0
+    logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
+    exit 1
 else
     for connector_path in ${connector_paths//,/ }
     do

diff --git a/scripts/cli/src/commands/connector/versions.sh b/scripts/cli/src/commands/connector/versions.sh
@@ -5,23 +5,11 @@ then
     logerror "File $test_file retrieved from $root_folder/playground.ini does not exist!"
     exit 1
 fi
-
-# determining the docker-compose file from from test_file
-docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-test_file_directory="$(dirname "${test_file}")"
-docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-then
-    logwarn "Skipping as docker-compose override file could not be detemined"
-    exit 0
-fi
-
-connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+get_connector_paths
 if [ "$connector_paths" == "" ]
 then
-    logwarn "Skipping as it is not an example with connector"
-    exit 0
+    logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
+    exit 1
 else
     current_tag=$(docker inspect -f '{{.Config.Image}}' broker 2> /dev/null | cut -d ":" -f 2)
     log "🎯 Version currently used for confluent platform"

diff --git a/scripts/cli/src/commands/repro/bootstrap.sh b/scripts/cli/src/commands/repro/bootstrap.sh
@@ -707,11 +707,10 @@ then
   { head -n $(($line-1)) $tmp_dir/tmp_file; cat $tmp_dir/build_custom_smt; tail -n +$line $tmp_dir/tmp_file; } > $repro_test_file
 
 
-  connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
   if [ "$connector_paths" == "" ]
   then
-    logerror "not a connector test"
-    exit 1
+      logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
+      exit 1
   else
     ###
     #  Loop on all connectors in CONNECT_PLUGIN_PATH and install custom SMT jar in lib folder

diff --git a/scripts/cli/src/commands/run-ccloud.sh b/scripts/cli/src/commands/run-ccloud.sh
@@ -57,21 +57,10 @@ if [[ -n "$connector_tag" ]]
 then
   if [ "$connector_tag" == " " ]
   then
-    # determining the docker-compose file from from test_file
-    docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-    test_file_directory="$(dirname "${test_file}")"
-    docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-    if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-    then
-        logwarn "❌ skipping as docker-compose override file could not be detemined"
-        exit 1
-    fi
-
-    connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+    get_connector_paths
     if [ "$connector_paths" == "" ]
     then
-        logwarn "❌ skipping as it is not an example with connector"
+        logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
         exit 1
     else
         connector_tags=""
@@ -296,6 +285,7 @@ cd $test_file_directory
 trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground fully-managed-connector status;playground connector versions' EXIT
 touch /tmp/playground-run-command-used
 playground generate-fzf-find-files &
+generate_connector_versions > /dev/null 2>&1 &
 bash $filename ${other_args[*]}
 ret=$?
 ELAPSED="took: $((($SECONDS / 60) % 60))min $(($SECONDS % 60))sec"

diff --git a/scripts/cli/src/commands/run.sh b/scripts/cli/src/commands/run.sh
@@ -56,9 +56,10 @@ fi
 
 if [[ -n "$environment" ]]
 then
-  if [[ "$dir1" != *connect ]] && [ "$environment" != "plaintext" ]
+  get_connector_paths
+  if [ "$connector_paths" == "" ] && [ "$environment" != "plaintext" ]
   then
-    logerror "❌ using --environment is only supported for connector examples in connect folder (folder is $dir1)"
+    logerror "❌ using --environment is only supported with connector examples"
     exit 1
   fi
 
@@ -73,21 +74,10 @@ if [[ -n "$connector_tag" ]]
 then
   if [ "$connector_tag" == " " ]
   then
-    # determining the docker-compose file from from test_file
-    docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-    test_file_directory="$(dirname "${test_file}")"
-    docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-    if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-    then
-        logwarn "❌ skipping as docker-compose override file could not be detemined"
-        exit 1
-    fi
-
-    connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+    get_connector_paths
     if [ "$connector_paths" == "" ]
     then
-        logwarn "❌ skipping as it is not an example with connector"
+        logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
         exit 1
     else
         connector_tags=""
@@ -261,6 +251,7 @@ else
   trap 'rm /tmp/playground-run-command-used;echo "";sleep 3;set +e;playground connector status;playground connector versions;playground open-docs --only-show-url' EXIT
 fi
 playground generate-fzf-find-files &
+generate_connector_versions > /dev/null 2>&1 &
 touch /tmp/playground-run-command-used
 bash $filename ${other_args[*]}
 ret=$?

diff --git a/scripts/cli/src/commands/update-version.sh b/scripts/cli/src/commands/update-version.sh
@@ -38,25 +38,14 @@ then
     exit 1
 fi
 
-# determining the docker-compose file from from test_file
-docker_compose_file=$(grep "start-environment" "$test_file" |  awk '{print $6}' | cut -d "/" -f 2 | cut -d '"' -f 1 | tail -n1 | xargs)
-test_file_directory="$(dirname "${test_file}")"
-docker_compose_file="${test_file_directory}/${docker_compose_file}"
-
-if [ "${docker_compose_file}" != "" ] && [ ! -f "${docker_compose_file}" ]
-then
-    logwarn "Skipping as docker-compose override file could not be detemined"
-    exit 0
-fi
-
 if [[ -n "$connector_tag" ]]
 then
   if [ "$connector_tag" == " " ]
   then
-    connector_paths=$(grep "CONNECT_PLUGIN_PATH" "${docker_compose_file}" | grep -v "KSQL_CONNECT_PLUGIN_PATH" | cut -d ":" -f 2  | tr -s " " | head -1)
+    get_connector_paths
     if [ "$connector_paths" == "" ]
     then
-        logwarn "❌ skipping as it is not an example with connector"
+        logwarn "❌ skipping as it is not an example with connector, but --connector-tag is set"
         exit 1
     else
         connector_tags=""