Adding retry to ssh

Signed-off-by: Joe Talerico aka rook <[email protected]>

cmd/k8s-netperf/k8s-netperf.go

@@ -219,7 +219,11 @@ var rootCmd = &cobra.Command{
 				}
 			}
 		} else {
-			k8s.SSHConnect(&s)
+			log.Info("Connecting via ssh to the VMI")
+			err = k8s.SSHConnect(&s)
+			if err != nil {
+				log.Fatal(err)
+			}
 		}
 
 		if pavail {

pkg/k8s/kubernetes.go

@@ -311,10 +311,11 @@ func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error {
 					return err
 				}
 			} else {
-				_, err := CreateVMClient(s.KClient, client, s.DClient, clientAcrossRole, clientAcrossRole, &cdpHostAcross.PodAntiAffinity, &cdpHostAcross.NodeAffinity)
+				host, err := CreateVMClient(s.KClient, client, s.DClient, clientAcrossRole, clientAcrossRole, &cdpHostAcross.PodAntiAffinity, &cdpHostAcross.NodeAffinity)
 				if err != nil {
 					return err
 				}
+				s.VMHost = host
 				WaitForVMI(s.KClient, clientAcrossRole)
 			}
 		}
@@ -324,10 +325,11 @@ func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error {
 				return err
 			}
 		} else {
-			_, err := CreateVMClient(s.KClient, client, s.DClient, clientAcrossRole, clientAcrossRole, &cdpAcross.PodAntiAffinity, &cdpHostAcross.NodeAffinity)
+			host, err := CreateVMClient(s.KClient, client, s.DClient, clientAcrossRole, clientAcrossRole, &cdpAcross.PodAntiAffinity, &cdpHostAcross.NodeAffinity)
 			if err != nil {
 				return err
 			}
+			s.VMHost = host
 			WaitForVMI(s.KClient, clientAcrossRole)
 		}
 

pkg/k8s/kubevirt.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	b64 "encoding/base64"
 
@@ -26,29 +27,50 @@ import (
 
 var (
 	sshPort = uint(32022)
+	retry   = 30
 )
 
+func connect(config *goph.Config) (*goph.Client, error) {
+	for i := 0; i < retry; i++ {
+		client, err := goph.NewConn(config)
+		if err != nil {
+			log.Debug("Waiting for ssh access to be available")
+			log.Debug(err)
+			time.Sleep(10 * time.Second)
+			continue
+		} else {
+			return client, nil
+		}
+	}
+	return nil, fmt.Errorf("Unable to connect via ssh after %d attempts", retry)
+}
+
 func SSHConnect(conf *config.PerfScenarios) error {
-	dirname, err := os.UserHomeDir()
+	dir, err := os.UserHomeDir()
 	if err != nil {
-		return err
+		return fmt.Errorf("Unable to retrieve users homedir. %s", err)
 	}
-	auth, err := goph.Key(fmt.Sprintf("%s/.ssh/id_rsa.pub", dirname), "")
+	key := fmt.Sprintf("%s/.ssh/id_rsa", dir)
+	keyd, err := os.ReadFile(key)
+	auth, err := goph.RawKey(string(keyd), "")
 	if err != nil {
-		log.Fatal(err)
+		return fmt.Errorf("Unable to retrieve sshkey. Error : %s", err)
 	}
 	user := "fedora"
 	addr := conf.VMHost
+	log.Infof("Attempting to connect with : %s@%s", user, addr)
 
-	client, err := goph.NewConn(&goph.Config{
+	config := goph.Config{
 		User:     user,
 		Addr:     addr,
 		Port:     sshPort,
 		Auth:     auth,
 		Callback: ssh.InsecureIgnoreHostKey(),
-	})
+	}
+
+	client, err := connect(&config)
 	if err != nil {
-		return err
+		return fmt.Errorf("Unable to connect via ssh. Error: %s", err)
 	}
 	defer client.Close()
 	out, err := client.Run("ls /etc")