From 5674c5219c4c86b15dd7cd404e421f8ac88457b1 Mon Sep 17 00:00:00 2001
From: Robert Hecht <robert.hecht@staffbase.com>
Date: Tue, 26 Sep 2023 17:27:37 +0200
Subject: [PATCH] retry should include creation of exec.Command

---
 pkg/spark/spark.go | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/pkg/spark/spark.go b/pkg/spark/spark.go
index 0a2dff1..0a8cd51 100644
--- a/pkg/spark/spark.go
+++ b/pkg/spark/spark.go
@@ -135,20 +135,22 @@ func (s *Spark) Submit(presetName string) error {
 	if err != nil {
 		return fmt.Errorf("couldn't build submit args, %w", err)
 	}
-
 	zap.L().Info("submit with args", zap.Any("args", args))
-	cmd := exec.Command(s.binaryPath, args...)
-	zap.L().Info("spark-submit", zap.Strings("args", args))
-	if s.debug {
-		writer := &zapio.Writer{Log: zap.L(), Level: zap.DebugLevel}
-		cmd.Stderr = writer
-		cmd.Stdout = writer
-		defer writer.Close()
-	}
-
 	go func() {
-		if err := retry(10, 1*time.Second, 2, 5*time.Minute, func() error {
-			retryCounter.WithLabelValues(presetName).Inc()
+		isFirstRun := true
+		if err := retry(10, 1*time.Second, 2, 3*time.Minute, func() error {
+			cmd := exec.Command(s.binaryPath, args...)
+			zap.L().Info("spark-submit", zap.Strings("args", args))
+			if s.debug {
+				writer := &zapio.Writer{Log: zap.L(), Level: zap.DebugLevel}
+				cmd.Stderr = writer
+				cmd.Stdout = writer
+				defer writer.Close()
+			}
+			if !isFirstRun {
+				retryCounter.WithLabelValues(presetName).Inc()
+			}
+			isFirstRun = false
 			return cmd.Run()
 		}); err != nil {
 			zap.L().Error("spark submit failed with retries", zap.Error(err))