K8s Test Runner - Update scripts (#865)

k8s-test-runner/cmd/internal/create.go

@@ -21,15 +21,14 @@ var Create = &cobra.Command{
 func init() {
 	Create.Flags().String("image-registry-url", "", "Image registry url (e.g. ECR url)")
 	Create.MarkFlagRequired("image-registry-url")
-	Create.Flags().String("image-tag", "", "Test name (e.g. mercury-load-test)")
-	Create.MarkFlagRequired("image-registry-url")
+	Create.Flags().String("image-name", "", "Image name (e.g. k8s-test-runner-binary)")
+	Create.MarkFlagRequired("image-name")
+	Create.Flags().String("image-tag", "", "Image tag (e.g. mercury-load-tests)")
+	Create.MarkFlagRequired("image-tag")
 	Create.Flags().String("test-runner-root-dir", "./", "Test runner root directory with default chart and Dockerfile.testbin")
+	Create.Flags().String("timeout", "10m", "Timeout for the test binary build and image push")
 }
 
-var (
-	K8sTestRunnerImageName = "k8s-test-runner"
-)
-
 func createRunnerImageRunE(cmd *cobra.Command, args []string) error {
 	testPackage, err := filepath.Abs(args[0])
 	if err != nil {
@@ -39,7 +38,12 @@ func createRunnerImageRunE(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("folder with the tests does not exist: %s", testPackage)
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
+	timeoutStr := cmd.Flag("timeout").Value.String()
+	timeout, err := time.ParseDuration(timeoutStr)
+	if err != nil {
+		return fmt.Errorf("error parsing timeout: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
 	rootDir, err := cmd.Flags().GetString("test-runner-root-dir")
@@ -64,11 +68,16 @@ func createRunnerImageRunE(cmd *cobra.Command, args []string) error {
 
 	fmt.Print("Creating docker image for the test binary..\n")
 
+	imageName, err := cmd.Flags().GetString("image-name")
+	if err != nil {
+		return fmt.Errorf("error getting image name: %v", err)
+	}
 	imageTag, err := cmd.Flags().GetString("image-tag")
 	if err != nil {
-		return fmt.Errorf("error getting test name: %v", err)
+		return fmt.Errorf("error getting image tag: %v", err)
 	}
-	buildTestBinDockerImageCmd := exec.CommandContext(ctx, "docker", "build", "--platform", "linux/amd64", "-f", "Dockerfile.testbin", "--build-arg", "TEST_BINARY=testbin", "-t", fmt.Sprintf("%s:%s", K8sTestRunnerImageName, imageTag), ".")
+
+	buildTestBinDockerImageCmd := exec.CommandContext(ctx, "docker", "build", "--platform", "linux/amd64", "-f", "Dockerfile.testbin", "--build-arg", "TEST_BINARY=testbin", "-t", fmt.Sprintf("%s:%s", imageName, imageTag), ".")
 	buildTestBinDockerImageCmd.Dir = rootDirAbs
 
 	fmt.Printf("Running command from %s: %s\n", rootDirAbs, buildTestBinDockerImageCmd.String())
@@ -78,14 +87,14 @@ func createRunnerImageRunE(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("error building test binary image: %v, output: %s", err, buildTestBinDockerImageOutput)
 	}
 
-	fmt.Printf("Done. Created docker image %s:%s\n", K8sTestRunnerImageName, imageTag)
+	fmt.Printf("Done. Created docker image %s:%s\n", imageName, imageTag)
 
 	registryURL, err := cmd.Flags().GetString("image-registry-url")
 	if err != nil {
 		return fmt.Errorf("error getting ECR registry name: %v", err)
 	}
 
-	err = pushDockerImageToECR(ctx, "us-west-2", registryURL, K8sTestRunnerImageName, imageTag)
+	err = pushDockerImageToECR(ctx, "us-west-2", registryURL, imageName, imageTag)
 	if err != nil {
 		return fmt.Errorf("error pushing docker image to ECR: %v", err)
 	}

k8s-test-runner/exec/exec.go

@@ -0,0 +1,49 @@
+package exec
+
+import (
+	"bufio"
+	"io"
+	"os/exec"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+)
+
+// Cmd executes os command, logging both streams
+func Cmd(command string) error {
+	return CmdWithStreamFunc(command, func(m string) {
+		log.Info().Str("Text", m).Msg("Command output")
+	})
+}
+
+// readStdPipe continuously read a pipe from the command
+func readStdPipe(pipe io.ReadCloser, streamFunc func(string)) {
+	scanner := bufio.NewScanner(pipe)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		m := scanner.Text()
+		if streamFunc != nil {
+			streamFunc(m)
+		}
+	}
+}
+
+// CmdWithStreamFunc executes command with stream function
+func CmdWithStreamFunc(command string, outputFunction func(string)) error {
+	c := strings.Split(command, " ")
+	cmd := exec.Command(c[0], c[1:]...)
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	go readStdPipe(stderr, outputFunction)
+	go readStdPipe(stdout, outputFunction)
+	return cmd.Wait()
+}

k8s-test-runner/k8s_client/client.go

@@ -9,6 +9,7 @@ import (
 	"github.com/aws/smithy-go/ptr"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
+	exec "github.com/smartcontractkit/chainlink-testing-framework/k8s-test-runner/exec"
 	batchV1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -202,8 +203,14 @@ func (m *Client) WaitUntilJobsComplete(ctx context.Context, namespace, syncLabel
 				LabelSelector:  labelSelector,
 				TimeoutSeconds: ptr.Int64(30), // query timeout
 			})
+
 			if err != nil {
-				log.Error().Err(err).Msg("Failed to list jobs, will retry...")
+				log.Error().Err(err).Str("labelSelector", labelSelector).Msg("Failed to list jobs, will retry...")
+
+				cmd := fmt.Sprintf("kubectl get jobs -l sync=%s -n wasp --v=7", syncLabelValue)
+				log.Info().Str("cmd", cmd).Msg("Running CLI command with verbose output to debug...")
+				_ = exec.Cmd(cmd)
+
 				time.Sleep(pollingInterval)
 				continue
 			}

k8s-test-runner/runner/runner.go

@@ -4,18 +4,18 @@ import (
 	"context"
 	_ "embed"
 	"fmt"
+	"os"
+
 	"path"
 	"runtime"
 	"time"
 
-	"helm.sh/helm/v3/pkg/action"
-	"helm.sh/helm/v3/pkg/chart/loader"
-	"helm.sh/helm/v3/pkg/cli"
-
 	"github.com/google/uuid"
 	"github.com/rs/zerolog/log"
 	"github.com/smartcontractkit/chainlink-testing-framework/k8s-test-runner/config"
+	"github.com/smartcontractkit/chainlink-testing-framework/k8s-test-runner/exec"
 	"github.com/smartcontractkit/chainlink-testing-framework/k8s-test-runner/k8s_client"
+	"gopkg.in/yaml.v2"
 )
 
 type K8sTestRun struct {
@@ -52,31 +52,34 @@ func (m *K8sTestRun) getChartPath() string {
 }
 
 func (m *K8sTestRun) deployHelm(testName string) error {
-	settings := cli.New() // Initialize the Helm environment settings
-	actionConfig := new(action.Configuration)
-	if err := actionConfig.Init(settings.RESTClientGetter(), m.cfg.Namespace, "", log.Printf); err != nil {
-		return fmt.Errorf("failed to initialize Helm action configuration: %w", err)
+	overridesYAML, err := yaml.Marshal(m.ChartOverrides)
+	if err != nil {
+		return fmt.Errorf("failed to convert overrides to YAML: %w", err)
 	}
 
-	install := action.NewInstall(actionConfig)
-	install.ReleaseName = testName
-	install.Namespace = m.cfg.Namespace
-	install.Timeout = time.Minute * 3
-
-	chart, err := loader.Load(m.getChartPath())
+	// Create a temporary file to save the overrides
+	tmpFile, err := os.CreateTemp("", "helm-overrides-*.yaml")
 	if err != nil {
-		return fmt.Errorf("failed to load chart: %w", err)
+		return fmt.Errorf("failed to create temp file: %w", err)
 	}
+	defer tmpFile.Close()
 
-	// Install helm chart with overrides
-	log.Info().Interface("Overrides", m.ChartOverrides).Msg("Installing chart with overrides")
-	release, err := install.Run(chart, m.ChartOverrides)
-	if err != nil {
-		return fmt.Errorf("failed to install chart with overrides: %w", err)
+	// Write the YAML data to the temp file
+	if _, err = tmpFile.Write(overridesYAML); err != nil {
+		return fmt.Errorf("failed to write overrides to temp file: %w", err)
+	}
+
+	// Ensure to flush writes to storage
+	if err = tmpFile.Sync(); err != nil {
+		return fmt.Errorf("failed to flush writes to temp file: %w", err)
 	}
-	log.Info().Str("releaseName", release.Name).Str("namespace", release.Namespace).Msg("Chart installed successfully")
 
-	return nil
+	// Run helm install command
+	cmd := fmt.Sprintf("helm install %s %s --namespace %s --values %s --timeout 30s --debug", testName, m.getChartPath(), m.cfg.Namespace, tmpFile.Name())
+	log.Info().Str("cmd", cmd).Msg("Running helm install...")
+	return exec.CmdWithStreamFunc(cmd, func(m string) {
+		fmt.Printf("%s\n", m)
+	})
 }
 
 // Run starts a new test