fix: retry in case of credentials error

Makefile

@@ -54,7 +54,7 @@ test:
 .PHONY: testacc
 testacc:
 	@echo "==> Running acceptance tests"
-	TF_ACC=1 go test ./castai/... '-run=^TestAcc' -v -timeout 16m
+	TF_ACC=1 go test ./castai/... '-run=^TestAcc' -v -timeout 20m
 
 .PHONY: validate-terraform-examples
 validate-terraform-examples:

castai/resource_eks_cluster.go

@@ -201,9 +201,10 @@ func updateClusterSettings(ctx context.Context, data *schema.ResourceData, clien
 		}
 		err = sdk.StatusOk(response)
 		// In case of malformed user request return error to user right away.
-		if response.StatusCode() == 400 {
+		if response.StatusCode() == 400 && !sdk.IsCredentialsError(response) {
 			return backoff.Permanent(err)
 		}
+
 		return err
 	}, backoff.NewExponentialBackOff()); err != nil {
 		return fmt.Errorf("updating cluster configuration: %w", err)

castai/resource_gke_cluster.go

@@ -209,7 +209,7 @@ func updateGKEClusterSettings(ctx context.Context, data *schema.ResourceData, cl
 		}
 		err = sdk.StatusOk(response)
 		// In case of malformed user request return error to user right away.
-		if response.StatusCode() == 400 {
+		if response.StatusCode() == 400 && !sdk.IsCredentialsError(response) {
 			return backoff.Permanent(err)
 		}
 		return err

castai/resource_node_template_test.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/samber/lo"
 	"io"
 	"net/http"
 	"testing"
 	"time"
 
+	"github.com/samber/lo"
+
 	"github.com/golang/mock/gomock"
 	"github.com/hashicorp/go-cty/cty"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
@@ -580,6 +581,7 @@ func testAccCheckNodeTemplateDestroy(s *terraform.State) error {
 
 		id := rs.Primary.ID
 		clusterID := rs.Primary.Attributes["cluster_id"]
+
 		response, err := client.NodeTemplatesAPIListNodeTemplatesWithResponse(ctx, clusterID, &sdk.NodeTemplatesAPIListNodeTemplatesParams{IncludeDefault: lo.ToPtr(false)})
 		if err != nil {
 			return err
@@ -592,7 +594,7 @@ func testAccCheckNodeTemplateDestroy(s *terraform.State) error {
 			return nil
 		}
 
-		return fmt.Errorf("node template %q still exists; %v", id, response)
+		return fmt.Errorf("node template %q still exists; %v", id, string(response.GetBody()))
 	}
 
 	return nil

castai/sdk/utils_response.go

@@ -1,6 +1,7 @@
 package sdk
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 )
@@ -32,3 +33,23 @@ func checkResponse(response Response, err error, expectedStatus int) error {
 
 	return nil
 }
+
+type ErrorResponse struct {
+	Message         string `json:"message"`
+	FieldViolations []struct {
+		Field       string `json:"field"`
+		Description string `json:"description"`
+	} `json:"fieldViolations"`
+}
+
+func IsCredentialsError(response Response) bool {
+	buf := response.GetBody()
+
+	var errResponse ErrorResponse
+	err := json.Unmarshal(buf, &errResponse)
+	if err != nil {
+		return false
+	}
+
+	return errResponse.Message == "Forbidden" && len(errResponse.FieldViolations) > 0 && errResponse.FieldViolations[0].Field == "credentials"
+}

castai/sdk/utils_response_test.go

@@ -0,0 +1,54 @@
+package sdk
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_IsCredentialsError(t *testing.T) {
+	t.Run("credentials error", func(t *testing.T) {
+		t.Parallel()
+		r := require.New(t)
+
+		body := `{
+	"message": "Forbidden",
+	"fieldViolations":
+	[{"field": "credentials", "description": ""}]
+}`
+		resp := ExternalClusterAPIReconcileClusterResponse{
+			Body: []byte(body),
+		}
+		result := IsCredentialsError(resp)
+		r.True(result)
+	})
+	t.Run("not credentials error", func(t *testing.T) {
+		t.Parallel()
+		r := require.New(t)
+
+		body := `{
+	"message": "something",
+	"fieldViolations":
+	[{"field": "credentials", "description": ""}]
+}`
+		resp := ExternalClusterAPIReconcileClusterResponse{
+			Body: []byte(body),
+		}
+		result := IsCredentialsError(resp)
+		r.False(result)
+	})
+	t.Run("no fields in message", func(t *testing.T) {
+		t.Parallel()
+		r := require.New(t)
+
+		body := `{
+	"message": "something",
+	"fieldViolations":[]
+}`
+		resp := ExternalClusterAPIReconcileClusterResponse{
+			Body: []byte(body),
+		}
+		result := IsCredentialsError(resp)
+		r.False(result)
+	})
+}