diff --git a/Makefile b/Makefile index ef3f1ec4..efd47214 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ generate-sdk: .PHONY: generate-docs generate-docs: go install github.com/hashicorp/terraform-plugin-docs/cmd/tfplugindocs@v0.14.1 - tfplugindocs generate --rendered-provider-name "CAST AI" --ignore-deprecated + tfplugindocs generate --rendered-provider-name "CAST AI" --ignore-deprecated --provider-name terraform-provider-castai .PHONY: generate-all generate-all: generate-sdk generate-docs @@ -63,7 +63,7 @@ test: .PHONY: testacc testacc: @echo "==> Running acceptance tests" - TF_ACC=1 go test ./castai/... '-run=^TestAcc' -v -timeout 40m + TF_ACC=1 go test ./castai/... '-run=^TestAcc' -v -timeout 50m .PHONY: validate-terraform-examples validate-terraform-examples: diff --git a/castai/cluster.go b/castai/cluster.go index 3ee1eaba..dccd7acd 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -2,10 +2,13 @@ package castai import ( "context" + "errors" "fmt" "log" "net/http" + "time" + "github.com/cenkalti/backoff/v4" "github.com/google/uuid" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry" @@ -132,6 +135,77 @@ func fetchClusterData(ctx context.Context, client *sdk.ClientWithResponses, clus return resp, nil } +// resourceCastaiClusterUpdate performs the update call to Cast API for a given cluster. +// Handles backoffs and data drift for fields that are not provider-specific. +// Caller is responsible to populate data and request parameters with all data. +func resourceCastaiClusterUpdate( + ctx context.Context, + client *sdk.ClientWithResponses, + data *schema.ResourceData, + request *sdk.ExternalClusterAPIUpdateClusterJSONRequestBody, +) error { + b := backoff.WithContext(backoff.NewExponentialBackOff(), ctx) + + var lastErr error + var credentialsID string + if err := backoff.RetryNotify(func() error { + response, err := client.ExternalClusterAPIUpdateClusterWithResponse(ctx, data.Id(), *request) + if err != nil { + return fmt.Errorf("error when calling update cluster API: %w", err) + } + + err = sdk.StatusOk(response) + + if err != nil { + // In case of malformed user request return error to user right away. + // Credentials error is omitted as permissions propagate eventually and sometimes aren't visible immediately. + if response.StatusCode() == 400 && !sdk.IsCredentialsError(response) { + return backoff.Permanent(err) + } + + if response.StatusCode() == 400 && sdk.IsCredentialsError(response) { + log.Printf("[WARN] Received credentials error from backend, will retry in case the issue is caused by IAM eventual consistency.") + } + return fmt.Errorf("error in update cluster response: %w", err) + } + + if response.JSON200.CredentialsId != nil { + credentialsID = *response.JSON200.CredentialsId + } + return nil + }, b, func(err error, _ time.Duration) { + // Only store non-context errors so we can surface the last "real" error to the user at the end + if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { + lastErr = err + } + log.Printf("[WARN] Encountered error while updating cluster settings, will retry: %v", err) + }); err != nil { + // Reset CredentialsID in state in case of failed updates. + // This is because TF will save the raw credentials in state even on failed updates. + // Since the raw values are not exposed via API, TF cannot see drift and will not try to re-apply them next time, leaving the caller stuck. + // Resetting this value here will trigger our credentialsID drift detection on Read() and force re-apply to fix the drift. + // Note: cannot use empty string; if first update failed then credentials will also be empty on remote => no drift on Read. + // Src: https://developer.hashicorp.com/terraform/plugin/framework/diagnostics#returning-errors-and-warnings + if err := data.Set(FieldClusterCredentialsId, "drift-protection-failed-update"); err != nil { + log.Printf("[ERROR] Failed to reset cluster credentials ID after failed update: %v", err) + } + + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return fmt.Errorf("updating cluster configuration failed due to context: %w; last observed error was: %v", err, lastErr) + } + return fmt.Errorf("updating cluster configuration: %w", err) + } + + // In case the update succeeded, we must update the state with the *generated* credentials_id before re-reading. + // This is because on update, the credentials_id always changes => read drift detection would see that and trigger infinite drift + err := data.Set(FieldClusterCredentialsId, credentialsID) + if err != nil { + return fmt.Errorf("failed to update credentials ID after successful update: %w", err) + } + + return nil +} + func createClusterToken(ctx context.Context, client *sdk.ClientWithResponses, clusterID string) (string, error) { resp, err := client.ExternalClusterAPICreateClusterTokenWithResponse(ctx, clusterID) if err != nil { diff --git a/castai/resource_aks_cluster.go b/castai/resource_aks_cluster.go index 6be929d1..44f60e5a 100644 --- a/castai/resource_aks_cluster.go +++ b/castai/resource_aks_cluster.go @@ -2,12 +2,10 @@ package castai import ( "context" - "errors" "fmt" "log" "time" - "github.com/cenkalti/backoff/v4" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" @@ -223,62 +221,5 @@ func updateAKSClusterSettings(ctx context.Context, data *schema.ResourceData, cl req.Credentials = &credentials - // Retries are required for newly created IAM resources to initialise on Azure side. - b := backoff.WithContext(backoff.WithMaxRetries(backoff.NewConstantBackOff(10*time.Second), 30), ctx) - var lastErr error - var credentialsID string - if err = backoff.RetryNotify(func() error { - response, err := client.ExternalClusterAPIUpdateClusterWithResponse(ctx, data.Id(), req) - if err != nil { - return fmt.Errorf("error when calling update cluster API: %w", err) - } - - err = sdk.StatusOk(response) - - if err != nil { - // In case of malformed user request return error to user right away. - // Credentials error is omitted as permissions propagate eventually and sometimes aren't visible immediately. - if response.StatusCode() == 400 && !sdk.IsCredentialsError(response) { - return backoff.Permanent(err) - } - - if response.StatusCode() == 400 && sdk.IsCredentialsError(response) { - log.Printf("[WARN] Received credentials error from backend, will retry in case the issue is caused by IAM eventual consistency.") - } - return fmt.Errorf("error in update cluster response: %w", err) - } - - credentialsID = *response.JSON200.CredentialsId - return nil - }, b, func(err error, _ time.Duration) { - // Only store non-context errors so we can surface the last "real" error to the user at the end - if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { - lastErr = err - } - log.Printf("[WARN] Encountered error while updating cluster settings, will retry: %v", err) - }); err != nil { - // Reset CredentialsID in state in case of failed updates. - // This is because TF will save the raw credentials in state even on failed updates. - // Since the raw values are not exposed via API, TF cannot see drift and will not try to re-apply them next time, leaving the caller stuck. - // Resetting this value here will trigger our credentialsID drift detection on Read() and force re-apply to fix the drift. - // Note: cannot use empty string; if first update failed then credentials will also be empty on remote => no drift on Read. - // Src: https://developer.hashicorp.com/terraform/plugin/framework/diagnostics#returning-errors-and-warnings - if err := data.Set(FieldClusterCredentialsId, "drift-protection-failed-update"); err != nil { - log.Printf("[ERROR] Failed to reset cluster credentials ID after failed update: %v", err) - } - - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - return fmt.Errorf("updating cluster configuration failed due to context: %w; last observed error was: %v", err, lastErr) - } - return fmt.Errorf("updating cluster configuration: %w", err) - } - - // In case the update succeeded, we must update the state with the *generated* credentials_id before re-reading. - // This is because on update, the credentials_id always changes => read drift detection would see that and trigger infinite drift - err = data.Set(FieldClusterCredentialsId, credentialsID) - if err != nil { - return fmt.Errorf("failed to update credentials ID after successful update: %w", err) - } - - return nil + return resourceCastaiClusterUpdate(ctx, client, data, &req) } diff --git a/castai/resource_aks_cluster_test.go b/castai/resource_aks_cluster_test.go index 851be7fb..07027c57 100644 --- a/castai/resource_aks_cluster_test.go +++ b/castai/resource_aks_cluster_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/stretchr/testify/require" @@ -22,7 +23,7 @@ import ( func TestAKSClusterResourceReadContext(t *testing.T) { ctx := context.Background() - clusterId := "b6bfc074-a267-400f-b8f1-db0850c369b1" + clusterID := "b6bfc074-a267-400f-b8f1-db0850c369b1" t.Run("read should populate data correctly", func(t *testing.T) { r := require.New(t) @@ -55,14 +56,14 @@ func TestAKSClusterResourceReadContext(t *testing.T) { "private": true }`))) mockClient.EXPECT(). - ExternalClusterAPIGetCluster(gomock.Any(), clusterId). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) aksResource := resourceAKSCluster() val := cty.ObjectVal(map[string]cty.Value{}) state := terraform.NewInstanceStateShimmedFromValue(val, 0) - state.ID = clusterId + state.ID = clusterID // If local credentials don't match remote, drift detection would trigger. // If local state has no credentials but remote has them, then the drift does exist so - there is separate test for that. state.Attributes[FieldClusterCredentialsId] = "9b8d0456-177b-4a3d-b162-e68030d656aa" @@ -115,14 +116,14 @@ Tainted = false body := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, tc.apiValue)))) mockClient.EXPECT(). - ExternalClusterAPIGetCluster(gomock.Any(), clusterId). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) aksResource := resourceAKSCluster() val := cty.ObjectVal(map[string]cty.Value{}) state := terraform.NewInstanceStateShimmedFromValue(val, 0) - state.ID = clusterId + state.ID = clusterID state.Attributes[FieldClusterCredentialsId] = tc.stateValue state.Attributes[FieldAKSClusterClientID] = clientIDBeforeRead @@ -171,14 +172,14 @@ Tainted = false body := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, tc.apiValue)))) mockClient.EXPECT(). - ExternalClusterAPIGetCluster(gomock.Any(), clusterId). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) aksResource := resourceAKSCluster() val := cty.ObjectVal(map[string]cty.Value{}) state := terraform.NewInstanceStateShimmedFromValue(val, 0) - state.ID = clusterId + state.ID = clusterID state.Attributes[FieldClusterCredentialsId] = tc.stateValue state.Attributes[FieldAKSClusterClientID] = clientIDBeforeRead @@ -196,6 +197,81 @@ Tainted = false }) } +func TestAKSClusterResourceUpdateContext(t *testing.T) { + clusterID := "b6bfc074-a267-400f-b8f1-db0850c369b1" + ctx := context.Background() + + t.Run("credentials_id special handling", func(t *testing.T) { + t.Run("on successful update, should avoid drift on the read", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + credentialsIDAfterUpdate := "after-update-credentialsid" + clientID := "clientID" + updateResponse := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, credentialsIDAfterUpdate)))) + readResponse := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, credentialsIDAfterUpdate)))) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: readResponse, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any()). + Return(&http.Response{StatusCode: 200, Body: updateResponse, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + aksResource := resourceAKSCluster() + + diff := map[string]any{ + FieldAKSClusterClientID: clientID, + FieldClusterCredentialsId: "before-update-credentialsid", + } + data := schema.TestResourceDataRaw(t, aksResource.Schema, diff) + data.SetId(clusterID) + diagnostics := aksResource.UpdateContext(ctx, data, provider) + + r.Empty(diagnostics) + + r.Equal(credentialsIDAfterUpdate, data.Get(FieldClusterCredentialsId)) + r.Equal(clientID, data.Get(FieldAKSClusterClientID)) + }) + + t.Run("on failed update, should overwrite credentialsID to force drift on next read", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any()). + Return(&http.Response{StatusCode: 400, Body: http.NoBody}, nil) + + aksResource := resourceAKSCluster() + + credentialsID := "credentialsID-before-updates" + diff := map[string]any{ + FieldClusterCredentialsId: credentialsID, + } + data := schema.TestResourceDataRaw(t, aksResource.Schema, diff) + data.SetId(clusterID) + diagnostics := aksResource.UpdateContext(ctx, data, provider) + + r.NotEmpty(diagnostics) + + valueAfter := data.Get(FieldClusterCredentialsId) + r.NotEqual(credentialsID, valueAfter) + r.Contains(valueAfter, "drift") + }) + }) +} + func TestAccResourceAKSCluster(t *testing.T) { rName := fmt.Sprintf("%v-aks-%v", ResourcePrefix, acctest.RandString(8)) resourceName := "castai_aks_cluster.test" diff --git a/castai/resource_eks_cluster.go b/castai/resource_eks_cluster.go index 2f154a25..17cf51e0 100644 --- a/castai/resource_eks_cluster.go +++ b/castai/resource_eks_cluster.go @@ -6,7 +6,6 @@ import ( "log" "time" - "github.com/cenkalti/backoff/v4" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" @@ -142,10 +141,6 @@ func resourceCastaiEKSClusterRead(ctx context.Context, data *schema.ResourceData return nil } - if err := data.Set(FieldClusterCredentialsId, *resp.JSON200.CredentialsId); err != nil { - return diag.FromErr(fmt.Errorf("setting credentials id: %w", err)) - } - if eks := resp.JSON200.Eks; eks != nil { if err := data.Set(FieldEKSClusterAccountId, toString(eks.AccountId)); err != nil { return diag.FromErr(fmt.Errorf("setting account id: %w", err)) @@ -161,6 +156,20 @@ func resourceCastaiEKSClusterRead(ctx context.Context, data *schema.ResourceData } } + // Catch if credentials_id ever gets reset on cast side (since it holds credentials to access the role used for cross-account access). + // Drift in role is already caught above, but we want to force TF to update and regenerate the credentials ID. + if resp.JSON200.CredentialsId != nil && *resp.JSON200.CredentialsId != data.Get(FieldClusterCredentialsId) { + log.Printf("[WARN] Drift in credentials from state (%q) and in API (%q), resetting credentials JSON to force re-applying credentials from configuration", + data.Get(FieldClusterCredentialsId), *resp.JSON200.CredentialsId) + if err := data.Set(FieldEKSClusterAssumeRoleArn, "credentials-drift-detected-force-apply"); err != nil { + return diag.FromErr(fmt.Errorf("setting client ID: %w", err)) + } + } + + if err := data.Set(FieldClusterCredentialsId, *resp.JSON200.CredentialsId); err != nil { + return diag.FromErr(fmt.Errorf("setting credentials id: %w", err)) + } + return nil } @@ -194,23 +203,7 @@ func updateClusterSettings(ctx context.Context, data *schema.ResourceData, clien req.Eks.AssumeRoleArn = toPtr(assumeRoleARN.(string)) } - if err := backoff.Retry(func() error { - response, err := client.ExternalClusterAPIUpdateClusterWithResponse(ctx, data.Id(), req) - if err != nil { - return err - } - err = sdk.StatusOk(response) - // In case of malformed user request return error to user right away. - if response.StatusCode() == 400 && !sdk.IsCredentialsError(response) { - return backoff.Permanent(err) - } - - return err - }, backoff.NewExponentialBackOff()); err != nil { - return fmt.Errorf("updating cluster configuration: %w", err) - } - - return nil + return resourceCastaiClusterUpdate(ctx, client, data, &req) } func getOptionalBool(data *schema.ResourceData, field string, defaultValue bool) *bool { diff --git a/castai/resource_eks_cluster_test.go b/castai/resource_eks_cluster_test.go index 7447dd47..f5d25fcd 100644 --- a/castai/resource_eks_cluster_test.go +++ b/castai/resource_eks_cluster_test.go @@ -3,6 +3,7 @@ package castai import ( "bytes" "context" + "fmt" "io" "net/http" "testing" @@ -18,20 +19,22 @@ import ( ) func TestEKSClusterResourceReadContext(t *testing.T) { - r := require.New(t) - mockctrl := gomock.NewController(t) - mockClient := mock_sdk.NewMockClientInterface(mockctrl) - ctx := context.Background() - provider := &ProviderConfig{ - api: &sdk.ClientWithResponses{ - ClientInterface: mockClient, - }, - } - clusterId := "b6bfc074-a267-400f-b8f1-db0850c369b1" + clusterID := "b6bfc074-a267-400f-b8f1-db0850c369b1" - body := io.NopCloser(bytes.NewReader([]byte(`{ + t.Run("read should populate data correctly", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + body := io.NopCloser(bytes.NewReader([]byte(`{ "id": "b6bfc074-a267-400f-b8f1-db0850c369b1", "name": "eks-cluster", "organizationId": "2836f775-aaaa-eeee-bbbb-3d3c29512692", @@ -60,21 +63,22 @@ func TestEKSClusterResourceReadContext(t *testing.T) { "clusterNameId": "eks-cluster-b6bfc074", "private": true }`))) - mockClient.EXPECT(). - ExternalClusterAPIGetCluster(gomock.Any(), clusterId). - Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) - resource := resourceEKSCluster() + resource := resourceEKSCluster() - val := cty.ObjectVal(map[string]cty.Value{}) - state := terraform.NewInstanceStateShimmedFromValue(val, 0) - state.ID = clusterId + val := cty.ObjectVal(map[string]cty.Value{}) + state := terraform.NewInstanceStateShimmedFromValue(val, 0) + state.ID = clusterID + state.Attributes[FieldClusterCredentialsId] = "9b8d0456-177b-4a3d-b162-e68030d656aa" - data := resource.Data(state) - result := resource.ReadContext(ctx, data, provider) - r.Nil(result) - r.False(result.HasError()) - r.Equal(`ID = b6bfc074-a267-400f-b8f1-db0850c369b1 + data := resource.Data(state) + result := resource.ReadContext(ctx, data, provider) + r.Nil(result) + r.False(result.HasError()) + r.Equal(`ID = b6bfc074-a267-400f-b8f1-db0850c369b1 account_id = 487609000000 assume_role_arn = credentials_id = 9b8d0456-177b-4a3d-b162-e68030d656aa @@ -82,6 +86,125 @@ name = eks-cluster region = eu-central-1 Tainted = false `, data.State().String()) + }) + + t.Run("on credentials drift, changes role_arn to trigger drift and re-apply", func(t *testing.T) { + testCase := []struct { + name string + stateValue string + apiValue string + }{ + { + name: "empty credentials in remote", + stateValue: "credentials-id-local", + apiValue: "", + }, + { + name: "different credentials in remote", + stateValue: "credentials-id-local", + apiValue: "credentials-id-remote", + }, + { + name: "empty credentials in local but exist in remote", + stateValue: "", + apiValue: "credentials-id-remote", + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + roleARNBeforeRead := "dummy-rolearn" + + body := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, tc.apiValue)))) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + resource := resourceEKSCluster() + + val := cty.ObjectVal(map[string]cty.Value{}) + state := terraform.NewInstanceStateShimmedFromValue(val, 0) + state.ID = clusterID + state.Attributes[FieldClusterCredentialsId] = tc.stateValue + state.Attributes[FieldEKSClusterAssumeRoleArn] = roleARNBeforeRead + + data := resource.Data(state) + result := resource.ReadContext(ctx, data, provider) + r.Nil(result) + r.False(result.HasError()) + + roleARNAfter := data.Get(FieldEKSClusterAssumeRoleArn) + + r.NotEqual(roleARNBeforeRead, roleARNAfter) + r.NotEmpty(roleARNAfter) + }) + } + }) + + t.Run("when credentials match, no drift should be triggered", func(t *testing.T) { + testCase := []struct { + name string + stateValue string + apiValue string + }{ + { + name: "empty credentials in both", + stateValue: "", + apiValue: "", + }, + { + name: "matching credentials", + stateValue: "credentials-id", + apiValue: "credentials-id", + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + roleARNBefore := "dummy-roleARN" + + body := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, tc.apiValue)))) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + resource := resourceEKSCluster() + + val := cty.ObjectVal(map[string]cty.Value{}) + state := terraform.NewInstanceStateShimmedFromValue(val, 0) + state.ID = clusterID + state.Attributes[FieldClusterCredentialsId] = tc.stateValue + state.Attributes[FieldEKSClusterAssumeRoleArn] = roleARNBefore + + data := resource.Data(state) + result := resource.ReadContext(ctx, data, provider) + r.Nil(result) + r.False(result.HasError()) + + roleARNAfter := data.Get(FieldEKSClusterAssumeRoleArn) + + r.Equal(roleARNBefore, roleARNAfter) + r.NotEmpty(roleARNAfter) + }) + } + }) + } func TestEKSClusterResourceReadContextArchived(t *testing.T) { @@ -145,34 +268,108 @@ func TestEKSClusterResourceReadContextArchived(t *testing.T) { } func TestEKSClusterResourceUpdateError(t *testing.T) { - r := require.New(t) - mockctrl := gomock.NewController(t) - mockClient := mock_sdk.NewMockClientInterface(mockctrl) - + clusterID := "b6bfc074-a267-400f-b8f1-db0850c36gk3d" ctx := context.Background() - provider := &ProviderConfig{ - api: &sdk.ClientWithResponses{ - ClientInterface: mockClient, - }, - } - clusterId := "b6bfc074-a267-400f-b8f1-db0850c36gk3d" - mockClient.EXPECT(). - ExternalClusterAPIUpdateCluster(gomock.Any(), clusterId, gomock.Any(), gomock.Any()). - Return(&http.Response{StatusCode: 400, Body: io.NopCloser(bytes.NewBufferString(`{"message":"Bad Request", "fieldViolations":[{"field":"credentials","description":"error"}]}`)), Header: map[string][]string{"Content-Type": {"json"}}}, nil) + t.Run("resource update error generic propagated", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) - resource := resourceEKSCluster() + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } - raw := make(map[string]interface{}) - raw[FieldEKSClusterAssumeRoleArn] = "something" + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any(), gomock.Any()). + Return(&http.Response{StatusCode: 400, Body: io.NopCloser(bytes.NewBufferString(`{"message":"Bad Request", "fieldViolations":[{"field":"credentials","description":"error"}]}`)), Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + resource := resourceEKSCluster() + + raw := make(map[string]interface{}) + raw[FieldEKSClusterAssumeRoleArn] = "something" + + data := schema.TestResourceDataRaw(t, resource.Schema, raw) + _ = data.Set(FieldEKSClusterAssumeRoleArn, "creds") + data.SetId(clusterID) + result := resource.UpdateContext(ctx, data, provider) + r.NotNil(result) + r.True(result.HasError()) + r.Equal("updating cluster configuration: expected status code 200, received: status=400 body={\"message\":\"Bad Request\", \"fieldViolations\":[{\"field\":\"credentials\",\"description\":\"error\"}]}", result[0].Summary) + }) + + t.Run("credentials_id special handling", func(t *testing.T) { + t.Run("on successful update, should avoid drift on the read", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + credentialsIDAfterUpdate := "after-update-credentialsid" + roleARN := "aws-role" + updateResponse := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, credentialsIDAfterUpdate)))) + readResponse := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, credentialsIDAfterUpdate)))) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: readResponse, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any()). + Return(&http.Response{StatusCode: 200, Body: updateResponse, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + awsResource := resourceEKSCluster() + + diff := map[string]any{ + FieldEKSClusterAssumeRoleArn: roleARN, + FieldClusterCredentialsId: "before-update-credentialsid", + } + data := schema.TestResourceDataRaw(t, awsResource.Schema, diff) + data.SetId(clusterID) + diagnostics := awsResource.UpdateContext(ctx, data, provider) + + r.Empty(diagnostics) + + r.Equal(credentialsIDAfterUpdate, data.Get(FieldClusterCredentialsId)) + r.Equal(roleARN, data.Get(FieldEKSClusterAssumeRoleArn)) + }) + + t.Run("on failed update, should overwrite credentialsID", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any()). + Return(&http.Response{StatusCode: 400, Body: http.NoBody}, nil) + + awsResource := resourceEKSCluster() + + credentialsID := "credentialsID-before-updates" + diff := map[string]any{ + FieldClusterCredentialsId: credentialsID, + } + data := schema.TestResourceDataRaw(t, awsResource.Schema, diff) + data.SetId(clusterID) + diagnostics := awsResource.UpdateContext(ctx, data, provider) + + r.NotEmpty(diagnostics) + + valueAfter := data.Get(FieldClusterCredentialsId) + r.NotEqual(credentialsID, valueAfter) + r.Contains(valueAfter, "drift") + }) + }) - data := schema.TestResourceDataRaw(t, resource.Schema, raw) - _ = data.Set(FieldEKSClusterAssumeRoleArn, "creds") - data.SetId(clusterId) - result := resource.UpdateContext(ctx, data, provider) - r.NotNil(result) - r.True(result.HasError()) - r.Equal("updating cluster configuration: expected status code 200, received: status=400 body={\"message\":\"Bad Request\", \"fieldViolations\":[{\"field\":\"credentials\",\"description\":\"error\"}]}", result[0].Summary) } func TestEKSClusterResourceUpdateRetry(t *testing.T) { diff --git a/castai/resource_gke_cluster.go b/castai/resource_gke_cluster.go index 1a44d63e..d6583ace 100644 --- a/castai/resource_gke_cluster.go +++ b/castai/resource_gke_cluster.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "github.com/cenkalti/backoff/v4" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" @@ -16,11 +15,10 @@ import ( ) const ( - FieldGKEClusterName = "name" - FieldGKEClusterProjectId = "project_id" - FieldGKEClusterLocation = "location" - FieldGKEClusterCredentialsId = "credentials_id" - FieldGKEClusterCredentials = "credentials_json" + FieldGKEClusterName = "name" + FieldGKEClusterProjectId = "project_id" + FieldGKEClusterLocation = "location" + FieldGKEClusterCredentials = "credentials_json" ) func resourceGKECluster() *schema.Resource { @@ -46,7 +44,7 @@ func resourceGKECluster() *schema.Resource { ValidateDiagFunc: validation.ToDiagFunc(validation.StringIsNotWhiteSpace), Description: "GKE cluster name", }, - FieldGKEClusterCredentialsId: { + FieldClusterCredentialsId: { Type: schema.TypeString, Computed: true, Description: "CAST AI credentials id for cluster", @@ -153,7 +151,15 @@ func resourceCastaiGKEClusterRead(ctx context.Context, data *schema.ResourceData return nil } - if err := data.Set(FieldGKEClusterCredentialsId, toString(resp.JSON200.CredentialsId)); err != nil { + if resp.JSON200.CredentialsId != nil && *resp.JSON200.CredentialsId != data.Get(FieldClusterCredentialsId) { + log.Printf("[WARN] Drift in credentials from state (%q) and in API (%q), resetting credentials JSON to force re-applying credentials from configuration", + data.Get(FieldClusterCredentialsId), *resp.JSON200.CredentialsId) + if err := data.Set(FieldGKEClusterCredentials, "credentials-drift-detected-force-apply"); err != nil { + return diag.FromErr(fmt.Errorf("setting client ID: %w", err)) + } + } + + if err := data.Set(FieldClusterCredentialsId, toString(resp.JSON200.CredentialsId)); err != nil { return diag.FromErr(fmt.Errorf("setting credentials id: %w", err)) } if GKE := resp.JSON200.Gke; GKE != nil { @@ -184,6 +190,7 @@ func resourceCastaiGKEClusterUpdate(ctx context.Context, data *schema.ResourceDa func updateGKEClusterSettings(ctx context.Context, data *schema.ResourceData, client *sdk.ClientWithResponses) error { if !data.HasChanges( FieldGKEClusterCredentials, + FieldClusterCredentialsId, ) { log.Printf("[INFO] Nothing to update in cluster setttings.") return nil @@ -198,22 +205,7 @@ func updateGKEClusterSettings(ctx context.Context, data *schema.ResourceData, cl req.Credentials = toPtr(credentialsJSON.(string)) } - if err := backoff.Retry(func() error { - response, err := client.ExternalClusterAPIUpdateClusterWithResponse(ctx, data.Id(), req) - if err != nil { - return err - } - err = sdk.StatusOk(response) - // In case of malformed user request return error to user right away. - if response.StatusCode() == 400 && !sdk.IsCredentialsError(response) { - return backoff.Permanent(err) - } - return err - }, backoff.NewExponentialBackOff()); err != nil { - return fmt.Errorf("updating cluster configuration: %w", err) - } - - return nil + return resourceCastaiClusterUpdate(ctx, client, data, &req) } func resourceCastaiGKEClusterDelete(ctx context.Context, data *schema.ResourceData, meta interface{}) diag.Diagnostics { diff --git a/castai/resource_gke_cluster_test.go b/castai/resource_gke_cluster_test.go index 51d4a8f3..6192f14e 100644 --- a/castai/resource_gke_cluster_test.go +++ b/castai/resource_gke_cluster_test.go @@ -3,6 +3,7 @@ package castai import ( "bytes" "context" + "fmt" "io" "net/http" "testing" @@ -18,20 +19,21 @@ import ( ) func TestGKEClusterResourceReadContext(t *testing.T) { - r := require.New(t) - mockctrl := gomock.NewController(t) - mockClient := mock_sdk.NewMockClientInterface(mockctrl) - ctx := context.Background() - provider := &ProviderConfig{ - api: &sdk.ClientWithResponses{ - ClientInterface: mockClient, - }, - } + clusterID := "b6bfc074-a267-400f-b8f1-db0850c36gke" - clusterId := "b6bfc074-a267-400f-b8f1-db0850c36gke" + t.Run("read should populate data correctly", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) - body := io.NopCloser(bytes.NewReader([]byte(`{ + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + body := io.NopCloser(bytes.NewReader([]byte(`{ "id": "b6bfc074-a267-400f-b8f1-db0850c36gk3", "name": "gke-cluster", "organizationId": "2836f775-aaaa-eeee-bbbb-3d3c29512GKE", @@ -53,27 +55,148 @@ func TestGKEClusterResourceReadContext(t *testing.T) { }, "clusterNameId": "gke-cluster-b6bfc074" }`))) - mockClient.EXPECT(). - ExternalClusterAPIGetCluster(gomock.Any(), clusterId). - Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) - resource := resourceGKECluster() + resource := resourceGKECluster() - val := cty.ObjectVal(map[string]cty.Value{}) - state := terraform.NewInstanceStateShimmedFromValue(val, 0) - state.ID = clusterId + val := cty.ObjectVal(map[string]cty.Value{}) + state := terraform.NewInstanceStateShimmedFromValue(val, 0) + state.ID = clusterID + state.Attributes[FieldClusterCredentialsId] = "9b8d0456-177b-4a3d-b162-e68030d65GKE" // Avoid drift detection - data := resource.Data(state) - result := resource.ReadContext(ctx, data, provider) - r.Nil(result) - r.False(result.HasError()) - r.Equal(`ID = b6bfc074-a267-400f-b8f1-db0850c36gke + data := resource.Data(state) + result := resource.ReadContext(ctx, data, provider) + r.Nil(result) + r.False(result.HasError()) + r.Equal(`ID = b6bfc074-a267-400f-b8f1-db0850c36gke credentials_id = 9b8d0456-177b-4a3d-b162-e68030d65GKE location = eu-central-1 name = gke-cluster project_id = project-id Tainted = false `, data.State().String()) + }) + + t.Run("on credentials drift, changes credentials_json to trigger drift and re-apply", func(t *testing.T) { + testCase := []struct { + name string + stateValue string + apiValue string + }{ + { + name: "empty credentials in remote", + stateValue: "credentials-id-local", + apiValue: "", + }, + { + name: "different credentials in remote", + stateValue: "credentials-id-local", + apiValue: "credentials-id-remote", + }, + { + name: "empty credentials in local but exist in remote", + stateValue: "", + apiValue: "credentials-id-remote", + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + credentialsBeforeRead := "dummy-credentials" + + body := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, tc.apiValue)))) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + gkeResource := resourceGKECluster() + + val := cty.ObjectVal(map[string]cty.Value{}) + state := terraform.NewInstanceStateShimmedFromValue(val, 0) + state.ID = clusterID + state.Attributes[FieldClusterCredentialsId] = tc.stateValue + state.Attributes[FieldGKEClusterCredentials] = credentialsBeforeRead + + data := gkeResource.Data(state) + result := gkeResource.ReadContext(ctx, data, provider) + r.Nil(result) + r.False(result.HasError()) + + credentialsAfterRead := data.Get(FieldGKEClusterCredentials) + + r.NotEqual(credentialsBeforeRead, credentialsAfterRead) + r.NotEmpty(credentialsAfterRead) + r.Contains(credentialsAfterRead, "drift") + }) + } + }) + + t.Run("when credentials match, no drift should be triggered", func(t *testing.T) { + testCase := []struct { + name string + stateValue string + apiValue string + }{ + { + name: "empty credentials in both", + stateValue: "", + apiValue: "", + }, + { + name: "matching credentials", + stateValue: "credentials-id", + apiValue: "credentials-id", + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + credentialsBeforeRead := "dummy-credentials" + + body := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, tc.apiValue)))) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: body, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + gkeResource := resourceGKECluster() + + val := cty.ObjectVal(map[string]cty.Value{}) + state := terraform.NewInstanceStateShimmedFromValue(val, 0) + state.ID = clusterID + state.Attributes[FieldClusterCredentialsId] = tc.stateValue + state.Attributes[FieldGKEClusterCredentials] = credentialsBeforeRead + + data := gkeResource.Data(state) + result := gkeResource.ReadContext(ctx, data, provider) + r.Nil(result) + r.False(result.HasError()) + + credentialsAfterRead := data.Get(FieldGKEClusterCredentials) + + r.Equal(credentialsBeforeRead, credentialsAfterRead) + r.NotEmpty(credentialsAfterRead) + }) + } + }) } func TestGKEClusterResourceReadContextArchived(t *testing.T) { @@ -131,33 +254,107 @@ func TestGKEClusterResourceReadContextArchived(t *testing.T) { r.Equal(``, data.State().String()) } -func TestGKEClusterResourceUpdateError(t *testing.T) { - r := require.New(t) - mockctrl := gomock.NewController(t) - mockClient := mock_sdk.NewMockClientInterface(mockctrl) - +func TestGKEClusterResourceUpdate(t *testing.T) { + clusterID := "b6bfc074-a267-400f-b8f1-db0850c36gk3d" ctx := context.Background() - provider := &ProviderConfig{ - api: &sdk.ClientWithResponses{ - ClientInterface: mockClient, - }, - } - clusterId := "b6bfc074-a267-400f-b8f1-db0850c36gk3d" - mockClient.EXPECT(). - ExternalClusterAPIUpdateCluster(gomock.Any(), clusterId, gomock.Any(), gomock.Any()). - Return(&http.Response{StatusCode: 400, Body: io.NopCloser(bytes.NewBufferString(`{"message":"Bad Request", "fieldViolations":[{"field":"credentials","description":"error"}]}`)), Header: map[string][]string{"Content-Type": {"json"}}}, nil) + t.Run("resource update error generic propagated", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) - resource := resourceGKECluster() + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any(), gomock.Any()). + Return(&http.Response{StatusCode: 400, Body: io.NopCloser(bytes.NewBufferString(`{"message":"Bad Request", "fieldViolations":[{"field":"credentials","description":"error"}]}`)), Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + resource := resourceGKECluster() + + raw := make(map[string]interface{}) + raw[FieldGKEClusterCredentials] = "something" + + data := schema.TestResourceDataRaw(t, resource.Schema, raw) + _ = data.Set(FieldGKEClusterCredentials, "creds") + data.SetId(clusterID) + result := resource.UpdateContext(ctx, data, provider) + r.NotNil(result) + r.True(result.HasError()) + r.Equal("updating cluster configuration: expected status code 200, received: status=400 body={\"message\":\"Bad Request\", \"fieldViolations\":[{\"field\":\"credentials\",\"description\":\"error\"}]}", result[0].Summary) + + }) + + t.Run("credentials_id special handling", func(t *testing.T) { + t.Run("on successful update, should avoid drift on the read", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + credentialsIDAfterUpdate := "after-update-credentialsid" + googleCredentials := "google-creds" + updateResponse := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, credentialsIDAfterUpdate)))) + readResponse := io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"credentialsId": "%s"}`, credentialsIDAfterUpdate)))) + mockClient.EXPECT(). + ExternalClusterAPIGetCluster(gomock.Any(), clusterID). + Return(&http.Response{StatusCode: 200, Body: readResponse, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any()). + Return(&http.Response{StatusCode: 200, Body: updateResponse, Header: map[string][]string{"Content-Type": {"json"}}}, nil) + + gkeResource := resourceGKECluster() + + diff := map[string]any{ + FieldGKEClusterCredentials: googleCredentials, + FieldClusterCredentialsId: "before-update-credentialsid", + } + data := schema.TestResourceDataRaw(t, gkeResource.Schema, diff) + data.SetId(clusterID) + diagnostics := gkeResource.UpdateContext(ctx, data, provider) + + r.Empty(diagnostics) + + r.Equal(credentialsIDAfterUpdate, data.Get(FieldClusterCredentialsId)) + r.Equal(googleCredentials, data.Get(FieldGKEClusterCredentials)) + }) + + t.Run("on failed update, should overwrite credentialsID to force drift on next read", func(t *testing.T) { + r := require.New(t) + mockctrl := gomock.NewController(t) + mockClient := mock_sdk.NewMockClientInterface(mockctrl) + provider := &ProviderConfig{ + api: &sdk.ClientWithResponses{ + ClientInterface: mockClient, + }, + } + + mockClient.EXPECT(). + ExternalClusterAPIUpdateCluster(gomock.Any(), clusterID, gomock.Any()). + Return(&http.Response{StatusCode: 400, Body: http.NoBody}, nil) + + gkeResource := resourceGKECluster() + + credentialsID := "credentialsID-before-updates" + diff := map[string]any{ + FieldClusterCredentialsId: credentialsID, + } + data := schema.TestResourceDataRaw(t, gkeResource.Schema, diff) + data.SetId(clusterID) + diagnostics := gkeResource.UpdateContext(ctx, data, provider) - raw := make(map[string]interface{}) - raw[FieldGKEClusterCredentials] = "something" + r.NotEmpty(diagnostics) - data := schema.TestResourceDataRaw(t, resource.Schema, raw) - _ = data.Set(FieldGKEClusterCredentials, "creds") - data.SetId(clusterId) - result := resource.UpdateContext(ctx, data, provider) - r.NotNil(result) - r.True(result.HasError()) - r.Equal("updating cluster configuration: expected status code 200, received: status=400 body={\"message\":\"Bad Request\", \"fieldViolations\":[{\"field\":\"credentials\",\"description\":\"error\"}]}", result[0].Summary) + valueAfter := data.Get(FieldClusterCredentialsId) + r.NotEqual(credentialsID, valueAfter) + r.Contains(valueAfter, "drift") + }) + }) } diff --git a/castai/sdk/api.gen.go b/castai/sdk/api.gen.go index fb4ffa65..8e4fec04 100644 --- a/castai/sdk/api.gen.go +++ b/castai/sdk/api.gen.go @@ -3334,6 +3334,7 @@ type WorkloadoptimizationV1AggregatedMetrics struct { type WorkloadoptimizationV1AntiAffinitySettings struct { // Defines if anti-affinity should be considered when scaling the workload. // When true, requiring host ports, or having anti-affinity on hostname will force all recommendations to be deferred. + // When not set or missing the default value is true. ConsiderAntiAffinity *bool `json:"considerAntiAffinity"` }