From 0fe78ae4fff4434e0a71d059b503fcf9ff50f472 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Mon, 13 Nov 2023 15:47:25 +0100 Subject: [PATCH 01/12] fix: delete cluster from console --- castai/cluster.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/castai/cluster.go b/castai/cluster.go index 6780449b..10150ad9 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -37,7 +37,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, agentStatus := *clusterResponse.JSON200.AgentStatus log.Printf("[INFO] Current cluster status=%s, agent_status=%s", clusterStatus, agentStatus) - if clusterStatus == sdk.ClusterStatusDeleted || clusterStatus == sdk.ClusterStatusArchived { + if clusterStatus == sdk.ClusterStatusArchived { log.Printf("[INFO] Cluster is already deleted, removing from state.") data.SetId("") return nil @@ -51,6 +51,10 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return retry.RetryableError(fmt.Errorf("triggered cluster deletion")) } + if agentStatus == sdk.ClusterAgentStatusDisconnected { + return triggerDelete() + } + // If cluster doesn't have credentials we have to call delete cluster instead of disconnect because disconnect // will do nothing on cluster with empty credentials. if toString(clusterResponse.JSON200.CredentialsId) == "" { @@ -86,7 +90,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return triggerDelete() } - return retry.RetryableError(fmt.Errorf("retrying")) + return retry.RetryableError(fmt.Errorf("retrying cluster status %s agent status %s", clusterStatus, agentStatus)) }) if err != nil { From 6cb0c16729c90935a3591c44db3b6e00e20f8758 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Mon, 13 Nov 2023 16:19:36 +0100 Subject: [PATCH 02/12] fix: tests - aks onboarding is done in node configuration tests --- castai/resource_aks_cluster_test.go | 39 ----------------------------- 1 file changed, 39 deletions(-) diff --git a/castai/resource_aks_cluster_test.go b/castai/resource_aks_cluster_test.go index b2b5240b..cefcca83 100644 --- a/castai/resource_aks_cluster_test.go +++ b/castai/resource_aks_cluster_test.go @@ -10,8 +10,6 @@ import ( "github.com/golang/mock/gomock" "github.com/hashicorp/go-cty/cty" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/stretchr/testify/require" @@ -74,43 +72,6 @@ Tainted = false `, data.State().String()) } -func TestAccResourceAKSCluster(t *testing.T) { - rName := fmt.Sprintf("%v-aks-%v", ResourcePrefix, acctest.RandString(8)) - resourceName := "castai_aks_cluster.test" - clusterName := "core-tf-acc" - resourceGroupName := "core-tf-acc" - nodeResourceGroupName := "core-tf-acc-ng" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: providerFactories, - // Destroy of the cluster is not working properly. Cluster wasn't full onboarded and it's getting destroyed. - // https://castai.atlassian.net/browse/CORE-2868 should solve the issue - //CheckDestroy: testAccCheckAKSClusterDestroy, - Steps: []resource.TestStep{ - { - Config: testAccAKSClusterConfig(rName, clusterName, resourceGroupName, nodeResourceGroupName), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr(resourceName, "name", clusterName), - resource.TestCheckResourceAttrSet(resourceName, "credentials_id"), - resource.TestCheckResourceAttr(resourceName, "region", "westeurope"), - resource.TestCheckResourceAttrSet(resourceName, "cluster_token"), - ), - }, - }, - ExternalProviders: map[string]resource.ExternalProvider{ - "azurerm": { - Source: "hashicorp/azurerm", - VersionConstraint: "~> 3.7.0", - }, - "azuread": { - Source: "hashicorp/azuread", - VersionConstraint: "~> 2.22.0", - }, - }, - }) -} - func testAccAKSClusterConfig(rName string, clusterName string, resourceGroupName, nodeResourceGroup string) string { return ConfigCompose(testAccAzureConfig(rName, resourceGroupName, nodeResourceGroup), fmt.Sprintf(` resource "castai_aks_cluster" "test" { From deed8079575c5c55e187133c3bbf54c010eaa0c7 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Mon, 13 Nov 2023 16:28:14 +0100 Subject: [PATCH 03/12] Revert "fix: tests - aks onboarding is done in node configuration tests" This reverts commit 7e8da3aa727e9bcddcf46cb2cb368c63590d3dc1. --- castai/resource_aks_cluster_test.go | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/castai/resource_aks_cluster_test.go b/castai/resource_aks_cluster_test.go index cefcca83..b2b5240b 100644 --- a/castai/resource_aks_cluster_test.go +++ b/castai/resource_aks_cluster_test.go @@ -10,6 +10,8 @@ import ( "github.com/golang/mock/gomock" "github.com/hashicorp/go-cty/cty" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/stretchr/testify/require" @@ -72,6 +74,43 @@ Tainted = false `, data.State().String()) } +func TestAccResourceAKSCluster(t *testing.T) { + rName := fmt.Sprintf("%v-aks-%v", ResourcePrefix, acctest.RandString(8)) + resourceName := "castai_aks_cluster.test" + clusterName := "core-tf-acc" + resourceGroupName := "core-tf-acc" + nodeResourceGroupName := "core-tf-acc-ng" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: providerFactories, + // Destroy of the cluster is not working properly. Cluster wasn't full onboarded and it's getting destroyed. + // https://castai.atlassian.net/browse/CORE-2868 should solve the issue + //CheckDestroy: testAccCheckAKSClusterDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAKSClusterConfig(rName, clusterName, resourceGroupName, nodeResourceGroupName), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(resourceName, "name", clusterName), + resource.TestCheckResourceAttrSet(resourceName, "credentials_id"), + resource.TestCheckResourceAttr(resourceName, "region", "westeurope"), + resource.TestCheckResourceAttrSet(resourceName, "cluster_token"), + ), + }, + }, + ExternalProviders: map[string]resource.ExternalProvider{ + "azurerm": { + Source: "hashicorp/azurerm", + VersionConstraint: "~> 3.7.0", + }, + "azuread": { + Source: "hashicorp/azuread", + VersionConstraint: "~> 2.22.0", + }, + }, + }) +} + func testAccAKSClusterConfig(rName string, clusterName string, resourceGroupName, nodeResourceGroup string) string { return ConfigCompose(testAccAzureConfig(rName, resourceGroupName, nodeResourceGroup), fmt.Sprintf(` resource "castai_aks_cluster" "test" { From aeee917262898cc29c211cadc49cc885b6a0d458 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Mon, 13 Nov 2023 16:31:58 +0100 Subject: [PATCH 04/12] fix: trying to fix tests --- castai/cluster.go | 8 ++++---- castai/resource_aks_cluster.go | 2 +- castai/resource_eks_cluster.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/castai/cluster.go b/castai/cluster.go index 10150ad9..d679dc8d 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -51,7 +51,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return retry.RetryableError(fmt.Errorf("triggered cluster deletion")) } - if agentStatus == sdk.ClusterAgentStatusDisconnected { + if agentStatus == sdk.ClusterAgentStatusDisconnected || agentStatus == "connecting" { return triggerDelete() } @@ -66,11 +66,11 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, } if agentStatus == sdk.ClusterAgentStatusDisconnecting { - return retry.RetryableError(fmt.Errorf("agent is disconnecting")) + return retry.RetryableError(fmt.Errorf("agent is disconnecting cluster status %s agent status %s", clusterStatus, agentStatus)) } if clusterStatus == sdk.ClusterStatusDeleting { - return retry.RetryableError(fmt.Errorf("cluster is deleting")) + return retry.RetryableError(fmt.Errorf("cluster is deleting cluster status %s agent status", clusterStatus, agentStatus)) } if toString(clusterResponse.JSON200.CredentialsId) != "" && agentStatus != sdk.ClusterAgentStatusDisconnected { @@ -83,7 +83,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return retry.NonRetryableError(err) } - return retry.RetryableError(fmt.Errorf("triggered agent disconnection")) + return retry.RetryableError(fmt.Errorf("triggered agent disconnection cluster status %s agent status %s", clusterStatus, agentStatus)) } if agentStatus == sdk.ClusterAgentStatusDisconnected && clusterStatus != sdk.ClusterStatusDeleted { diff --git a/castai/resource_aks_cluster.go b/castai/resource_aks_cluster.go index 9fbcdf68..92bd5a64 100644 --- a/castai/resource_aks_cluster.go +++ b/castai/resource_aks_cluster.go @@ -36,7 +36,7 @@ func resourceAKSCluster() *schema.Resource { Timeouts: &schema.ResourceTimeout{ Create: schema.DefaultTimeout(5 * time.Minute), Update: schema.DefaultTimeout(1 * time.Minute), - Delete: schema.DefaultTimeout(2 * time.Minute), + Delete: schema.DefaultTimeout(6 * time.Minute), }, Schema: map[string]*schema.Schema{ diff --git a/castai/resource_eks_cluster.go b/castai/resource_eks_cluster.go index 8243fc11..76f84192 100644 --- a/castai/resource_eks_cluster.go +++ b/castai/resource_eks_cluster.go @@ -33,7 +33,7 @@ func resourceEKSCluster() *schema.Resource { Timeouts: &schema.ResourceTimeout{ Create: schema.DefaultTimeout(5 * time.Minute), Update: schema.DefaultTimeout(1 * time.Minute), - Delete: schema.DefaultTimeout(5 * time.Minute), + Delete: schema.DefaultTimeout(6 * time.Minute), }, Schema: map[string]*schema.Schema{ From bc46193c178b573b031ee91a462a481c01268b24 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Mon, 13 Nov 2023 16:34:56 +0100 Subject: [PATCH 05/12] fix: trying to fix tests --- castai/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/castai/cluster.go b/castai/cluster.go index d679dc8d..b9e25b01 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -70,7 +70,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, } if clusterStatus == sdk.ClusterStatusDeleting { - return retry.RetryableError(fmt.Errorf("cluster is deleting cluster status %s agent status", clusterStatus, agentStatus)) + return retry.RetryableError(fmt.Errorf("cluster is deleting cluster status %s agent status %s", clusterStatus, agentStatus)) } if toString(clusterResponse.JSON200.CredentialsId) != "" && agentStatus != sdk.ClusterAgentStatusDisconnected { From 28ee8ed5fdb043fd3d21b54e4b028c0106953f70 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Mon, 13 Nov 2023 16:46:28 +0100 Subject: [PATCH 06/12] fix: trying to fix tests --- castai/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/castai/cluster.go b/castai/cluster.go index b9e25b01..9fa2bc32 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -51,7 +51,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return retry.RetryableError(fmt.Errorf("triggered cluster deletion")) } - if agentStatus == sdk.ClusterAgentStatusDisconnected || agentStatus == "connecting" { + if agentStatus == sdk.ClusterAgentStatusDisconnected || agentStatus == "waiting-connection" { return triggerDelete() } From 80889019b8b5508500d040ce316769236f367043 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 14 Nov 2023 12:38:04 +0100 Subject: [PATCH 07/12] fix: diconnect before delete --- castai/cluster.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/castai/cluster.go b/castai/cluster.go index 9fa2bc32..e9201de4 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -26,6 +26,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, clusterId := data.Id() log.Printf("[INFO] Checking current status of the cluster.") + diconnectionTriggerd := false err := retry.RetryContext(ctx, data.Timeout(schema.TimeoutDelete), func() *retry.RetryError { clusterResponse, err := client.ExternalClusterAPIGetClusterWithResponse(ctx, clusterId) @@ -51,10 +52,27 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return retry.RetryableError(fmt.Errorf("triggered cluster deletion")) } - if agentStatus == sdk.ClusterAgentStatusDisconnected || agentStatus == "waiting-connection" { + if agentStatus == sdk.ClusterAgentStatusDisconnected { return triggerDelete() } + if agentStatus == "waiting-connection" { + response, err := client.ExternalClusterAPIDisconnectClusterWithResponse(ctx, clusterId, sdk.ExternalClusterAPIDisconnectClusterJSONRequestBody{ + DeleteProvisionedNodes: getOptionalBool(data, FieldDeleteNodesOnDisconnect, false), + KeepKubernetesResources: toPtr(true), + }) + + if checkErr := sdk.CheckOKResponse(response, err); checkErr != nil { + return retry.NonRetryableError(err) + } + + if diconnectionTriggerd { + return triggerDelete() + } + + diconnectionTriggerd = true + } + // If cluster doesn't have credentials we have to call delete cluster instead of disconnect because disconnect // will do nothing on cluster with empty credentials. if toString(clusterResponse.JSON200.CredentialsId) == "" { From 8d870b7e4ce3f58517312cf0b210a513a5b81156 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 14 Nov 2023 12:55:45 +0100 Subject: [PATCH 08/12] fix: diconnect before delete --- castai/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/castai/cluster.go b/castai/cluster.go index e9201de4..f369661b 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -47,7 +47,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, triggerDelete := func() *retry.RetryError { log.Printf("[INFO] Deleting cluster.") if err := sdk.CheckResponseNoContent(client.ExternalClusterAPIDeleteClusterWithResponse(ctx, clusterId)); err != nil { - return retry.NonRetryableError(err) + return retry.NonRetryableError(fmt.Errorf("cluster status %s agent status %s error: %w", clusterStatus, agentStatus, err)) } return retry.RetryableError(fmt.Errorf("triggered cluster deletion")) } From ba7e819cc7dbd7d9be846ad1dedb72a84e666fc2 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 14 Nov 2023 13:09:56 +0100 Subject: [PATCH 09/12] fix: diconnect before delete --- castai/cluster.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/castai/cluster.go b/castai/cluster.go index f369661b..f87df5e0 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -26,7 +26,6 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, clusterId := data.Id() log.Printf("[INFO] Checking current status of the cluster.") - diconnectionTriggerd := false err := retry.RetryContext(ctx, data.Timeout(schema.TimeoutDelete), func() *retry.RetryError { clusterResponse, err := client.ExternalClusterAPIGetClusterWithResponse(ctx, clusterId) @@ -65,12 +64,6 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, if checkErr := sdk.CheckOKResponse(response, err); checkErr != nil { return retry.NonRetryableError(err) } - - if diconnectionTriggerd { - return triggerDelete() - } - - diconnectionTriggerd = true } // If cluster doesn't have credentials we have to call delete cluster instead of disconnect because disconnect From bc4fd95a0a2d9901707ccac035f5867386cf55cc Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 14 Nov 2023 13:30:09 +0100 Subject: [PATCH 10/12] fix: diconnect before delete --- castai/cluster.go | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/castai/cluster.go b/castai/cluster.go index f87df5e0..693b64aa 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -43,10 +43,27 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return nil } + triggerDisconnect := func() *retry.RetryError { + response, err := client.ExternalClusterAPIDisconnectClusterWithResponse(ctx, clusterId, sdk.ExternalClusterAPIDisconnectClusterJSONRequestBody{ + DeleteProvisionedNodes: getOptionalBool(data, FieldDeleteNodesOnDisconnect, false), + KeepKubernetesResources: toPtr(true), + }) + if checkErr := sdk.CheckOKResponse(response, err); checkErr != nil { + return retry.NonRetryableError(err) + } + + return retry.RetryableError(fmt.Errorf("triggered agent disconnection cluster status %s agent status %s", clusterStatus, agentStatus)) + } + triggerDelete := func() *retry.RetryError { log.Printf("[INFO] Deleting cluster.") - if err := sdk.CheckResponseNoContent(client.ExternalClusterAPIDeleteClusterWithResponse(ctx, clusterId)); err != nil { - return retry.NonRetryableError(fmt.Errorf("cluster status %s agent status %s error: %w", clusterStatus, agentStatus, err)) + res, err := client.ExternalClusterAPIDeleteClusterWithResponse(ctx, clusterId) + if res.StatusCode() == 400 { + return triggerDisconnect() + } + + if chekErr := sdk.CheckResponseNoContent(res, err); chekErr != nil { + return retry.NonRetryableError(fmt.Errorf("error when deleting cluster status %s agent status %s error: %w", clusterStatus, agentStatus, err)) } return retry.RetryableError(fmt.Errorf("triggered cluster deletion")) } @@ -55,17 +72,6 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return triggerDelete() } - if agentStatus == "waiting-connection" { - response, err := client.ExternalClusterAPIDisconnectClusterWithResponse(ctx, clusterId, sdk.ExternalClusterAPIDisconnectClusterJSONRequestBody{ - DeleteProvisionedNodes: getOptionalBool(data, FieldDeleteNodesOnDisconnect, false), - KeepKubernetesResources: toPtr(true), - }) - - if checkErr := sdk.CheckOKResponse(response, err); checkErr != nil { - return retry.NonRetryableError(err) - } - } - // If cluster doesn't have credentials we have to call delete cluster instead of disconnect because disconnect // will do nothing on cluster with empty credentials. if toString(clusterResponse.JSON200.CredentialsId) == "" { @@ -86,15 +92,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, if toString(clusterResponse.JSON200.CredentialsId) != "" && agentStatus != sdk.ClusterAgentStatusDisconnected { log.Printf("[INFO] Disconnecting cluster.") - response, err := client.ExternalClusterAPIDisconnectClusterWithResponse(ctx, clusterId, sdk.ExternalClusterAPIDisconnectClusterJSONRequestBody{ - DeleteProvisionedNodes: getOptionalBool(data, FieldDeleteNodesOnDisconnect, false), - KeepKubernetesResources: toPtr(true), - }) - if checkErr := sdk.CheckOKResponse(response, err); checkErr != nil { - return retry.NonRetryableError(err) - } - - return retry.RetryableError(fmt.Errorf("triggered agent disconnection cluster status %s agent status %s", clusterStatus, agentStatus)) + return triggerDisconnect() } if agentStatus == sdk.ClusterAgentStatusDisconnected && clusterStatus != sdk.ClusterStatusDeleted { From a600f6fe3d04c7c3c6a7538e0b82b50d42bebcf2 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 14 Nov 2023 13:44:48 +0100 Subject: [PATCH 11/12] fix: diconnect before delete --- castai/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/castai/cluster.go b/castai/cluster.go index 693b64aa..0d348352 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -68,7 +68,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return retry.RetryableError(fmt.Errorf("triggered cluster deletion")) } - if agentStatus == sdk.ClusterAgentStatusDisconnected { + if agentStatus == sdk.ClusterAgentStatusDisconnected || clusterStatus == sdk.ClusterStatusDeleted { return triggerDelete() } From 43ee0088d4c74c106de9d17bbc070ba0ba294a34 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Tue, 14 Nov 2023 13:59:08 +0100 Subject: [PATCH 12/12] fix: diconnect before delete --- Makefile | 2 +- castai/cluster.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ec3c3926..fb3c1833 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ test: testacc: @echo "==> Running acceptance tests" - TF_ACC=1 go test ./castai/... '-run=^TestAcc' -v -timeout 10m + TF_ACC=1 go test ./castai/... '-run=^TestAcc' -v -timeout 16m validate-terraform-examples: for examples in examples/eks examples/gke examples/aks ; do \ diff --git a/castai/cluster.go b/castai/cluster.go index 0d348352..3ee1eaba 100644 --- a/castai/cluster.go +++ b/castai/cluster.go @@ -62,7 +62,7 @@ func resourceCastaiClusterDelete(ctx context.Context, data *schema.ResourceData, return triggerDisconnect() } - if chekErr := sdk.CheckResponseNoContent(res, err); chekErr != nil { + if checkErr := sdk.CheckResponseNoContent(res, err); checkErr != nil { return retry.NonRetryableError(fmt.Errorf("error when deleting cluster status %s agent status %s error: %w", clusterStatus, agentStatus, err)) } return retry.RetryableError(fmt.Errorf("triggered cluster deletion"))