Merge pull request #712 from fluxcd/oci-int-destroy-only

oci/int: Add separate resource cleanup step

.github/workflows/integration-azure.yaml

@@ -57,3 +57,15 @@ jobs:
           TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
           TF_VAR_wi_k8s_sa_name: test-workload-id
           TF_VAR_wi_k8s_sa_ns: default
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        run: . .env && make destroy-azure
+        env:
+          ARM_CLIENT_ID: ${{ secrets.OCI_E2E_AZ_ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.OCI_E2E_AZ_ARM_CLIENT_SECRET }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.OCI_E2E_AZ_ARM_SUBSCRIPTION_ID }}
+          ARM_TENANT_ID: ${{ secrets.OCI_E2E_AZ_ARM_TENANT_ID }}
+          TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }}
+          TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
+          TF_VAR_wi_k8s_sa_name: test-workload-id
+          TF_VAR_wi_k8s_sa_ns: default

.github/workflows/integration-gcp.yaml

@@ -71,3 +71,13 @@ jobs:
           TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
           TF_VAR_wi_k8s_sa_name: test-workload-id
           TF_VAR_wi_k8s_sa_ns: default
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        run: . .env && make destroy-gcp
+        env:
+          TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }}
+          TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }}
+          TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }}
+          TF_VAR_enable_wi: ${{ matrix.enable-workload-id }}
+          TF_VAR_wi_k8s_sa_name: test-workload-id
+          TF_VAR_wi_k8s_sa_ns: default

oci/tests/integration/Makefile

@@ -25,3 +25,15 @@ test-azure:
 
 test-gcp:
 	$(MAKE) test PROVIDER_ARG="-provider gcp"
+
+destroy:
+	go test -timeout $(TEST_TIMEOUT) -v ./ $(GO_TEST_ARGS) $(PROVIDER_ARG) -destroy-only --tags=integration
+
+destroy-aws:
+	$(MAKE) destroy PROVIDER_ARG="-provider aws"
+
+destroy-azure:
+	$(MAKE) destroy PROVIDER_ARG="-provider azure"
+
+destroy-gcp:
+	$(MAKE) destroy PROVIDER_ARG="-provider gcp"

oci/tests/integration/README.md

@@ -79,6 +79,10 @@ variables using
 use the terraform configuration below. Please make sure all the requirements of
 azure-gh-actions are followed before running it.
 
+**NOTE:** When running the following for a repo under an organization, set the
+environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the
+`github` provider doesn't work.
+
 ```hcl
 provider "github" {
   owner = "fluxcd"
@@ -189,6 +193,10 @@ variables using
 use the terraform configuration below. Please make sure all the requirements of
 gcp-gh-actions are followed before running it.
 
+**NOTE:** When running the following for a repo under an organization, set the
+environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the
+`github` provider doesn't work.
+
 ```hcl
 provider "google" {}
 
@@ -244,6 +252,14 @@ TEST_IMG=fluxcd/testapp:test go test -timeout 30m -v ./ -verbose -retain -provid
 ...
 ```
 
+If not configured explicitly to retain the infrastructure, at the end of the
+test, the test infrastructure is deleted. In case of any failure due to which
+the resources don't get deleted, the `make destroy-*` commands can be run for
+the respective provider. This will run terraform destroy in the respective
+provider's terraform configuration directory. This can be used to quickly
+destroy the infrastructure without going through the provision-test-destroy
+steps.
+
 ## Workload Identity
 
 By default, the tests use node identity for authentication. To run the integration tests on clusters with workload identity

oci/tests/integration/go.mod

@@ -6,8 +6,9 @@ replace github.com/fluxcd/pkg/oci => ../../
 
 require (
 	github.com/fluxcd/pkg/oci v0.32.0
-	github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22
+	github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa
 	github.com/google/go-containerregistry v0.17.0
+	github.com/hashicorp/terraform-exec v0.18.1
 	github.com/hashicorp/terraform-json v0.17.1
 	github.com/onsi/gomega v1.30.0
 	k8s.io/api v0.28.4
@@ -65,7 +66,6 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hc-install v0.5.1 // indirect
-	github.com/hashicorp/terraform-exec v0.18.1 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

oci/tests/integration/go.sum

@@ -71,8 +71,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc
 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
 github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
 github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22 h1:1f0EQM0kPX2px9FanVMyD/UrHFQ9zqFg58M42Y8bBts=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230720084205-d40ee5473f22/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa h1:JdI+rVwGF5gBYt+UBijOVzXtq7aAU80vgksMNXSCCfU=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=

oci/tests/integration/suite_test.go

@@ -29,6 +29,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/terraform-exec/tfexec"
 	tfjson "github.com/hashicorp/terraform-json"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -80,6 +81,9 @@ var (
 	// verbose flag to enable output of terraform execution.
 	verbose = flag.Bool("verbose", false, "verbose output of the environment setup")
 
+	// destroyOnly flag to destroy any provisioned infrastructure.
+	destroyOnly = flag.Bool("destroy-only", false, "run in destroy-only mode and delete any existing infrastructure")
+
 	// testRepos is a map of registry common name and URL of the test
 	// repositories. This is used as the test cases to run the tests against.
 	// The registry common name need not be the actual registry address but an
@@ -153,15 +157,6 @@ func TestMain(m *testing.M) {
 	flag.Parse()
 	ctx := context.TODO()
 
-	appImg := os.Getenv("TEST_IMG")
-	if appImg == "" {
-		log.Fatal("TEST_IMG must be set to the test application image, cannot be empty")
-	}
-
-	localImgs := map[string]string{
-		"app": appImg,
-	}
-
 	// Validate the provider.
 	if *targetProvider == "" {
 		log.Fatalf("-provider flag must be set to one of %v", supportedProviders)
@@ -181,6 +176,30 @@ func TestMain(m *testing.M) {
 		log.Fatalf("Failed to get provider config for %q", *targetProvider)
 	}
 
+	// Run destroy-only mode if enabled.
+	if *destroyOnly {
+		log.Println("Running in destroy-only mode...")
+		envOpts := []tftestenv.EnvironmentOption{
+			tftestenv.WithVerbose(*verbose),
+			// Ignore any state lock in destroy-only mode.
+			tftestenv.WithTfDestroyOptions(tfexec.Lock(false)),
+		}
+		if err := tftestenv.Destroy(ctx, providerCfg.terraformPath, envOpts...); err != nil {
+			panic(err)
+		}
+		os.Exit(0)
+	}
+
+	// Check the test app image.
+	appImg := os.Getenv("TEST_IMG")
+	if appImg == "" {
+		log.Fatal("TEST_IMG must be set to the test application image, cannot be empty")
+	}
+
+	localImgs := map[string]string{
+		"app": appImg,
+	}
+
 	// Construct scheme to be added to the kubeclient.
 	scheme := runtime.NewScheme()