Revert "e2e: remove skip-undeploy flag and go cleanup logic"

This partially reverts commit 213a7e5.

The release test action does not have a cleanup step, and implementing
one would be difficult because it uses the default namespace which we
can't just delete. Instead, we add the skip-undeploy flag again, flip
its default so that most tests are not cleaned up from go and add the
flag to the release e2e.

.github/workflows/release.yml

@@ -417,7 +417,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          nix shell -L .#contrast.e2e --command release.test -test.v --tag ${{ inputs.version }} --platform ${{ matrix.platform.name }}
+          nix shell -L .#contrast.e2e --command release.test -test.v --tag ${{ inputs.version }} --platform ${{ matrix.platform.name }} --skip-undeploy=false
 
   create-github-stuff:
     name: Create backport label and milestone

e2e/internal/contrasttest/contrasttest.go

@@ -41,6 +41,7 @@ type testFlags struct {
 	ImageReplacementsFile string
 	NamespaceFile         string
 	NamespaceSuffix       string
+	SkipUndeploy          bool
 }
 
 // RegisterFlags registers the flags that are shared between all tests.
@@ -49,6 +50,7 @@ func RegisterFlags() {
 	flag.StringVar(&Flags.NamespaceFile, "namespace-file", "", "file to store the namespace in")
 	flag.StringVar(&Flags.NamespaceSuffix, "namespace-suffix", "", "suffix to append to the namespace")
 	flag.StringVar(&Flags.PlatformStr, "platform", "", "Deployment platform")
+	flag.BoolVar(&Flags.SkipUndeploy, "skip-undeploy", true, "Skip undeploying the test namespace")
 }
 
 // ContrastTest is the Contrast test helper struct.
@@ -60,6 +62,7 @@ type ContrastTest struct {
 	ImageReplacementsFile string
 	Platform              platforms.Platform
 	NamespaceFile         string
+	SkipUndeploy          bool
 	Kubeclient            *kubeclient.Kubeclient
 
 	// outputs of contrast subcommands
@@ -78,6 +81,7 @@ func New(t *testing.T) *ContrastTest {
 		ImageReplacementsFile: Flags.ImageReplacementsFile,
 		Platform:              platform,
 		NamespaceFile:         Flags.NamespaceFile,
+		SkipUndeploy:          Flags.SkipUndeploy,
 		Kubeclient:            kubeclient.NewForTest(t),
 	}
 }
@@ -131,6 +135,20 @@ func (ct *ContrastTest) Init(t *testing.T, resources []any) {
 			ct.Kubeclient.LogDebugInfo(ctx)
 		}
 
+		if !ct.SkipUndeploy {
+			// Deleting the namespace sometimes fails when the cluster is
+			// unavailable (e.g. after a K3s restart). Retry deleting for up to
+			// 30 seconds.
+			for range 30 {
+				if err := ct.Kubeclient.Delete(ctx, namespace...); err != nil {
+					t.Logf("Could not delete namespace %q: %v", ct.Namespace, err)
+					time.Sleep(1 * time.Second)
+				} else {
+					break
+				}
+			}
+		}
+
 		if fifo != nil {
 			if err := fifo.Done(ctx); err != nil {
 				t.Logf("Could not mark fifo ticket as done: %v", err)