test: Refactor smoke tests and add improve nightly workflow (#1566)

* refactor smoke tests and update Makefile and nightly_smoke_tests.yml * clean up output on smoke tests and add slack notifications in nightly_smoke_tests.yml * gha test 1 * gha test 2 * Fix makefile smoke-test command * add option to input sha value in workflow file * set env for nightly_smoke_tests.yml * add conditions to cron in nightly_smoke_tests.yml * fix syntax for slack message * setting pipefail status before piping output * set to bash terminal instead of sh * set to bash terminal instead of sh * Change slack notification * resolve gha issues * set slack webhook env * Change slack message * Change slack message * Fix slack payload * Fix slack payload # 2 * Fix slack payload # 3 * Fix slack payload # 3 * revert slack gha package * add submodule checks before running integration and smoke tests * addressing pr comment to remove global smoke test map variable for simplicity * fix syntax in slack message * Change slack notification * Add repo name in payload * Add repo name in slack payload * bump slackgha version and add slack notification to integration workflow * fix typo in workflow * add condition to send test results and notification only on main repository
linode · Sep 26, 2024 · 7d781d5 · 7d781d5
1 parent 3108a2e
commit 7d781d5
Show file tree

Hide file tree

Showing 19 changed files with 377 additions and 10 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -81,7 +81,7 @@ jobs:
   process-upload-report:
     runs-on: ubuntu-latest
     needs: [integration_tests]
-    if: always() # Run even if previous job fails
+    if: always() && github.repository == 'linode/terraform-provider-linode' # Run even if integration tests fail and only on main repository
 
     steps:
       - name: Checkout code
@@ -145,4 +145,70 @@ jobs:
         run: |
           cd e2e_scripts/cloud_security_scripts/lke_calico_rules/ && ./lke_calico_rules_e2e.sh
         env:
-          LINODE_TOKEN: ${{ secrets.LINODE_TOKEN_USER_4 }}
+          LINODE_TOKEN: ${{ secrets.LINODE_TOKEN_USER_4 }}
+
+  notify-slack:
+    runs-on: ubuntu-latest
+    needs: [integration_tests]
+    if: always() && github.repository == 'linode/terraform-provider-linode' # Run even if integration tests fail and only on main repository
+
+    steps:
+      - name: Notify Slack
+        uses: slackapi/[email protected]
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Build Result:*\n${{ steps.integration_tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch:*\n`${{ github.ref_name }}`"
+                    }
+                  ]
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
+                    }
+                  ]
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`"
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.github/workflows/nightly_smoke_tests.yml b/.github/workflows/nightly_smoke_tests.yml
@@ -4,25 +4,99 @@ on:
   schedule:
     - cron: "0 0 * * *"
   workflow_dispatch:
+    inputs:
+      sha:
+        description: 'Commit SHA to test'
+        required: false
+        default: ''
+        type: string
 
 jobs:
   smoke_tests:
+    if: github.repository == 'linode/terraform-provider-linode' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: dev
+          fetch-depth: 0
+          submodules: 'recursive'
+          ref: ${{ github.event.inputs.sha || github.ref }}
 
-      - name: Set up go
+      - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 'stable'
+          go-version: '1.x'
 
-      - run: go version
+      - name: Install Dependencies
+        run: |
+          make deps
 
       - name: Run smoke tests
-        run: make smoke-test
+        id: smoke_tests
+        run: |
+          make smoke-test
         env:
           LINODE_TOKEN: ${{ secrets.DX_LINODE_TOKEN }}
+
+      - name: Notify Slack
+        if: always()
+        uses: slackapi/[email protected]
+        with:
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Build Result:*\n${{ steps.smoke_tests.outcome == 'success' && ':large_green_circle: Build Passed' || ':red_circle: Build Failed' }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Branch:*\n`${{ github.ref_name }}`"
+                    }
+                  ]
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit Hash:*\n<${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Run URL:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
+                    }
+                  ]
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "context",
+                  "elements": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "Triggered by: :bust_in_silhouette: `${{ github.actor }}`"
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/Makefile b/Makefile
@@ -78,16 +78,34 @@ include-env: $(IP_ENV_FILE)
 
 generate-ip-env-fw-e2e: $(IP_ENV_FILE)
 
+SUBMODULE_DIR := e2e_scripts
+
 $(IP_ENV_FILE):
 	# Generate env file for E2E cloud firewall
-	. ./e2e_scripts/cloud_security_scripts/cloud_e2e_firewall/terraform-provider-linode/generate_ip_env_fw_e2e.sh || touch $(IP_ENV_FILE)
+	@if [ ! -d $(SUBMODULE_DIR) ]; then \
+		echo "Submodule directory $(SUBMODULE_DIR) does not exist. Updating submodules..."; \
+		git submodule update --init --recursive; \
+	else \
+		echo "Submodule directory $(SUBMODULE_DIR) already exists. Skipping update."; \
+	fi
+	. ./e2e_scripts/cloud_security_scripts/cloud_e2e_firewall/terraform-provider-linode/generate_ip_env_fw_e2e.sh
 
 .PHONY: smoke-test
-smoke-test: fmt-check
+smoke-test: fmt-check generate-ip-env-fw-e2e include-env
 	TF_ACC=1 \
 	LINODE_API_VERSION="v4beta" \
 	RUN_LONG_TESTS=$(RUN_LONG_TESTS) \
-	go test -v -run smoke ./linode/... -count $(COUNT) -timeout $(TIMEOUT) -parallel=$(PARALLEL) -ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc"
+	TF_VAR_ipv4_addr=${PUBLIC_IPV4} \
+	TF_VAR_ipv6_addr=${PUBLIC_IPV6} \
+	bash -c 'set -o pipefail && go test -v ./linode/... -run TestSmokeTests -tags=integration \
+		-count $(COUNT) \
+		-timeout $(TIMEOUT) \
+		-parallel=$(PARALLEL) \
+		-ldflags="-X=github.com/linode/terraform-provider-linode/v2/version.ProviderVersion=acc" \
+		| sed -e "/testing: warning: no tests to run/,+1d" -e "/\[no test files\]/d" -e "/\[no tests to run\]/d"; \
+	exit_status=$$?; \
+	exit $$exit_status'
+
 
 .PHONY: docs-check
 docs-check:

diff --git a/linode/childaccount/datasource_test.go b/linode/childaccount/datasource_test.go
@@ -10,6 +10,19 @@ import (
 	"github.com/linode/terraform-provider-linode/v2/linode/childaccount/tmpl"
 )
 
+func TestSmokeTests_childaccount(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccDataSourceChildAccount_basic_smoke", TestAccDataSourceChildAccount_basic_smoke},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccDataSourceChildAccount_basic_smoke(t *testing.T) {
 	t.Parallel()
 

diff --git a/linode/databasepostgresql/resource_test.go b/linode/databasepostgresql/resource_test.go
@@ -75,6 +75,19 @@ func sweep(prefix string) error {
 	return nil
 }
 
+func TestSmokeTests_databasepostgresql(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccResourceDatabasePostgres_basic_smoke", TestAccResourceDatabasePostgres_basic_smoke},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccResourceDatabasePostgres_basic_smoke(t *testing.T) {
 	acceptance.LongRunningTest(t)
 

diff --git a/linode/domain/datasource_test.go b/linode/domain/datasource_test.go
@@ -12,6 +12,19 @@ import (
 	"github.com/linode/terraform-provider-linode/v2/linode/domain/tmpl"
 )
 
+func TestSmokeTests_domain_datasource(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccDataSourceDomain_basic_smoke", TestAccDataSourceDomain_basic_smoke},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccDataSourceDomain_basic_smoke(t *testing.T) {
 	t.Parallel()
 

diff --git a/linode/domain/resource_test.go b/linode/domain/resource_test.go
@@ -48,6 +48,19 @@ func sweep(prefix string) error {
 	return nil
 }
 
+func TestSmokeTests_domain_resource(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccResourceDomain_basic_smoke", TestAccResourceDomain_basic_smoke},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccResourceDomain_basic_smoke(t *testing.T) {
 	t.Parallel()
 

diff --git a/linode/firewall/framework_resource_test.go b/linode/firewall/framework_resource_test.go
@@ -57,6 +57,19 @@ func sweep(prefix string) error {
 	return nil
 }
 
+func TestSmokeTests_firewall(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccLinodeFirewall_basic", TestAccLinodeFirewall_basic},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccLinodeFirewall_basic(t *testing.T) {
 	t.Parallel()
 

diff --git a/linode/firewalldevice/resource_test.go b/linode/firewalldevice/resource_test.go
@@ -29,6 +29,19 @@ func init() {
 	testRegion = region
 }
 
+func TestSmokeTests_firewalldevice(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccResourceFirewallDevice_basic_smoke", TestAccResourceFirewallDevice_basic_smoke},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccResourceFirewallDevice_basic_smoke(t *testing.T) {
 	t.Parallel()
 

diff --git a/linode/instance/resource_test.go b/linode/instance/resource_test.go
@@ -64,6 +64,19 @@ func sweep(prefix string) error {
 	return nil
 }
 
+func TestSmokeTests_instance(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccResourceInstance_basic_smoke", TestAccResourceInstance_basic_smoke},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccResourceInstance_basic_smoke(t *testing.T) {
 	t.Parallel()
 

diff --git a/linode/instancedisk/resource_test.go b/linode/instancedisk/resource_test.go
@@ -29,6 +29,19 @@ func init() {
 	testRegion = region
 }
 
+func TestSmokeTests_instancedisk(t *testing.T) {
+	tests := []struct {
+		name string
+		test func(*testing.T)
+	}{
+		{"TestAccResourceInstanceDisk_basic_smoke", TestAccResourceInstanceDisk_basic_smoke},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, tt.test)
+	}
+}
+
 func TestAccResourceInstanceDisk_basic_smoke(t *testing.T) {
 	t.Parallel()